Repository: oozie Updated Branches: refs/heads/pya [created] a37835fec
http://git-wip-us.apache.org/repos/asf/oozie/blob/a37835fe/hadooplibs/hadoop-utils-0.23/src/main/java/org/apache/oozie/action/hadoop/LauncherMainHadoopUtils.java ---------------------------------------------------------------------- diff --git a/hadooplibs/hadoop-utils-0.23/src/main/java/org/apache/oozie/action/hadoop/LauncherMainHadoopUtils.java b/hadooplibs/hadoop-utils-0.23/src/main/java/org/apache/oozie/action/hadoop/LauncherMainHadoopUtils.java deleted file mode 100644 index dca7820..0000000 --- a/hadooplibs/hadoop-utils-0.23/src/main/java/org/apache/oozie/action/hadoop/LauncherMainHadoopUtils.java +++ /dev/null @@ -1,35 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.oozie.action.hadoop; - -import org.apache.hadoop.conf.Configuration; - - -public class LauncherMainHadoopUtils { - - public static final String CHILD_MAPREDUCE_JOB_TAGS = "oozie.child.mapreduce.job.tags"; - public static final String OOZIE_JOB_LAUNCH_TIME = "oozie.job.launch.time"; - - private LauncherMainHadoopUtils() { - } - - public static void killChildYarnJobs(Configuration actionConf) { - // no-op - } -} http://git-wip-us.apache.org/repos/asf/oozie/blob/a37835fe/hadooplibs/hadoop-utils-0.23/src/main/java/org/apache/oozie/hadoop/utils/HadoopShims.java ---------------------------------------------------------------------- diff --git a/hadooplibs/hadoop-utils-0.23/src/main/java/org/apache/oozie/hadoop/utils/HadoopShims.java b/hadooplibs/hadoop-utils-0.23/src/main/java/org/apache/oozie/hadoop/utils/HadoopShims.java deleted file mode 100644 index 799dffb..0000000 --- a/hadooplibs/hadoop-utils-0.23/src/main/java/org/apache/oozie/hadoop/utils/HadoopShims.java +++ /dev/null @@ -1,51 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.oozie.hadoop.utils; - -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import java.io.IOException; - -public class HadoopShims { - FileSystem fs; - - public HadoopShims(FileSystem fs) { - this.fs = fs; - } - - public static boolean isSymlinkSupported() { - return false; - } - - public Path getSymLinkTarget(Path p) throws IOException { - return p; - } - - public boolean isSymlink(Path p) throws IOException { - return false; - } - - public void createSymlink(Path target, Path link, boolean createParent) throws IOException { - } - - public static boolean isYARN() { - return false; - } - -} http://git-wip-us.apache.org/repos/asf/oozie/blob/a37835fe/hadooplibs/hadoop-utils-1/pom.xml ---------------------------------------------------------------------- diff --git a/hadooplibs/hadoop-utils-1/pom.xml b/hadooplibs/hadoop-utils-1/pom.xml deleted file mode 100644 index 26f1ed0..0000000 --- a/hadooplibs/hadoop-utils-1/pom.xml +++ /dev/null @@ -1,41 +0,0 @@ -<?xml version="1.0" encoding="UTF-8"?> -<!-- - Licensed to the Apache Software Foundation (ASF) under one - or more contributor license agreements. See the NOTICE file - distributed with this work for additional information - regarding copyright ownership. The ASF licenses this file - to you under the Apache License, Version 2.0 (the - "License"); you may not use this file except in compliance - with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. ---> -<project xmlns="http://maven.apache.org/POM/4.0.0"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; - xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd";> - <modelVersion>4.0.0</modelVersion> - <parent> - <groupId>org.apache.oozie</groupId> - <artifactId>oozie-main</artifactId> - <version>4.3.0-SNAPSHOT</version> - <relativePath>../../pom.xml</relativePath> - </parent> - <groupId>org.apache.oozie</groupId> - <artifactId>oozie-hadoop-utils</artifactId> - <version>hadoop-1-4.3.0-SNAPSHOT</version> - <description>Apache Oozie Hadoop Utils</description> - <name>Apache Oozie Hadoop Utils</name> - <packaging>jar</packaging> - - <dependencies> - <dependency> - <groupId>org.apache.hadoop</groupId> - <artifactId>hadoop-client</artifactId> - </dependency> - </dependencies> -</project> http://git-wip-us.apache.org/repos/asf/oozie/blob/a37835fe/hadooplibs/hadoop-utils-1/src/main/java/org/apache/oozie/action/hadoop/LauncherMainHadoopUtils.java ---------------------------------------------------------------------- diff --git a/hadooplibs/hadoop-utils-1/src/main/java/org/apache/oozie/action/hadoop/LauncherMainHadoopUtils.java b/hadooplibs/hadoop-utils-1/src/main/java/org/apache/oozie/action/hadoop/LauncherMainHadoopUtils.java deleted file mode 100644 index dca7820..0000000 --- a/hadooplibs/hadoop-utils-1/src/main/java/org/apache/oozie/action/hadoop/LauncherMainHadoopUtils.java +++ /dev/null @@ -1,35 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.oozie.action.hadoop; - -import org.apache.hadoop.conf.Configuration; - - -public class LauncherMainHadoopUtils { - - public static final String CHILD_MAPREDUCE_JOB_TAGS = "oozie.child.mapreduce.job.tags"; - public static final String OOZIE_JOB_LAUNCH_TIME = "oozie.job.launch.time"; - - private LauncherMainHadoopUtils() { - } - - public static void killChildYarnJobs(Configuration actionConf) { - // no-op - } -} http://git-wip-us.apache.org/repos/asf/oozie/blob/a37835fe/hadooplibs/hadoop-utils-1/src/main/java/org/apache/oozie/hadoop/utils/HadoopShims.java ---------------------------------------------------------------------- diff --git a/hadooplibs/hadoop-utils-1/src/main/java/org/apache/oozie/hadoop/utils/HadoopShims.java b/hadooplibs/hadoop-utils-1/src/main/java/org/apache/oozie/hadoop/utils/HadoopShims.java deleted file mode 100644 index 799dffb..0000000 --- a/hadooplibs/hadoop-utils-1/src/main/java/org/apache/oozie/hadoop/utils/HadoopShims.java +++ /dev/null @@ -1,51 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.oozie.hadoop.utils; - -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import java.io.IOException; - -public class HadoopShims { - FileSystem fs; - - public HadoopShims(FileSystem fs) { - this.fs = fs; - } - - public static boolean isSymlinkSupported() { - return false; - } - - public Path getSymLinkTarget(Path p) throws IOException { - return p; - } - - public boolean isSymlink(Path p) throws IOException { - return false; - } - - public void createSymlink(Path target, Path link, boolean createParent) throws IOException { - } - - public static boolean isYARN() { - return false; - } - -} http://git-wip-us.apache.org/repos/asf/oozie/blob/a37835fe/hadooplibs/hadoop-utils-2/pom.xml ---------------------------------------------------------------------- diff --git a/hadooplibs/hadoop-utils-2/pom.xml b/hadooplibs/hadoop-utils-2/pom.xml deleted file mode 100644 index a20426d..0000000 --- a/hadooplibs/hadoop-utils-2/pom.xml +++ /dev/null @@ -1,42 +0,0 @@ -<?xml version="1.0" encoding="UTF-8"?> -<!-- - Licensed to the Apache Software Foundation (ASF) under one - or more contributor license agreements. See the NOTICE file - distributed with this work for additional information - regarding copyright ownership. The ASF licenses this file - to you under the Apache License, Version 2.0 (the - "License"); you may not use this file except in compliance - with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. ---> -<project xmlns="http://maven.apache.org/POM/4.0.0"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; - xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd";> - <modelVersion>4.0.0</modelVersion> - <parent> - <groupId>org.apache.oozie</groupId> - <artifactId>oozie-main</artifactId> - <version>4.3.0-SNAPSHOT</version> - <relativePath>../../pom.xml</relativePath> - </parent> - <groupId>org.apache.oozie</groupId> - <artifactId>oozie-hadoop-utils</artifactId> - <version>hadoop-2-4.3.0-SNAPSHOT</version> - <description>Apache Oozie Hadoop Utils ${project.version}</description> - <name>Apache Oozie Hadoop Utils ${project.version}</name> - <packaging>jar</packaging> - - <dependencies> - <dependency> - <groupId>org.apache.hadoop</groupId> - <artifactId>hadoop-client</artifactId> - <scope>provided</scope> - </dependency> - </dependencies> -</project> http://git-wip-us.apache.org/repos/asf/oozie/blob/a37835fe/hadooplibs/hadoop-utils-2/src/main/java/org/apache/oozie/action/hadoop/LauncherMainHadoopUtils.java ---------------------------------------------------------------------- diff --git a/hadooplibs/hadoop-utils-2/src/main/java/org/apache/oozie/action/hadoop/LauncherMainHadoopUtils.java b/hadooplibs/hadoop-utils-2/src/main/java/org/apache/oozie/action/hadoop/LauncherMainHadoopUtils.java deleted file mode 100644 index ce8c14f..0000000 --- a/hadooplibs/hadoop-utils-2/src/main/java/org/apache/oozie/action/hadoop/LauncherMainHadoopUtils.java +++ /dev/null @@ -1,118 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.oozie.action.hadoop; - -import java.io.IOException; -import java.lang.String; -import java.util.Collections; -import java.util.HashSet; -import java.util.List; -import java.util.Set; - -import org.apache.commons.lang.StringUtils; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.yarn.api.ApplicationClientProtocol; -import org.apache.hadoop.yarn.api.protocolrecords.ApplicationsRequestScope; -import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsRequest; -import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsResponse; -import org.apache.hadoop.yarn.api.records.ApplicationId; -import org.apache.hadoop.yarn.api.records.ApplicationReport; -import org.apache.hadoop.yarn.client.ClientRMProxy; -import org.apache.hadoop.yarn.client.api.YarnClient; -import org.apache.hadoop.yarn.exceptions.YarnException; - -public class LauncherMainHadoopUtils { - - public static final String CHILD_MAPREDUCE_JOB_TAGS = "oozie.child.mapreduce.job.tags"; - public static final String OOZIE_JOB_LAUNCH_TIME = "oozie.job.launch.time"; - - private LauncherMainHadoopUtils() { - } - - private static Set<ApplicationId> getChildYarnJobs(Configuration actionConf) { - System.out.println("Fetching child yarn jobs"); - Set<ApplicationId> childYarnJobs = new HashSet<ApplicationId>(); - String tag = actionConf.get(CHILD_MAPREDUCE_JOB_TAGS); - if (tag == null) { - System.out.print("Could not find Yarn tags property " + CHILD_MAPREDUCE_JOB_TAGS); - return childYarnJobs; - } - System.out.println("tag id : " + tag); - long startTime = 0L; - try { - startTime = Long.parseLong(System.getProperty(OOZIE_JOB_LAUNCH_TIME)); - } catch(NumberFormatException nfe) { - throw new RuntimeException("Could not find Oozie job launch time", nfe); - } - - GetApplicationsRequest gar = GetApplicationsRequest.newInstance(); - gar.setScope(ApplicationsRequestScope.OWN); - gar.setApplicationTags(Collections.singleton(tag)); - long endTime = System.currentTimeMillis(); - if (startTime > endTime) { - System.out.println("WARNING: Clock skew between the Oozie server host and this host detected. Please fix this. " + - "Attempting to work around..."); - // We don't know which one is wrong (relative to the RM), so to be safe, let's assume they're both wrong and add an - // offset in both directions - long diff = 2 * (startTime - endTime); - startTime = startTime - diff; - endTime = endTime + diff; - } - gar.setStartRange(startTime, endTime); - try { - ApplicationClientProtocol proxy = ClientRMProxy.createRMProxy(actionConf, ApplicationClientProtocol.class); - GetApplicationsResponse apps = proxy.getApplications(gar); - List<ApplicationReport> appsList = apps.getApplicationList(); - for(ApplicationReport appReport : appsList) { - childYarnJobs.add(appReport.getApplicationId()); - } - } catch (IOException ioe) { - throw new RuntimeException("Exception occurred while finding child jobs", ioe); - } catch (YarnException ye) { - throw new RuntimeException("Exception occurred while finding child jobs", ye); - } - - System.out.println("Child yarn jobs are found - " + StringUtils.join(childYarnJobs, ",")); - return childYarnJobs; - } - - public static void killChildYarnJobs(Configuration actionConf) { - try { - Set<ApplicationId> childYarnJobs = getChildYarnJobs(actionConf); - if (!childYarnJobs.isEmpty()) { - System.out.println(); - System.out.println("Found [" + childYarnJobs.size() + "] Map-Reduce jobs from this launcher"); - System.out.println("Killing existing jobs and starting over:"); - YarnClient yarnClient = YarnClient.createYarnClient(); - yarnClient.init(actionConf); - yarnClient.start(); - for (ApplicationId app : childYarnJobs) { - System.out.print("Killing job [" + app + "] ... "); - yarnClient.killApplication(app); - System.out.println("Done"); - } - System.out.println(); - } - } catch (YarnException ye) { - throw new RuntimeException("Exception occurred while killing child job(s)", ye); - } catch (IOException ioe) { - throw new RuntimeException("Exception occurred while killing child job(s)", ioe); - } - } -} http://git-wip-us.apache.org/repos/asf/oozie/blob/a37835fe/hadooplibs/hadoop-utils-2/src/main/java/org/apache/oozie/hadoop/utils/HadoopShims.java ---------------------------------------------------------------------- diff --git a/hadooplibs/hadoop-utils-2/src/main/java/org/apache/oozie/hadoop/utils/HadoopShims.java b/hadooplibs/hadoop-utils-2/src/main/java/org/apache/oozie/hadoop/utils/HadoopShims.java deleted file mode 100644 index c21884c..0000000 --- a/hadooplibs/hadoop-utils-2/src/main/java/org/apache/oozie/hadoop/utils/HadoopShims.java +++ /dev/null @@ -1,68 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.oozie.hadoop.utils; - -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; - -import java.io.IOException; -import java.net.URI; - -public class HadoopShims { - FileSystem fs; - - public HadoopShims(FileSystem fs) { - this.fs = fs; - } - - public static boolean isSymlinkSupported() { - return true; - } - - public Path getSymLinkTarget(Path p) throws IOException { - try { - //getSymlink doesn't work with fragment name, need to remove fragment before calling getSymlink - Path tempPath = new URI(p.toString()).getFragment() == null ? p : new Path(new URI(p.toString()).getPath()); - return fs.getFileLinkStatus(tempPath).getSymlink(); - } - catch (java.net.URISyntaxException e) { - throw new IOException(e); - } - } - - public boolean isSymlink(Path p) throws IOException { - try { - //isSymlink doesn't work with fragment name, need to remove fragment before checking for symlink - Path tempPath = new URI(p.toString()).getFragment() == null ? p : new Path(new URI(p.toString()).getPath()); - return fs.getFileLinkStatus(tempPath).isSymlink(); - } - catch (java.net.URISyntaxException e) { - throw new IOException(e); - } - } - - public void createSymlink(Path target, Path link, boolean createParent) throws IOException { - fs.createSymlink(target, link, createParent); - } - - public static boolean isYARN() { - return true; - } - -} http://git-wip-us.apache.org/repos/asf/oozie/blob/a37835fe/hadooplibs/hadoop-utils-3/pom.xml ---------------------------------------------------------------------- diff --git a/hadooplibs/hadoop-utils-3/pom.xml b/hadooplibs/hadoop-utils-3/pom.xml deleted file mode 100644 index 4013164..0000000 --- a/hadooplibs/hadoop-utils-3/pom.xml +++ /dev/null @@ -1,42 +0,0 @@ -<?xml version="1.0" encoding="UTF-8"?> -<!-- - Licensed to the Apache Software Foundation (ASF) under one - or more contributor license agreements. See the NOTICE file - distributed with this work for additional information - regarding copyright ownership. The ASF licenses this file - to you under the Apache License, Version 2.0 (the - "License"); you may not use this file except in compliance - with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. ---> -<project xmlns="http://maven.apache.org/POM/4.0.0"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; - xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd";> - <modelVersion>4.0.0</modelVersion> - <parent> - <groupId>org.apache.oozie</groupId> - <artifactId>oozie-main</artifactId> - <version>4.3.0-SNAPSHOT</version> - <relativePath>../../pom.xml</relativePath> - </parent> - <groupId>org.apache.oozie</groupId> - <artifactId>oozie-hadoop-utils</artifactId> - <version>hadoop-3-4.3.0-SNAPSHOT</version> - <description>Apache Oozie Hadoop Utils ${project.version}</description> - <name>Apache Oozie Hadoop Utils ${project.version}</name> - <packaging>jar</packaging> - - <dependencies> - <dependency> - <groupId>org.apache.hadoop</groupId> - <artifactId>hadoop-client</artifactId> - <scope>provided</scope> - </dependency> - </dependencies> -</project> http://git-wip-us.apache.org/repos/asf/oozie/blob/a37835fe/hadooplibs/hadoop-utils-3/src/main/java/org/apache/oozie/action/hadoop/LauncherMainHadoopUtils.java ---------------------------------------------------------------------- diff --git a/hadooplibs/hadoop-utils-3/src/main/java/org/apache/oozie/action/hadoop/LauncherMainHadoopUtils.java b/hadooplibs/hadoop-utils-3/src/main/java/org/apache/oozie/action/hadoop/LauncherMainHadoopUtils.java deleted file mode 100644 index 94e01ea..0000000 --- a/hadooplibs/hadoop-utils-3/src/main/java/org/apache/oozie/action/hadoop/LauncherMainHadoopUtils.java +++ /dev/null @@ -1,115 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.oozie.action.hadoop; - -import java.io.IOException; -import java.util.Collections; -import java.util.HashSet; -import java.util.List; -import java.util.Set; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.yarn.api.ApplicationClientProtocol; -import org.apache.hadoop.yarn.api.protocolrecords.ApplicationsRequestScope; -import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsRequest; -import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsResponse; -import org.apache.hadoop.yarn.api.records.ApplicationId; -import org.apache.hadoop.yarn.api.records.ApplicationReport; -import org.apache.hadoop.yarn.client.ClientRMProxy; -import org.apache.hadoop.yarn.client.api.YarnClient; -import org.apache.hadoop.yarn.exceptions.YarnException; - -public class LauncherMainHadoopUtils { - - public static final String CHILD_MAPREDUCE_JOB_TAGS = "oozie.child.mapreduce.job.tags"; - public static final String OOZIE_JOB_LAUNCH_TIME = "oozie.job.launch.time"; - - private LauncherMainHadoopUtils() { - } - - private static Set<ApplicationId> getChildYarnJobs(Configuration actionConf) { - System.out.println("Fetching child yarn jobs"); - Set<ApplicationId> childYarnJobs = new HashSet<ApplicationId>(); - String tag = actionConf.get(CHILD_MAPREDUCE_JOB_TAGS); - if (tag == null) { - System.out.print("Could not find Yarn tags property " + CHILD_MAPREDUCE_JOB_TAGS); - return childYarnJobs; - } - System.out.println("tag id : " + tag); - long startTime = 0L; - try { - startTime = Long.parseLong((System.getProperty(OOZIE_JOB_LAUNCH_TIME))); - } catch(NumberFormatException nfe) { - throw new RuntimeException("Could not find Oozie job launch time", nfe); - } - - GetApplicationsRequest gar = GetApplicationsRequest.newInstance(); - gar.setScope(ApplicationsRequestScope.OWN); - gar.setApplicationTags(Collections.singleton(tag)); - long endTime = System.currentTimeMillis(); - if (startTime > endTime) { - System.out.println("WARNING: Clock skew between the Oozie server host and this host detected. Please fix this. " + - "Attempting to work around..."); - // We don't know which one is wrong (relative to the RM), so to be safe, let's assume they're both wrong and add an - // offset in both directions - long diff = 2 * (startTime - endTime); - startTime = startTime - diff; - endTime = endTime + diff; - } - gar.setStartRange(startTime, endTime); - try { - ApplicationClientProtocol proxy = ClientRMProxy.createRMProxy(actionConf, ApplicationClientProtocol.class); - GetApplicationsResponse apps = proxy.getApplications(gar); - List<ApplicationReport> appsList = apps.getApplicationList(); - for(ApplicationReport appReport : appsList) { - childYarnJobs.add(appReport.getApplicationId()); - } - } catch (IOException ioe) { - throw new RuntimeException("Exception occurred while finding child jobs", ioe); - } catch (YarnException ye) { - throw new RuntimeException("Exception occurred while finding child jobs", ye); - } - - System.out.println("Child yarn jobs are found - " + StringUtils.join(childYarnJobs, ",")); - return childYarnJobs; - } - - public static void killChildYarnJobs(Configuration actionConf) { - try { - Set<ApplicationId> childYarnJobs = getChildYarnJobs(actionConf); - if (!childYarnJobs.isEmpty()) { - System.out.println(); - System.out.println("Found [" + childYarnJobs.size() + "] Map-Reduce jobs from this launcher"); - System.out.println("Killing existing jobs and starting over:"); - YarnClient yarnClient = YarnClient.createYarnClient(); - yarnClient.init(actionConf); - yarnClient.start(); - for (ApplicationId app : childYarnJobs) { - System.out.print("Killing job [" + app + "] ... "); - yarnClient.killApplication(app); - System.out.println("Done"); - } - System.out.println(); - } - } catch (YarnException ye) { - throw new RuntimeException("Exception occurred while killing child job(s)", ye); - } catch (IOException ioe) { - throw new RuntimeException("Exception occurred while killing child job(s)", ioe); - } - } -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/oozie/blob/a37835fe/hadooplibs/hadoop-utils-3/src/main/java/org/apache/oozie/hadoop/utils/HadoopShims.java ---------------------------------------------------------------------- diff --git a/hadooplibs/hadoop-utils-3/src/main/java/org/apache/oozie/hadoop/utils/HadoopShims.java b/hadooplibs/hadoop-utils-3/src/main/java/org/apache/oozie/hadoop/utils/HadoopShims.java deleted file mode 100644 index c21884c..0000000 --- a/hadooplibs/hadoop-utils-3/src/main/java/org/apache/oozie/hadoop/utils/HadoopShims.java +++ /dev/null @@ -1,68 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.oozie.hadoop.utils; - -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; - -import java.io.IOException; -import java.net.URI; - -public class HadoopShims { - FileSystem fs; - - public HadoopShims(FileSystem fs) { - this.fs = fs; - } - - public static boolean isSymlinkSupported() { - return true; - } - - public Path getSymLinkTarget(Path p) throws IOException { - try { - //getSymlink doesn't work with fragment name, need to remove fragment before calling getSymlink - Path tempPath = new URI(p.toString()).getFragment() == null ? p : new Path(new URI(p.toString()).getPath()); - return fs.getFileLinkStatus(tempPath).getSymlink(); - } - catch (java.net.URISyntaxException e) { - throw new IOException(e); - } - } - - public boolean isSymlink(Path p) throws IOException { - try { - //isSymlink doesn't work with fragment name, need to remove fragment before checking for symlink - Path tempPath = new URI(p.toString()).getFragment() == null ? p : new Path(new URI(p.toString()).getPath()); - return fs.getFileLinkStatus(tempPath).isSymlink(); - } - catch (java.net.URISyntaxException e) { - throw new IOException(e); - } - } - - public void createSymlink(Path target, Path link, boolean createParent) throws IOException { - fs.createSymlink(target, link, createParent); - } - - public static boolean isYARN() { - return true; - } - -} http://git-wip-us.apache.org/repos/asf/oozie/blob/a37835fe/hadooplibs/pom.xml ---------------------------------------------------------------------- diff --git a/hadooplibs/pom.xml b/hadooplibs/pom.xml deleted file mode 100644 index 17f1f07..0000000 --- a/hadooplibs/pom.xml +++ /dev/null @@ -1,93 +0,0 @@ -<?xml version="1.0" encoding="UTF-8"?> -<!-- - Licensed to the Apache Software Foundation (ASF) under one - or more contributor license agreements. See the NOTICE file - distributed with this work for additional information - regarding copyright ownership. The ASF licenses this file - to you under the Apache License, Version 2.0 (the - "License"); you may not use this file except in compliance - with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. ---> -<project xmlns="http://maven.apache.org/POM/4.0.0"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; - xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd";> - <modelVersion>4.0.0</modelVersion> - <parent> - <groupId>org.apache.oozie</groupId> - <artifactId>oozie-main</artifactId> - <version>4.3.0-SNAPSHOT</version> - </parent> - <groupId>org.apache.oozie</groupId> - <artifactId>oozie-hadooplibs</artifactId> - <version>4.3.0-SNAPSHOT</version> - <description>Apache Oozie Hadoop Libs</description> - <name>Apache Oozie Hadoop Libs</name> - <packaging>pom</packaging> - - <build> - <plugins> - <plugin> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-deploy-plugin</artifactId> - <configuration> - <skip>true</skip> - </configuration> - </plugin> - </plugins> - </build> - - <profiles> - <profile> - <id>hadoop-1</id> - <activation> - <activeByDefault>true</activeByDefault> - </activation> - <modules> - <module>hadoop-utils-1</module> - <module>hadoop-distcp-1</module> - <module>hadoop-auth-1</module> - </modules> - </profile> - <profile> - <id>hadoop-0.23</id> - <activation> - <activeByDefault>false</activeByDefault> - </activation> - <modules> - <module>hadoop-utils-0.23</module> - <module>hadoop-distcp-0.23</module> - <module>hadoop-auth-0.23</module> - </modules> - </profile> - <profile> - <id>hadoop-2</id> - <activation> - <activeByDefault>false</activeByDefault> - </activation> - <modules> - <module>hadoop-utils-2</module> - <module>hadoop-distcp-2</module> - <module>hadoop-auth-2</module> - </modules> - </profile> - <profile> - <id>hadoop-3</id> - <activation> - <activeByDefault>false</activeByDefault> - </activation> - <modules> - <module>hadoop-distcp-3</module> - <module>hadoop-auth-3</module> - <module>hadoop-utils-3</module> - </modules> - </profile> - </profiles> -</project> - http://git-wip-us.apache.org/repos/asf/oozie/blob/a37835fe/pom.xml ---------------------------------------------------------------------- diff --git a/pom.xml b/pom.xml index 5b7a863..9d86e71 100644 --- a/pom.xml +++ b/pom.xml @@ -76,26 +76,23 @@ </oozie.test.default.config.file> <oozie.test.config.file>${oozie.test.default.config.file}</oozie.test.config.file> - <hadoop.version>1.2.1</hadoop.version> - <hadoop.majorversion>1</hadoop.majorversion> - <hadooplib.version>hadoop-${hadoop.majorversion}-${project.version}</hadooplib.version> - <oozie.hadoop-utils.version>hadoop-${hadoop.majorversion}-${project.version}</oozie.hadoop-utils.version> + <hadoop.version>2.4.0</hadoop.version> <hbase.version>0.94.27</hbase.version> <dropwizard.metrics.version>3.1.0</dropwizard.metrics.version> <clover.license>/home/jenkins/tools/clover/latest/lib/clover.license</clover.license> + <!-- Sharelib component versions --> <hive.version>0.13.1</hive.version> <pig.version>0.12.1</pig.version> - <pig.classifier></pig.classifier> + <pig.classifier>h2</pig.classifier> <sqoop.version>1.4.3</sqoop.version> <spark.version>1.6.1</spark.version> <spark.guava.version>14.0.1</spark.guava.version> - <sqoop.classifier>hadoop100</sqoop.classifier> + <sqoop.classifier>hadoop200</sqoop.classifier> <streaming.version>${hadoop.version}</streaming.version> - <distcp.version>${hadoop.version}</distcp.version> <hadoop.auth.version>${hadoop.version}</hadoop.auth.version> <!-- Tomcat version --> @@ -104,12 +101,11 @@ <openjpa.version>2.2.2</openjpa.version> <xerces.version>2.10.0</xerces.version> <curator.version>2.5.0</curator.version> - <jackson.version>1.8.8</jackson.version> + <jackson.version>1.9.13</jackson.version> <log4j.version>1.2.17</log4j.version> </properties> <modules> - <module>hadooplibs</module> <module>client</module> <module>core</module> <module>sharelib</module> @@ -311,13 +307,6 @@ </dependency> <dependency> - <groupId>org.apache.oozie</groupId> - <artifactId>oozie-hadoop-auth</artifactId> - <version>${hadooplib.version}</version> - <scope>provided</scope> - </dependency> - - <dependency> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-auth</artifactId> <version>${hadoop.auth.version}</version> @@ -430,7 +419,7 @@ <dependency> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-distcp</artifactId> - <version>${distcp.version}</version> + <version>${hadoop.version}</version> <scope>compile</scope> <exclusions> <exclusion> @@ -443,7 +432,7 @@ <dependency> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-tools</artifactId> - <version>${distcp.version}</version> + <version>${hadoop.version}</version> <scope>compile</scope> <exclusions> <exclusion> @@ -454,13 +443,6 @@ </dependency> <dependency> - <groupId>org.apache.oozie</groupId> - <artifactId>oozie-hadoop-distcp</artifactId> - <version>${hadooplib.version}</version> - <scope>provided</scope> - </dependency> - - <dependency> <groupId>org.apache.hive.hcatalog</groupId> <artifactId>hive-webhcat-java-client</artifactId> <version>${hive.version}</version> @@ -1452,12 +1434,6 @@ </dependency> <dependency> - <groupId>org.apache.oozie</groupId> - <artifactId>oozie-hadoop-utils</artifactId> - <version>${oozie.hadoop-utils.version}</version> - </dependency> - - <dependency> <groupId>dk.brics.automaton</groupId> <artifactId>automaton</artifactId> <version>1.11-8</version> @@ -1572,6 +1548,7 @@ <plugin> <groupId>com.atlassian.maven.plugins</groupId> <artifactId>maven-clover2-plugin</artifactId> + <version>4.0.6</version> <configuration> <licenseLocation>${clover.license}</licenseLocation> <generateXml>true</generateXml> @@ -1585,6 +1562,7 @@ <plugin> <groupId>org.codehaus.mojo</groupId> <artifactId>findbugs-maven-plugin</artifactId> + <version>3.0.1</version> <configuration> <xmlOutput>true</xmlOutput> <findbugsXmlOutput>true</findbugsXmlOutput> @@ -1770,57 +1748,9 @@ </plugin> </plugins> </build> + </profile> - <profile> - <id>hadoop-1</id> - <activation> - <activeByDefault>true</activeByDefault> - </activation> - <properties> - <hadoop.version>1.2.1</hadoop.version> - <hadoop.majorversion>1</hadoop.majorversion> - <pig.classifier></pig.classifier> - <sqoop.classifier>hadoop100</sqoop.classifier> - </properties> - </profile> - <profile> - <id>hadoop-0.23</id> - <activation> - <activeByDefault>false</activeByDefault> - </activation> - <properties> - <hadoop.version>0.23.5</hadoop.version> - <hadoop.majorversion>0.23</hadoop.majorversion> - <pig.classifier>h2</pig.classifier> - <sqoop.classifier>hadoop23</sqoop.classifier> - </properties> - </profile> - <profile> - <id>hadoop-2</id> - <activation> - <activeByDefault>false</activeByDefault> - </activation> - <properties> - <hadoop.version>2.4.0</hadoop.version> - <hadoop.majorversion>2</hadoop.majorversion> - <pig.classifier>h2</pig.classifier> - <sqoop.classifier>hadoop200</sqoop.classifier> - <jackson.version>1.9.13</jackson.version> - </properties> - </profile> - <profile> - <id>hadoop-3</id> - <activation> - <activeByDefault>false</activeByDefault> - </activation> - <properties> - <hadoop.version>3.0.0-SNAPSHOT</hadoop.version> - <hadoop.majorversion>3</hadoop.majorversion> - <pig.classifier>h2</pig.classifier> - <sqoop.classifier>hadoop200</sqoop.classifier> - <jackson.version>1.9.13</jackson.version> - </properties> - </profile> + <profile> <id>loginServerExample</id> <activation> http://git-wip-us.apache.org/repos/asf/oozie/blob/a37835fe/release-log.txt ---------------------------------------------------------------------- diff --git a/release-log.txt b/release-log.txt index 189ca21..3a23cbf 100644 --- a/release-log.txt +++ b/release-log.txt @@ -1,5 +1,6 @@ -- Oozie 4.3.0 release (trunk - unreleased) +OOZIE-2316 Drop support for Hadoop 1 and 0.23 (asasvari via rkanter) OOZIE-2503 show ChildJobURLs to spark action (satishsaley via puru) OOZIE-2551 Feature request: epoch timestamp generation (jtolar via puru) OOZIE-2542 Option to disable OpenJPA BrokerImpl finalization (puru) http://git-wip-us.apache.org/repos/asf/oozie/blob/a37835fe/sharelib/distcp/pom.xml ---------------------------------------------------------------------- diff --git a/sharelib/distcp/pom.xml b/sharelib/distcp/pom.xml index 0b56f07..c8cc47c 100644 --- a/sharelib/distcp/pom.xml +++ b/sharelib/distcp/pom.xml @@ -39,8 +39,8 @@ <dependencies> <dependency> - <groupId>org.apache.oozie</groupId> - <artifactId>oozie-hadoop-distcp</artifactId> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-distcp</artifactId> <scope>compile</scope> </dependency> http://git-wip-us.apache.org/repos/asf/oozie/blob/a37835fe/sharelib/distcp/src/main/java/org/apache/oozie/action/hadoop/DistcpMain.java ---------------------------------------------------------------------- diff --git a/sharelib/distcp/src/main/java/org/apache/oozie/action/hadoop/DistcpMain.java b/sharelib/distcp/src/main/java/org/apache/oozie/action/hadoop/DistcpMain.java index 6ac5ad6..0fb8e8b 100644 --- a/sharelib/distcp/src/main/java/org/apache/oozie/action/hadoop/DistcpMain.java +++ b/sharelib/distcp/src/main/java/org/apache/oozie/action/hadoop/DistcpMain.java @@ -48,7 +48,7 @@ public class DistcpMain extends JavaMain { protected void run(String[] args) throws Exception { Configuration actionConf = loadActionConf(); - LauncherMainHadoopUtils.killChildYarnJobs(actionConf); + LauncherMain.killChildYarnJobs(actionConf); String logFile = setUpDistcpLog4J(actionConf); Class<?> klass = actionConf.getClass(LauncherMapper.CONF_OOZIE_ACTION_MAIN_CLASS, org.apache.hadoop.tools.DistCp.class); http://git-wip-us.apache.org/repos/asf/oozie/blob/a37835fe/sharelib/hive/pom.xml ---------------------------------------------------------------------- diff --git a/sharelib/hive/pom.xml b/sharelib/hive/pom.xml index 73802da..d10d7b8 100644 --- a/sharelib/hive/pom.xml +++ b/sharelib/hive/pom.xml @@ -122,18 +122,10 @@ <classifier>tests</classifier> <scope>test</scope> </dependency> - - <dependency> - <groupId>org.apache.hadoop</groupId> - <artifactId>hadoop-client</artifactId> - <scope>provided</scope> - </dependency> - <dependency> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-minicluster</artifactId> </dependency> - <dependency> <groupId>junit</groupId> <artifactId>junit</artifactId> @@ -150,8 +142,8 @@ <scope>provided</scope> </dependency> <dependency> - <groupId>org.apache.oozie</groupId> - <artifactId>oozie-hadoop-utils</artifactId> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-client</artifactId> <scope>provided</scope> </dependency> </dependencies> http://git-wip-us.apache.org/repos/asf/oozie/blob/a37835fe/sharelib/hive/src/main/java/org/apache/oozie/action/hadoop/HiveMain.java ---------------------------------------------------------------------- diff --git a/sharelib/hive/src/main/java/org/apache/oozie/action/hadoop/HiveMain.java b/sharelib/hive/src/main/java/org/apache/oozie/action/hadoop/HiveMain.java index cb0172d..69c1012 100644 --- a/sharelib/hive/src/main/java/org/apache/oozie/action/hadoop/HiveMain.java +++ b/sharelib/hive/src/main/java/org/apache/oozie/action/hadoop/HiveMain.java @@ -294,7 +294,7 @@ public class HiveMain extends LauncherMain { } System.out.println(); - LauncherMainHadoopUtils.killChildYarnJobs(hiveConf); + LauncherMain.killChildYarnJobs(hiveConf); System.out.println("================================================================="); System.out.println(); http://git-wip-us.apache.org/repos/asf/oozie/blob/a37835fe/sharelib/hive2/pom.xml ---------------------------------------------------------------------- diff --git a/sharelib/hive2/pom.xml b/sharelib/hive2/pom.xml index d660b90..ce967c5 100644 --- a/sharelib/hive2/pom.xml +++ b/sharelib/hive2/pom.xml @@ -115,11 +115,7 @@ <classifier>tests</classifier> <scope>test</scope> </dependency> - <dependency> - <groupId>org.apache.oozie</groupId> - <artifactId>oozie-hadoop-utils</artifactId> - <scope>provided</scope> - </dependency> + <dependency> <groupId>junit</groupId> <artifactId>junit</artifactId> http://git-wip-us.apache.org/repos/asf/oozie/blob/a37835fe/sharelib/hive2/src/main/java/org/apache/oozie/action/hadoop/Hive2Main.java ---------------------------------------------------------------------- diff --git a/sharelib/hive2/src/main/java/org/apache/oozie/action/hadoop/Hive2Main.java b/sharelib/hive2/src/main/java/org/apache/oozie/action/hadoop/Hive2Main.java index e122608..f783e6f 100644 --- a/sharelib/hive2/src/main/java/org/apache/oozie/action/hadoop/Hive2Main.java +++ b/sharelib/hive2/src/main/java/org/apache/oozie/action/hadoop/Hive2Main.java @@ -228,7 +228,7 @@ public class Hive2Main extends LauncherMain { } System.out.println(); - LauncherMainHadoopUtils.killChildYarnJobs(actionConf); + LauncherMain.killChildYarnJobs(actionConf); System.out.println("================================================================="); System.out.println(); http://git-wip-us.apache.org/repos/asf/oozie/blob/a37835fe/sharelib/oozie/pom.xml ---------------------------------------------------------------------- diff --git a/sharelib/oozie/pom.xml b/sharelib/oozie/pom.xml index 3fa2872..dd95b45 100644 --- a/sharelib/oozie/pom.xml +++ b/sharelib/oozie/pom.xml @@ -60,12 +60,6 @@ <artifactId>junit</artifactId> <scope>test</scope> </dependency> - - <dependency> - <groupId>org.apache.oozie</groupId> - <artifactId>oozie-hadoop-utils</artifactId> - <scope>compile</scope> - </dependency> </dependencies> <build> http://git-wip-us.apache.org/repos/asf/oozie/blob/a37835fe/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/JavaMain.java ---------------------------------------------------------------------- diff --git a/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/JavaMain.java b/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/JavaMain.java index 10a1b12..32c7434 100644 --- a/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/JavaMain.java +++ b/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/JavaMain.java @@ -42,7 +42,7 @@ public class JavaMain extends LauncherMain { setYarnTag(actionConf); - LauncherMainHadoopUtils.killChildYarnJobs(actionConf); + LauncherMain.killChildYarnJobs(actionConf); Class<?> klass = actionConf.getClass(JAVA_MAIN_CLASS, Object.class); System.out.println("Main class : " + klass.getName()); http://git-wip-us.apache.org/repos/asf/oozie/blob/a37835fe/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/LauncherMain.java ---------------------------------------------------------------------- diff --git a/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/LauncherMain.java b/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/LauncherMain.java index fb190d6..fef6523 100644 --- a/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/LauncherMain.java +++ b/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/LauncherMain.java @@ -28,8 +28,12 @@ import java.io.InputStream; import java.io.OutputStream; import java.io.StringWriter; import java.util.Collection; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; import java.util.Map; import java.util.Properties; +import java.util.Set; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -38,11 +42,22 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.util.Shell; import org.apache.commons.lang.StringUtils; import org.apache.hadoop.mapred.JobConf; +import org.apache.hadoop.yarn.api.ApplicationClientProtocol; +import org.apache.hadoop.yarn.api.protocolrecords.ApplicationsRequestScope; +import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsRequest; +import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsResponse; +import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.api.records.ApplicationReport; +import org.apache.hadoop.yarn.client.ClientRMProxy; +import org.apache.hadoop.yarn.client.api.YarnClient; +import org.apache.hadoop.yarn.exceptions.YarnException; public abstract class LauncherMain { public static final String HADOOP_JOBS = "hadoopJobs"; public static final String MAPREDUCE_JOB_TAGS = "mapreduce.job.tags"; + public static final String CHILD_MAPREDUCE_JOB_TAGS = "oozie.child.mapreduce.job.tags"; + public static final String OOZIE_JOB_LAUNCH_TIME = "oozie.job.launch.time"; protected static void run(Class<? extends LauncherMain> klass, String[] args) throws Exception { LauncherMain main = klass.newInstance(); @@ -104,6 +119,77 @@ public abstract class LauncherMain { } } + private static Set<ApplicationId> getChildYarnJobs(Configuration actionConf) { + System.out.println("Fetching child yarn jobs"); + Set<ApplicationId> childYarnJobs = new HashSet<ApplicationId>(); + String tag = actionConf.get(CHILD_MAPREDUCE_JOB_TAGS); + if (tag == null) { + System.out.print("Could not find Yarn tags property " + CHILD_MAPREDUCE_JOB_TAGS); + return childYarnJobs; + } + System.out.println("tag id : " + tag); + long startTime = 0L; + try { + startTime = Long.parseLong(System.getProperty(OOZIE_JOB_LAUNCH_TIME)); + } catch(NumberFormatException nfe) { + throw new RuntimeException("Could not find Oozie job launch time", nfe); + } + + GetApplicationsRequest gar = GetApplicationsRequest.newInstance(); + gar.setScope(ApplicationsRequestScope.OWN); + gar.setApplicationTags(Collections.singleton(tag)); + long endTime = System.currentTimeMillis(); + if (startTime > endTime) { + System.out.println("WARNING: Clock skew between the Oozie server host and this host detected. Please fix this. " + + "Attempting to work around..."); + // We don't know which one is wrong (relative to the RM), so to be safe, let's assume they're both wrong and add an + // offset in both directions + long diff = 2 * (startTime - endTime); + startTime = startTime - diff; + endTime = endTime + diff; + } + gar.setStartRange(startTime, endTime); + try { + ApplicationClientProtocol proxy = ClientRMProxy.createRMProxy(actionConf, ApplicationClientProtocol.class); + GetApplicationsResponse apps = proxy.getApplications(gar); + List<ApplicationReport> appsList = apps.getApplicationList(); + for(ApplicationReport appReport : appsList) { + childYarnJobs.add(appReport.getApplicationId()); + } + } catch (IOException ioe) { + throw new RuntimeException("Exception occurred while finding child jobs", ioe); + } catch (YarnException ye) { + throw new RuntimeException("Exception occurred while finding child jobs", ye); + } + + System.out.println("Child yarn jobs are found - " + StringUtils.join(childYarnJobs, ",")); + return childYarnJobs; + } + + public static void killChildYarnJobs(Configuration actionConf) { + try { + Set<ApplicationId> childYarnJobs = getChildYarnJobs(actionConf); + if (!childYarnJobs.isEmpty()) { + System.out.println(); + System.out.println("Found [" + childYarnJobs.size() + "] Map-Reduce jobs from this launcher"); + System.out.println("Killing existing jobs and starting over:"); + YarnClient yarnClient = YarnClient.createYarnClient(); + yarnClient.init(actionConf); + yarnClient.start(); + for (ApplicationId app : childYarnJobs) { + System.out.print("Killing job [" + app + "] ... "); + yarnClient.killApplication(app); + System.out.println("Done"); + } + System.out.println(); + } + } catch (YarnException ye) { + throw new RuntimeException("Exception occurred while killing child job(s)", ye); + } catch (IOException ioe) { + throw new RuntimeException("Exception occurred while killing child job(s)", ioe); + } + } + protected abstract void run(String[] args) throws Exception; /** @@ -202,13 +288,13 @@ public abstract class LauncherMain { } protected static void setYarnTag(Configuration actionConf) { - if(actionConf.get(LauncherMainHadoopUtils.CHILD_MAPREDUCE_JOB_TAGS) != null) { + if(actionConf.get(CHILD_MAPREDUCE_JOB_TAGS) != null) { // in case the user set their own tags, appending the launcher tag. if(actionConf.get(MAPREDUCE_JOB_TAGS) != null) { actionConf.set(MAPREDUCE_JOB_TAGS, actionConf.get(MAPREDUCE_JOB_TAGS) + "," - + actionConf.get(LauncherMainHadoopUtils.CHILD_MAPREDUCE_JOB_TAGS)); + + actionConf.get(CHILD_MAPREDUCE_JOB_TAGS)); } else { - actionConf.set(MAPREDUCE_JOB_TAGS, actionConf.get(LauncherMainHadoopUtils.CHILD_MAPREDUCE_JOB_TAGS)); + actionConf.set(MAPREDUCE_JOB_TAGS, actionConf.get(CHILD_MAPREDUCE_JOB_TAGS)); } } } http://git-wip-us.apache.org/repos/asf/oozie/blob/a37835fe/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/LauncherMapper.java ---------------------------------------------------------------------- diff --git a/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/LauncherMapper.java b/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/LauncherMapper.java index 545f8e2..f2cba13 100644 --- a/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/LauncherMapper.java +++ b/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/LauncherMapper.java @@ -28,8 +28,6 @@ import java.io.InputStreamReader; import java.io.OutputStreamWriter; import java.io.PrintWriter; import java.io.StringWriter; -import java.io.OutputStream; -import java.io.FileOutputStream; import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; import java.security.Permission; @@ -349,9 +347,9 @@ public class LauncherMapper<K1, V1, K2, V2> implements Mapper<K1, V1, K2, V2>, R // loading action conf prepared by Oozie Configuration actionConf = LauncherMain.loadActionConf(); - if(actionConf.get(LauncherMainHadoopUtils.CHILD_MAPREDUCE_JOB_TAGS) != null) { + if(actionConf.get(LauncherMain.CHILD_MAPREDUCE_JOB_TAGS) != null) { propagationConf.set(LauncherMain.MAPREDUCE_JOB_TAGS, - actionConf.get(LauncherMainHadoopUtils.CHILD_MAPREDUCE_JOB_TAGS)); + actionConf.get(LauncherMain.CHILD_MAPREDUCE_JOB_TAGS)); } propagationConf.writeXml(new FileWriter(PROPAGATION_CONF_XML)); @@ -470,9 +468,9 @@ public class LauncherMapper<K1, V1, K2, V2> implements Mapper<K1, V1, K2, V2>, R System.setProperty(ACTION_PREFIX + ACTION_DATA_NEW_ID, new File(ACTION_DATA_NEW_ID).getAbsolutePath()); System.setProperty(ACTION_PREFIX + ACTION_DATA_OUTPUT_PROPS, new File(ACTION_DATA_OUTPUT_PROPS).getAbsolutePath()); System.setProperty(ACTION_PREFIX + ACTION_DATA_ERROR_PROPS, new File(ACTION_DATA_ERROR_PROPS).getAbsolutePath()); - if (getJobConf().get(LauncherMainHadoopUtils.OOZIE_JOB_LAUNCH_TIME) != null) { - System.setProperty(LauncherMainHadoopUtils.OOZIE_JOB_LAUNCH_TIME, - getJobConf().get(LauncherMainHadoopUtils.OOZIE_JOB_LAUNCH_TIME)); + if (getJobConf().get(LauncherMain.OOZIE_JOB_LAUNCH_TIME) != null) { + System.setProperty(LauncherMain.OOZIE_JOB_LAUNCH_TIME, + getJobConf().get(LauncherMain.OOZIE_JOB_LAUNCH_TIME)); } String actionConfigClass = getJobConf().get(OOZIE_ACTION_CONFIG_CLASS); http://git-wip-us.apache.org/repos/asf/oozie/blob/a37835fe/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/MapReduceMain.java ---------------------------------------------------------------------- diff --git a/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/MapReduceMain.java b/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/MapReduceMain.java index 23447cf..96d59b9 100644 --- a/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/MapReduceMain.java +++ b/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/MapReduceMain.java @@ -50,7 +50,7 @@ public class MapReduceMain extends LauncherMain { JobConf jobConf = new JobConf(); addActionConf(jobConf, actionConf); - LauncherMainHadoopUtils.killChildYarnJobs(jobConf); + LauncherMain.killChildYarnJobs(jobConf); // Run a config class if given to update the job conf runConfigClass(jobConf); http://git-wip-us.apache.org/repos/asf/oozie/blob/a37835fe/sharelib/pig/pom.xml ---------------------------------------------------------------------- diff --git a/sharelib/pig/pom.xml b/sharelib/pig/pom.xml index bb3eea5..562c530 100644 --- a/sharelib/pig/pom.xml +++ b/sharelib/pig/pom.xml @@ -83,12 +83,6 @@ <dependency> <groupId>org.apache.hadoop</groupId> - <artifactId>hadoop-client</artifactId> - <scope>provided</scope> - </dependency> - - <dependency> - <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-minicluster</artifactId> </dependency> @@ -103,8 +97,8 @@ <scope>provided</scope> </dependency> <dependency> - <groupId>org.apache.oozie</groupId> - <artifactId>oozie-hadoop-utils</artifactId> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-client</artifactId> <scope>provided</scope> </dependency> <dependency> http://git-wip-us.apache.org/repos/asf/oozie/blob/a37835fe/sharelib/pig/src/main/java/org/apache/oozie/action/hadoop/PigMain.java ---------------------------------------------------------------------- diff --git a/sharelib/pig/src/main/java/org/apache/oozie/action/hadoop/PigMain.java b/sharelib/pig/src/main/java/org/apache/oozie/action/hadoop/PigMain.java index 7ba22f1..3d1110b 100644 --- a/sharelib/pig/src/main/java/org/apache/oozie/action/hadoop/PigMain.java +++ b/sharelib/pig/src/main/java/org/apache/oozie/action/hadoop/PigMain.java @@ -214,7 +214,7 @@ public class PigMain extends LauncherMain { System.out.println(" " + arg); } - LauncherMainHadoopUtils.killChildYarnJobs(actionConf); + LauncherMain.killChildYarnJobs(actionConf); System.out.println("================================================================="); System.out.println(); http://git-wip-us.apache.org/repos/asf/oozie/blob/a37835fe/sharelib/spark/pom.xml ---------------------------------------------------------------------- diff --git a/sharelib/spark/pom.xml b/sharelib/spark/pom.xml index 748ae06..46c6375 100644 --- a/sharelib/spark/pom.xml +++ b/sharelib/spark/pom.xml @@ -248,6 +248,103 @@ <artifactId>oozie-examples</artifactId> <scope>test</scope> </dependency> + + <dependency> + <groupId>org.apache.spark</groupId> + <artifactId>spark-yarn_2.10</artifactId> + <version>${spark.version}</version> + <scope>compile</scope> + <exclusions> + <exclusion> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-yarn-client</artifactId> + </exclusion> + <exclusion> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-yarn-api</artifactId> + </exclusion> + <exclusion> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-yarn-common</artifactId> + </exclusion> + <exclusion> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-yarn-server-common</artifactId> + </exclusion> + <exclusion> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-yarn-server-web-proxy</artifactId> + </exclusion> + <exclusion> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-mapreduce-client-shuffle</artifactId> + </exclusion> + <exclusion> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-mapreduce-client-jobclient</artifactId> + </exclusion> + <exclusion> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-mapreduce-client-core</artifactId> + </exclusion> + <exclusion> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-mapreduce-client-common</artifactId> + </exclusion> + <exclusion> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-mapreduce-client-app</artifactId> + </exclusion> + <exclusion> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-annotations</artifactId> + </exclusion> + <exclusion> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-auth</artifactId> + </exclusion> + <exclusion> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-aws</artifactId> + </exclusion> + <exclusion> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-client</artifactId> + </exclusion> + <exclusion> + <groupId>org.spark-project.hive</groupId> + <artifactId>hive-beeline</artifactId> + </exclusion> + <exclusion> + <groupId>org.spark-project.hive</groupId> + <artifactId>hive-common</artifactId> + </exclusion> + <exclusion> + <groupId>org.spark-project.hive</groupId> + <artifactId>hive-exec</artifactId> + </exclusion> + <exclusion> + <groupId>org.spark-project.hive</groupId> + <artifactId>hive-jdbc</artifactId> + </exclusion> + <exclusion> + <groupId>org.spark-project.hive</groupId> + <artifactId>hive-metastore</artifactId> + </exclusion> + <exclusion> + <groupId>org.spark-project.hive</groupId> + <artifactId>hive-serde</artifactId> + </exclusion> + <exclusion> + <groupId>org.spark-project.hive</groupId> + <artifactId>hive-service</artifactId> + </exclusion> + <exclusion> + <groupId>org.spark-project.hive</groupId> + <artifactId>hive-shims</artifactId> + </exclusion> + </exclusions> + </dependency> </dependencies> <build> @@ -320,109 +417,5 @@ </plugin> </plugins> </build> - - <profiles> - <profile> - <id>hadoop-2</id> - <dependencies> - <dependency> - <groupId>org.apache.spark</groupId> - <artifactId>spark-yarn_2.10</artifactId> - <version>${spark.version}</version> - <scope>compile</scope> - <exclusions> - <exclusion> - <groupId>org.apache.hadoop</groupId> - <artifactId>hadoop-yarn-client</artifactId> - </exclusion> - <exclusion> - <groupId>org.apache.hadoop</groupId> - <artifactId>hadoop-yarn-api</artifactId> - </exclusion> - <exclusion> - <groupId>org.apache.hadoop</groupId> - <artifactId>hadoop-yarn-common</artifactId> - </exclusion> - <exclusion> - <groupId>org.apache.hadoop</groupId> - <artifactId>hadoop-yarn-server-common</artifactId> - </exclusion> - <exclusion> - <groupId>org.apache.hadoop</groupId> - <artifactId>hadoop-yarn-server-web-proxy</artifactId> - </exclusion> - <exclusion> - <groupId>org.apache.hadoop</groupId> - <artifactId>hadoop-mapreduce-client-shuffle</artifactId> - </exclusion> - <exclusion> - <groupId>org.apache.hadoop</groupId> - <artifactId>hadoop-mapreduce-client-jobclient</artifactId> - </exclusion> - <exclusion> - <groupId>org.apache.hadoop</groupId> - <artifactId>hadoop-mapreduce-client-core</artifactId> - </exclusion> - <exclusion> - <groupId>org.apache.hadoop</groupId> - <artifactId>hadoop-mapreduce-client-common</artifactId> - </exclusion> - <exclusion> - <groupId>org.apache.hadoop</groupId> - <artifactId>hadoop-mapreduce-client-app</artifactId> - </exclusion> - <exclusion> - <groupId>org.apache.hadoop</groupId> - <artifactId>hadoop-annotations</artifactId> - </exclusion> - <exclusion> - <groupId>org.apache.hadoop</groupId> - <artifactId>hadoop-auth</artifactId> - </exclusion> - <exclusion> - <groupId>org.apache.hadoop</groupId> - <artifactId>hadoop-aws</artifactId> - </exclusion> - <exclusion> - <groupId>org.apache.hadoop</groupId> - <artifactId>hadoop-client</artifactId> - </exclusion> - <exclusion> - <groupId>org.spark-project.hive</groupId> - <artifactId>hive-beeline</artifactId> - </exclusion> - <exclusion> - <groupId>org.spark-project.hive</groupId> - <artifactId>hive-common</artifactId> - </exclusion> - <exclusion> - <groupId>org.spark-project.hive</groupId> - <artifactId>hive-exec</artifactId> - </exclusion> - <exclusion> - <groupId>org.spark-project.hive</groupId> - <artifactId>hive-jdbc</artifactId> - </exclusion> - <exclusion> - <groupId>org.spark-project.hive</groupId> - <artifactId>hive-metastore</artifactId> - </exclusion> - <exclusion> - <groupId>org.spark-project.hive</groupId> - <artifactId>hive-serde</artifactId> - </exclusion> - <exclusion> - <groupId>org.spark-project.hive</groupId> - <artifactId>hive-service</artifactId> - </exclusion> - <exclusion> - <groupId>org.spark-project.hive</groupId> - <artifactId>hive-shims</artifactId> - </exclusion> - </exclusions> - </dependency> - </dependencies> - </profile> - </profiles> </project> http://git-wip-us.apache.org/repos/asf/oozie/blob/a37835fe/sharelib/spark/src/main/java/org/apache/oozie/action/hadoop/SparkMain.java ---------------------------------------------------------------------- diff --git a/sharelib/spark/src/main/java/org/apache/oozie/action/hadoop/SparkMain.java b/sharelib/spark/src/main/java/org/apache/oozie/action/hadoop/SparkMain.java index 0e6e271..9dbd12f 100644 --- a/sharelib/spark/src/main/java/org/apache/oozie/action/hadoop/SparkMain.java +++ b/sharelib/spark/src/main/java/org/apache/oozie/action/hadoop/SparkMain.java @@ -65,7 +65,7 @@ public class SparkMain extends LauncherMain { boolean isPyspark = false; Configuration actionConf = loadActionConf(); setYarnTag(actionConf); - LauncherMainHadoopUtils.killChildYarnJobs(actionConf); + LauncherMain.killChildYarnJobs(actionConf); String logFile = setUpSparkLog4J(actionConf); List<String> sparkArgs = new ArrayList<String>(); http://git-wip-us.apache.org/repos/asf/oozie/blob/a37835fe/sharelib/sqoop/pom.xml ---------------------------------------------------------------------- diff --git a/sharelib/sqoop/pom.xml b/sharelib/sqoop/pom.xml index 5b80a4f..d875c93 100644 --- a/sharelib/sqoop/pom.xml +++ b/sharelib/sqoop/pom.xml @@ -214,11 +214,6 @@ <artifactId>oozie-sharelib-oozie</artifactId> <scope>provided</scope> </dependency> - <dependency> - <groupId>org.apache.oozie</groupId> - <artifactId>oozie-hadoop-utils</artifactId> - <scope>provided</scope> - </dependency> </dependencies> <build> http://git-wip-us.apache.org/repos/asf/oozie/blob/a37835fe/sharelib/sqoop/src/main/java/org/apache/oozie/action/hadoop/SqoopMain.java ---------------------------------------------------------------------- diff --git a/sharelib/sqoop/src/main/java/org/apache/oozie/action/hadoop/SqoopMain.java b/sharelib/sqoop/src/main/java/org/apache/oozie/action/hadoop/SqoopMain.java index 360cf1e..1f66c4a 100644 --- a/sharelib/sqoop/src/main/java/org/apache/oozie/action/hadoop/SqoopMain.java +++ b/sharelib/sqoop/src/main/java/org/apache/oozie/action/hadoop/SqoopMain.java @@ -167,7 +167,7 @@ public class SqoopMain extends LauncherMain { System.out.println(" " + arg); } - LauncherMainHadoopUtils.killChildYarnJobs(sqoopConf); + LauncherMain.killChildYarnJobs(sqoopConf); System.out.println("================================================================="); System.out.println(); http://git-wip-us.apache.org/repos/asf/oozie/blob/a37835fe/webapp/pom.xml ---------------------------------------------------------------------- diff --git a/webapp/pom.xml b/webapp/pom.xml index 7e73572..d0c1806 100644 --- a/webapp/pom.xml +++ b/webapp/pom.xml @@ -37,8 +37,8 @@ <artifactId>oozie-client</artifactId> <exclusions> <exclusion> - <groupId>org.apache.oozie</groupId> - <artifactId>oozie-hadoop-auth</artifactId> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-auth</artifactId> </exclusion> </exclusions> </dependency> @@ -271,8 +271,8 @@ <scope>compile</scope> </dependency> <dependency> - <groupId>org.apache.oozie</groupId> - <artifactId>oozie-hadoop-auth</artifactId> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-auth</artifactId> <scope>compile</scope> </dependency> <dependency>
