OOZIE-2675 Drop support for hadoop 0.23 (abhishekbafna via rohini)
Project: http://git-wip-us.apache.org/repos/asf/oozie/repo Commit: http://git-wip-us.apache.org/repos/asf/oozie/commit/2fd64fa6 Tree: http://git-wip-us.apache.org/repos/asf/oozie/tree/2fd64fa6 Diff: http://git-wip-us.apache.org/repos/asf/oozie/diff/2fd64fa6 Branch: refs/heads/oya Commit: 2fd64fa646708ae684f2274729a0df6623598709 Parents: 9dc474e Author: Rohini Palaniswamy <roh...@apache.org> Authored: Mon Sep 19 09:53:25 2016 -0700 Committer: Rohini Palaniswamy <roh...@apache.org> Committed: Mon Sep 19 09:53:25 2016 -0700 ---------------------------------------------------------------------- .../oozie/action/hadoop/HadoopELFunctions.java | 5 -- .../oozie/command/wf/SubmitMRXCommand.java | 3 -- .../java/org/apache/oozie/util/JobUtils.java | 2 +- .../java/org/apache/oozie/test/XTestCase.java | 4 +- docs/src/site/twiki/DG_QuickStart.twiki | 5 +- .../site/twiki/DG_SqoopActionExtension.twiki | 2 +- docs/src/site/twiki/ENG_Building.twiki | 5 +- hadooplibs/hadoop-auth-0.23/pom.xml | 45 ----------------- hadooplibs/hadoop-distcp-0.23/pom.xml | 43 ----------------- hadooplibs/hadoop-utils-0.23/pom.xml | 42 ---------------- .../action/hadoop/LauncherMainHadoopUtils.java | 35 -------------- .../apache/oozie/hadoop/utils/HadoopShims.java | 51 -------------------- hadooplibs/pom.xml | 11 ----- pom.xml | 12 ----- release-log.txt | 1 + 15 files changed, 8 insertions(+), 258 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/oozie/blob/2fd64fa6/core/src/main/java/org/apache/oozie/action/hadoop/HadoopELFunctions.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/oozie/action/hadoop/HadoopELFunctions.java b/core/src/main/java/org/apache/oozie/action/hadoop/HadoopELFunctions.java index c322887..ad2a71d 100644 --- a/core/src/main/java/org/apache/oozie/action/hadoop/HadoopELFunctions.java +++ b/core/src/main/java/org/apache/oozie/action/hadoop/HadoopELFunctions.java @@ -52,11 +52,6 @@ public class HadoopELFunctions { Map<String, Map<String, Long>> counters = (Map<String, Map<String, Long>>) obj; if (counters == null) { counters = getCounters(nodeName); - // In Hadoop 0.23 they deprecated 'org.apache.hadoop.mapred.Task$Counter' and they REMOVED IT - // Here we are getting the new Name and inserting it using the old name if the old name is not found - if (counters.get(RECORDS) == null) { - counters.put(RECORDS, counters.get(RECORDS_023)); - } instance.setTransientVar(nodeName + WorkflowInstance.NODE_VAR_SEPARATOR + HADOOP_COUNTERS, counters); } return counters; http://git-wip-us.apache.org/repos/asf/oozie/blob/2fd64fa6/core/src/main/java/org/apache/oozie/command/wf/SubmitMRXCommand.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/oozie/command/wf/SubmitMRXCommand.java b/core/src/main/java/org/apache/oozie/command/wf/SubmitMRXCommand.java index 9124a45..cc61d3d 100644 --- a/core/src/main/java/org/apache/oozie/command/wf/SubmitMRXCommand.java +++ b/core/src/main/java/org/apache/oozie/command/wf/SubmitMRXCommand.java @@ -43,9 +43,6 @@ public class SubmitMRXCommand extends SubmitHttpXCommand { SKIPPED_CONFS.add(WorkflowAppService.HADOOP_USER); SKIPPED_CONFS.add(XOozieClient.JT); SKIPPED_CONFS.add(XOozieClient.NN); - // a brillant mind made a change in Configuration that 'fs.default.name' key gets converted to 'fs.defaultFS' - // in Hadoop 0.23, we need skip that one too, keeping the old one because of Hadoop 1 - SKIPPED_CONFS.add(XOozieClient.NN_2); DEPRECATE_MAP.put(XOozieClient.NN, XOozieClient.NN_2); DEPRECATE_MAP.put(XOozieClient.JT, XOozieClient.JT_2); http://git-wip-us.apache.org/repos/asf/oozie/blob/2fd64fa6/core/src/main/java/org/apache/oozie/util/JobUtils.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/oozie/util/JobUtils.java b/core/src/main/java/org/apache/oozie/util/JobUtils.java index a4d7272..a7a53b3 100644 --- a/core/src/main/java/org/apache/oozie/util/JobUtils.java +++ b/core/src/main/java/org/apache/oozie/util/JobUtils.java @@ -162,7 +162,7 @@ public class JobUtils { URI uri = fs.makeQualified(file).toUri(); DistributedCache.addCacheFile(uri, conf); } - else { // Hadoop 0.23/2.x + else { // Hadoop 2.x DistributedCache.addFileToClassPath(file, conf, fs); } } http://git-wip-us.apache.org/repos/asf/oozie/blob/2fd64fa6/core/src/test/java/org/apache/oozie/test/XTestCase.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/oozie/test/XTestCase.java b/core/src/test/java/org/apache/oozie/test/XTestCase.java index 8ce44f3..3dd99d7 100644 --- a/core/src/test/java/org/apache/oozie/test/XTestCase.java +++ b/core/src/test/java/org/apache/oozie/test/XTestCase.java @@ -967,9 +967,7 @@ public abstract class XTestCase extends TestCase { conf.set("dfs.permissions", "true"); conf.set("hadoop.security.authentication", "simple"); - //Doing this because Hadoop 1.x does not support '*' and - //Hadoop 0.23.x does not process wildcard if the value is - // '*,127.0.0.1' + //Doing this because Hadoop 1.x does not support '*' if the value is '*,127.0.0.1' StringBuilder sb = new StringBuilder(); sb.append("127.0.0.1,localhost"); for (InetAddress i : InetAddress.getAllByName(InetAddress.getLocalHost().getHostName())) { http://git-wip-us.apache.org/repos/asf/oozie/blob/2fd64fa6/docs/src/site/twiki/DG_QuickStart.twiki ---------------------------------------------------------------------- diff --git a/docs/src/site/twiki/DG_QuickStart.twiki b/docs/src/site/twiki/DG_QuickStart.twiki index 2cfaa51..0653f08 100644 --- a/docs/src/site/twiki/DG_QuickStart.twiki +++ b/docs/src/site/twiki/DG_QuickStart.twiki @@ -41,10 +41,9 @@ suitable when same oozie package needs to be used in multiple set-ups with diffe 2. Build with -Puber which will bundle the required libraries in the oozie war. Further, the following options are available to customise the versions of the dependencies: --P<profile> - default hadoop-1. Valid are hadoop-1, hadoop-0.23, hadoop-2 or hadoop-3. Choose the correct hadoop +-P<profile> - default hadoop-1. Valid are hadoop-1, hadoop-2 or hadoop-3. Choose the correct hadoop profile depending on the hadoop version used. --Dhadoop.version=<version> - default 1.2.1 for hadoop-1, 0.23.5 for hadoop-0.23, 2.3.0 for hadoop-2 and 3.0.0-SNAPSHOT - for hadoop-3 +-Dhadoop.version=<version> - default 1.2.1 for hadoop-1, 2.3.0 for hadoop-2 and 3.0.0-SNAPSHOT for hadoop-3 -Dhadoop.auth.version=<version> - defaults to hadoop version -Ddistcp.version=<version> - defaults to hadoop version -Dpig.version=<version> - default 0.16.0 http://git-wip-us.apache.org/repos/asf/oozie/blob/2fd64fa6/docs/src/site/twiki/DG_SqoopActionExtension.twiki ---------------------------------------------------------------------- diff --git a/docs/src/site/twiki/DG_SqoopActionExtension.twiki b/docs/src/site/twiki/DG_SqoopActionExtension.twiki index 1e7f2a8..906ba54 100644 --- a/docs/src/site/twiki/DG_SqoopActionExtension.twiki +++ b/docs/src/site/twiki/DG_SqoopActionExtension.twiki @@ -10,7 +10,7 @@ ---++ Sqoop Action -*IMPORTANT:* The Sqoop action requires Apache Hadoop 0.23. +*IMPORTANT:* The Sqoop action requires Apache Hadoop 1.x or 2.x. The =sqoop= action runs a Sqoop job. http://git-wip-us.apache.org/repos/asf/oozie/blob/2fd64fa6/docs/src/site/twiki/ENG_Building.twiki ---------------------------------------------------------------------- diff --git a/docs/src/site/twiki/ENG_Building.twiki b/docs/src/site/twiki/ENG_Building.twiki index d138817..9864098 100644 --- a/docs/src/site/twiki/ENG_Building.twiki +++ b/docs/src/site/twiki/ENG_Building.twiki @@ -212,10 +212,9 @@ $ bin/mkdistro.sh [-DskipTests] Running =mkdistro.sh= will create the binary distribution of Oozie. The following options are available to customise the versions of the dependencies: -Puber - Bundle required hadoop and hcatalog libraries in oozie war --P<profile> - default hadoop-1. Valid are hadoop-1, hadoop-0.23, hadoop-2 or hadoop-3. Choose the correct hadoop +-P<profile> - default hadoop-1. Valid are hadoop-1, hadoop-2 or hadoop-3. Choose the correct hadoop profile depending on the hadoop version used. --Dhadoop.version=<version> - default 1.2.1 for hadoop-1, 0.23.5 for hadoop-0.23, 2.3.0 for hadoop-2 and 3.0 -.0-SNAPSHOT for hadoop-3 +-Dhadoop.version=<version> - default 1.2.1 for hadoop-1, 2.3.0 for hadoop-2 and 3.0.0-SNAPSHOT for hadoop-3 -Dhadoop.auth.version=<version> - defaults to hadoop version -Ddistcp.version=<version> - defaults to hadoop version -Dpig.version=<version> - default 0.16.0 http://git-wip-us.apache.org/repos/asf/oozie/blob/2fd64fa6/hadooplibs/hadoop-auth-0.23/pom.xml ---------------------------------------------------------------------- diff --git a/hadooplibs/hadoop-auth-0.23/pom.xml b/hadooplibs/hadoop-auth-0.23/pom.xml deleted file mode 100644 index 7b3b466..0000000 --- a/hadooplibs/hadoop-auth-0.23/pom.xml +++ /dev/null @@ -1,45 +0,0 @@ -<?xml version="1.0" encoding="UTF-8"?> -<!-- - Licensed to the Apache Software Foundation (ASF) under one - or more contributor license agreements. See the NOTICE file - distributed with this work for additional information - regarding copyright ownership. The ASF licenses this file - to you under the Apache License, Version 2.0 (the - "License"); you may not use this file except in compliance - with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. ---> -<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" - xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd"> - <modelVersion>4.0.0</modelVersion> - <parent> - <groupId>org.apache.oozie</groupId> - <artifactId>oozie-main</artifactId> - <version>4.3.0-SNAPSHOT</version> - <relativePath>../../pom.xml</relativePath> - </parent> - <groupId>org.apache.oozie</groupId> - <artifactId>oozie-hadoop-auth</artifactId> - <version>hadoop-0.23-4.3.0-SNAPSHOT</version> - <description>Apache Oozie Hadoop Auth</description> - <name>Apache Oozie Hadoop Auth ${project.version} Test</name> - <packaging>jar</packaging> - - <dependencies> - <dependency> - <groupId>org.apache.hadoop</groupId> - <artifactId>hadoop-auth</artifactId> - <scope>compile</scope> - </dependency> - </dependencies> - - -</project> - http://git-wip-us.apache.org/repos/asf/oozie/blob/2fd64fa6/hadooplibs/hadoop-distcp-0.23/pom.xml ---------------------------------------------------------------------- diff --git a/hadooplibs/hadoop-distcp-0.23/pom.xml b/hadooplibs/hadoop-distcp-0.23/pom.xml deleted file mode 100644 index 60c8e02..0000000 --- a/hadooplibs/hadoop-distcp-0.23/pom.xml +++ /dev/null @@ -1,43 +0,0 @@ -<?xml version="1.0" encoding="UTF-8"?> -<!-- - Licensed to the Apache Software Foundation (ASF) under one - or more contributor license agreements. See the NOTICE file - distributed with this work for additional information - regarding copyright ownership. The ASF licenses this file - to you under the Apache License, Version 2.0 (the - "License"); you may not use this file except in compliance - with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. ---> -<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" - xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd"> - <modelVersion>4.0.0</modelVersion> - <parent> - <groupId>org.apache.oozie</groupId> - <artifactId>oozie-main</artifactId> - <version>4.3.0-SNAPSHOT</version> - <relativePath>../../pom.xml</relativePath> - </parent> - <groupId>org.apache.oozie</groupId> - <artifactId>oozie-hadoop-distcp</artifactId> - <version>hadoop-0.23-4.3.0-SNAPSHOT</version> - <description>Apache Oozie Hadoop Distcp ${project.version}</description> - <name>Apache Oozie Hadoop Distcp ${project.version}</name> - <packaging>jar</packaging> - - <dependencies> - <dependency> - <groupId>org.apache.hadoop</groupId> - <artifactId>hadoop-distcp</artifactId> - <scope>compile</scope> - </dependency> - </dependencies> -</project> - http://git-wip-us.apache.org/repos/asf/oozie/blob/2fd64fa6/hadooplibs/hadoop-utils-0.23/pom.xml ---------------------------------------------------------------------- diff --git a/hadooplibs/hadoop-utils-0.23/pom.xml b/hadooplibs/hadoop-utils-0.23/pom.xml deleted file mode 100644 index a55e647..0000000 --- a/hadooplibs/hadoop-utils-0.23/pom.xml +++ /dev/null @@ -1,42 +0,0 @@ -<?xml version="1.0" encoding="UTF-8"?> -<!-- - Licensed to the Apache Software Foundation (ASF) under one - or more contributor license agreements. See the NOTICE file - distributed with this work for additional information - regarding copyright ownership. The ASF licenses this file - to you under the Apache License, Version 2.0 (the - "License"); you may not use this file except in compliance - with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. ---> -<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" - xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd"> - <modelVersion>4.0.0</modelVersion> - <parent> - <groupId>org.apache.oozie</groupId> - <artifactId>oozie-main</artifactId> - <version>4.3.0-SNAPSHOT</version> - <relativePath>../../pom.xml</relativePath> - </parent> - <groupId>org.apache.oozie</groupId> - <artifactId>oozie-hadoop-utils</artifactId> - <version>hadoop-0.23-4.3.0-SNAPSHOT</version> - <description>Apache Oozie Hadoop Utils</description> - <name>Apache Oozie Hadoop Utils</name> - <packaging>jar</packaging> - - <dependencies> - <dependency> - <groupId>org.apache.hadoop</groupId> - <artifactId>hadoop-client</artifactId> - <scope>provided</scope> - </dependency> - </dependencies> -</project> http://git-wip-us.apache.org/repos/asf/oozie/blob/2fd64fa6/hadooplibs/hadoop-utils-0.23/src/main/java/org/apache/oozie/action/hadoop/LauncherMainHadoopUtils.java ---------------------------------------------------------------------- diff --git a/hadooplibs/hadoop-utils-0.23/src/main/java/org/apache/oozie/action/hadoop/LauncherMainHadoopUtils.java b/hadooplibs/hadoop-utils-0.23/src/main/java/org/apache/oozie/action/hadoop/LauncherMainHadoopUtils.java deleted file mode 100644 index dca7820..0000000 --- a/hadooplibs/hadoop-utils-0.23/src/main/java/org/apache/oozie/action/hadoop/LauncherMainHadoopUtils.java +++ /dev/null @@ -1,35 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.oozie.action.hadoop; - -import org.apache.hadoop.conf.Configuration; - - -public class LauncherMainHadoopUtils { - - public static final String CHILD_MAPREDUCE_JOB_TAGS = "oozie.child.mapreduce.job.tags"; - public static final String OOZIE_JOB_LAUNCH_TIME = "oozie.job.launch.time"; - - private LauncherMainHadoopUtils() { - } - - public static void killChildYarnJobs(Configuration actionConf) { - // no-op - } -} http://git-wip-us.apache.org/repos/asf/oozie/blob/2fd64fa6/hadooplibs/hadoop-utils-0.23/src/main/java/org/apache/oozie/hadoop/utils/HadoopShims.java ---------------------------------------------------------------------- diff --git a/hadooplibs/hadoop-utils-0.23/src/main/java/org/apache/oozie/hadoop/utils/HadoopShims.java b/hadooplibs/hadoop-utils-0.23/src/main/java/org/apache/oozie/hadoop/utils/HadoopShims.java deleted file mode 100644 index 799dffb..0000000 --- a/hadooplibs/hadoop-utils-0.23/src/main/java/org/apache/oozie/hadoop/utils/HadoopShims.java +++ /dev/null @@ -1,51 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.oozie.hadoop.utils; - -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import java.io.IOException; - -public class HadoopShims { - FileSystem fs; - - public HadoopShims(FileSystem fs) { - this.fs = fs; - } - - public static boolean isSymlinkSupported() { - return false; - } - - public Path getSymLinkTarget(Path p) throws IOException { - return p; - } - - public boolean isSymlink(Path p) throws IOException { - return false; - } - - public void createSymlink(Path target, Path link, boolean createParent) throws IOException { - } - - public static boolean isYARN() { - return false; - } - -} http://git-wip-us.apache.org/repos/asf/oozie/blob/2fd64fa6/hadooplibs/pom.xml ---------------------------------------------------------------------- diff --git a/hadooplibs/pom.xml b/hadooplibs/pom.xml index 17f1f07..cd0f478 100644 --- a/hadooplibs/pom.xml +++ b/hadooplibs/pom.xml @@ -56,17 +56,6 @@ </modules> </profile> <profile> - <id>hadoop-0.23</id> - <activation> - <activeByDefault>false</activeByDefault> - </activation> - <modules> - <module>hadoop-utils-0.23</module> - <module>hadoop-distcp-0.23</module> - <module>hadoop-auth-0.23</module> - </modules> - </profile> - <profile> <id>hadoop-2</id> <activation> <activeByDefault>false</activeByDefault> http://git-wip-us.apache.org/repos/asf/oozie/blob/2fd64fa6/pom.xml ---------------------------------------------------------------------- diff --git a/pom.xml b/pom.xml index 5a8e5f5..2cbc91f 100644 --- a/pom.xml +++ b/pom.xml @@ -1797,18 +1797,6 @@ </properties> </profile> <profile> - <id>hadoop-0.23</id> - <activation> - <activeByDefault>false</activeByDefault> - </activation> - <properties> - <hadoop.version>0.23.5</hadoop.version> - <hadoop.majorversion>0.23</hadoop.majorversion> - <pig.classifier>h2</pig.classifier> - <sqoop.classifier>hadoop23</sqoop.classifier> - </properties> - </profile> - <profile> <id>hadoop-2</id> <activation> <activeByDefault>false</activeByDefault> http://git-wip-us.apache.org/repos/asf/oozie/blob/2fd64fa6/release-log.txt ---------------------------------------------------------------------- diff --git a/release-log.txt b/release-log.txt index d1e36be..681e5ee 100644 --- a/release-log.txt +++ b/release-log.txt @@ -1,5 +1,6 @@ -- Oozie 4.3.0 release (trunk - unreleased) +OOZIE-2675 Drop support for hadoop 0.23 (abhishekbafna via rohini) OOZIE-2588 Support getting credentials for same cluster hcat when credentials config is empty (satishsaley via rohini) OOZIE-2538 Update HttpClient versions to close security vulnerabilities (abhishekbafna via rkanter) OOZIE-2037 Add TLSv1.1,TLSv1.2 (rkanter)