http://git-wip-us.apache.org/repos/asf/flink/blob/8b8dfc16/flink-contrib/flink-tez/src/test/resources/log4j-test.properties ---------------------------------------------------------------------- diff --git a/flink-contrib/flink-tez/src/test/resources/log4j-test.properties b/flink-contrib/flink-tez/src/test/resources/log4j-test.properties deleted file mode 100644 index 0845c81..0000000 --- a/flink-contrib/flink-tez/src/test/resources/log4j-test.properties +++ /dev/null @@ -1,30 +0,0 @@ -################################################################################ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -################################################################################ - -# Set root logger level to OFF to not flood build logs -# set manually to INFO for debugging purposes -log4j.rootLogger=INFO, testlogger - -# A1 is set to be a ConsoleAppender. -log4j.appender.testlogger=org.apache.log4j.ConsoleAppender -log4j.appender.testlogger.target = System.err -log4j.appender.testlogger.layout=org.apache.log4j.PatternLayout -log4j.appender.testlogger.layout.ConversionPattern=%-4r [%t] %-5p %c %x - %m%n - -# suppress the irrelevant (wrong) warnings from the netty channel handler -log4j.logger.org.jboss.netty.channel.DefaultChannelPipeline=ERROR, testlogger \ No newline at end of file
http://git-wip-us.apache.org/repos/asf/flink/blob/8b8dfc16/flink-contrib/flink-tez/src/test/resources/logback-test.xml ---------------------------------------------------------------------- diff --git a/flink-contrib/flink-tez/src/test/resources/logback-test.xml b/flink-contrib/flink-tez/src/test/resources/logback-test.xml deleted file mode 100644 index 48e4374..0000000 --- a/flink-contrib/flink-tez/src/test/resources/logback-test.xml +++ /dev/null @@ -1,37 +0,0 @@ -<!-- - ~ Licensed to the Apache Software Foundation (ASF) under one - ~ or more contributor license agreements. See the NOTICE file - ~ distributed with this work for additional information - ~ regarding copyright ownership. The ASF licenses this file - ~ to you under the Apache License, Version 2.0 (the - ~ "License"); you may not use this file except in compliance - ~ with the License. You may obtain a copy of the License at - ~ - ~ http://www.apache.org/licenses/LICENSE-2.0 - ~ - ~ Unless required by applicable law or agreed to in writing, software - ~ distributed under the License is distributed on an "AS IS" BASIS, - ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - ~ See the License for the specific language governing permissions and - ~ limitations under the License. - --> - -<configuration> - <appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender"> - <encoder> - <pattern>%d{HH:mm:ss.SSS} %-5level [%thread] %logger{60} - %msg%n</pattern> - </encoder> - </appender> - - <root level="WARN"> - <appender-ref ref="STDOUT"/> - </root> - - <!--<logger name="org.apache.flink.runtime.operators.BatchTask" level="OFF"/>--> - <!--<logger name="org.apache.flink.runtime.client.JobClient" level="OFF"/>--> - <!--<logger name="org.apache.flink.runtime.taskmanager.Task" level="OFF"/>--> - <!--<logger name="org.apache.flink.runtime.jobmanager.JobManager" level="OFF"/>--> - <!--<logger name="org.apache.flink.runtime.taskmanager.TaskManager" level="OFF"/>--> - <!--<logger name="org.apache.flink.runtime.executiongraph.ExecutionGraph" level="OFF"/>--> - <!--<logger name="org.apache.flink.runtime.jobmanager.EventCollector" level="OFF"/>--> -</configuration> \ No newline at end of file http://git-wip-us.apache.org/repos/asf/flink/blob/8b8dfc16/flink-contrib/pom.xml ---------------------------------------------------------------------- diff --git a/flink-contrib/pom.xml b/flink-contrib/pom.xml index 5901621..78b0d76 100644 --- a/flink-contrib/pom.xml +++ b/flink-contrib/pom.xml @@ -44,15 +44,4 @@ under the License. <module>flink-operator-stats</module> <module>flink-connector-wikiedits</module> </modules> - - - <profiles> - <profile> - <id>include-tez</id> - <modules> - <module>flink-tez</module> - </modules> - </profile> - </profiles> - </project> http://git-wip-us.apache.org/repos/asf/flink/blob/8b8dfc16/flink-quickstart/flink-tez-quickstart/pom.xml ---------------------------------------------------------------------- diff --git a/flink-quickstart/flink-tez-quickstart/pom.xml b/flink-quickstart/flink-tez-quickstart/pom.xml deleted file mode 100644 index 495fa01..0000000 --- a/flink-quickstart/flink-tez-quickstart/pom.xml +++ /dev/null @@ -1,37 +0,0 @@ -<!-- -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, -software distributed under the License is distributed on an -"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -KIND, either express or implied. See the License for the -specific language governing permissions and limitations -under the License. ---> -<project xmlns="http://maven.apache.org/POM/4.0.0"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; - xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd";> - <modelVersion>4.0.0</modelVersion> - - <properties> - <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding> - </properties> - - <parent> - <groupId>org.apache.flink</groupId> - <artifactId>flink-quickstart</artifactId> - <version>1.0-SNAPSHOT</version> - <relativePath>..</relativePath> - </parent> - - <artifactId>flink-tez-quickstart</artifactId> - <packaging>maven-archetype</packaging> - -</project> http://git-wip-us.apache.org/repos/asf/flink/blob/8b8dfc16/flink-quickstart/flink-tez-quickstart/src/main/java/Dummy.java ---------------------------------------------------------------------- diff --git a/flink-quickstart/flink-tez-quickstart/src/main/java/Dummy.java b/flink-quickstart/flink-tez-quickstart/src/main/java/Dummy.java deleted file mode 100644 index c7749ff..0000000 --- a/flink-quickstart/flink-tez-quickstart/src/main/java/Dummy.java +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.flink.quickstart; - -/** - * This class solely exists to generate - * javadocs for the "quickstart-java" project. - **/ -public class Dummy { - // -} - http://git-wip-us.apache.org/repos/asf/flink/blob/8b8dfc16/flink-quickstart/flink-tez-quickstart/src/main/resources/META-INF/maven/archetype-metadata.xml ---------------------------------------------------------------------- diff --git a/flink-quickstart/flink-tez-quickstart/src/main/resources/META-INF/maven/archetype-metadata.xml b/flink-quickstart/flink-tez-quickstart/src/main/resources/META-INF/maven/archetype-metadata.xml deleted file mode 100644 index adb3b83..0000000 --- a/flink-quickstart/flink-tez-quickstart/src/main/resources/META-INF/maven/archetype-metadata.xml +++ /dev/null @@ -1,36 +0,0 @@ -<!-- -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, -software distributed under the License is distributed on an -"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -KIND, either express or implied. See the License for the -specific language governing permissions and limitations -under the License. ---> - -<archetype-descriptor - xmlns="http://maven.apache.org/plugins/maven-archetype-plugin/archetype-descriptor/1.0.0"; - xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; - xsi:schemaLocation="http://maven.apache.org/plugins/maven-archetype-plugin/archetype-descriptor/1.0.0 http://maven.apache.org/xsd/archetype-descriptor-1.0.0.xsd"; - name="flink-tez-quickstart"> - <fileSets> - <fileSet filtered="true" packaged="true" encoding="UTF-8"> - <directory>src/main/java</directory> - <includes> - <include>**/*.java</include> - </includes> - </fileSet> - <fileSet encoding="UTF-8"> - <directory>src/assembly</directory> - </fileSet> - </fileSets> -</archetype-descriptor> \ No newline at end of file http://git-wip-us.apache.org/repos/asf/flink/blob/8b8dfc16/flink-quickstart/flink-tez-quickstart/src/main/resources/archetype-resources/pom.xml ---------------------------------------------------------------------- diff --git a/flink-quickstart/flink-tez-quickstart/src/main/resources/archetype-resources/pom.xml b/flink-quickstart/flink-tez-quickstart/src/main/resources/archetype-resources/pom.xml deleted file mode 100644 index e175a6b..0000000 --- a/flink-quickstart/flink-tez-quickstart/src/main/resources/archetype-resources/pom.xml +++ /dev/null @@ -1,186 +0,0 @@ -<!-- -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, -software distributed under the License is distributed on an -"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -KIND, either express or implied. See the License for the -specific language governing permissions and limitations -under the License. ---> -<project xmlns="http://maven.apache.org/POM/4.0.0"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; - xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd";> - <modelVersion>4.0.0</modelVersion> - - <groupId>${groupId}</groupId> - <artifactId>${artifactId}</artifactId> - <version>${version}</version> - <packaging>jar</packaging> - - <name>Your Job's Name</name> - <url>http://www.myorganization.org</url> - - <properties> - <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding> - </properties> - - <repositories> - <repository> - <id>apache.snapshots</id> - <name>Apache Development Snapshot Repository</name> - <url>https://repository.apache.org/content/repositories/snapshots/</url> - <releases> - <enabled>false</enabled> - </releases> - <snapshots> - <enabled>true</enabled> - </snapshots> - </repository> - </repositories> - - <!-- These two requirements are the minimum to use and develop Flink. - You can add others like <artifactId>flink-scala</artifactId> for Scala! --> - <dependencies> - <dependency> - <groupId>org.apache.flink</groupId> - <artifactId>flink-java</artifactId> - <version>1.0-SNAPSHOT</version> - </dependency> - <dependency> - <groupId>org.apache.flink</groupId> - <artifactId>flink-clients</artifactId> - <version>1.0-SNAPSHOT</version> - </dependency> - <dependency> - <groupId>org.apache.flink</groupId> - <artifactId>flink-tez</artifactId> - <version>1.0-SNAPSHOT</version> - </dependency> - </dependencies> - - <build> - <plugins> - - <!-- We use the maven-assembly plugin to create a fat jar that contains all dependencies - except flink and it's transitive dependencies. The resulting fat-jar can be executed - on a cluster. Change the value of Program-Class if your program entry point changes. --> - <plugin> - <artifactId>maven-assembly-plugin</artifactId> - <version>2.4.1</version> - <configuration> - <descriptors> - <descriptor>src/assembly/flink-fat-jar.xml</descriptor> - </descriptors> - <archive> - <manifest> - <mainClass>${package}.Driver</mainClass> - </manifest> - </archive> - </configuration> - <executions> - <execution> - <id>make-assembly</id> - <phase>package</phase> - <goals> - <goal>single</goal> - </goals> - </execution> - </executions> - </plugin> - - <!-- Configure the jar plugin to add the main class as a manifest entry --> - <plugin> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-jar-plugin</artifactId> - <configuration> - <archive> - <manifestEntries> - <Main-Class>${package}.Driver</Main-Class> - </manifestEntries> - </archive> - </configuration> - </plugin> - - <plugin> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-compiler-plugin</artifactId> - <version>3.1</version> - <configuration> - <source>1.7</source> <!-- If you want to use Java 8, change this to "1.8" --> - <target>1.7</target> <!-- If you want to use Java 8, change this to "1.8" --> - </configuration> - </plugin> - </plugins> - - - <!-- If you want to use Java 8 Lambda Expressions uncomment the following lines --> - <!-- - <pluginManagement> - <plugins> - <plugin> - <artifactId>maven-compiler-plugin</artifactId> - <configuration> - <source>1.8</source> - <target>1.8</target> - <compilerId>jdt</compilerId> - </configuration> - <dependencies> - <dependency> - <groupId>org.eclipse.tycho</groupId> - <artifactId>tycho-compiler-jdt</artifactId> - <version>0.21.0</version> - </dependency> - </dependencies> - </plugin> - - <plugin> - <groupId>org.eclipse.m2e</groupId> - <artifactId>lifecycle-mapping</artifactId> - <version>1.0.0</version> - <configuration> - <lifecycleMappingMetadata> - <pluginExecutions> - <pluginExecution> - <pluginExecutionFilter> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-assembly-plugin</artifactId> - <versionRange>[2.4,)</versionRange> - <goals> - <goal>single</goal> - </goals> - </pluginExecutionFilter> - <action> - <ignore/> - </action> - </pluginExecution> - <pluginExecution> - <pluginExecutionFilter> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-compiler-plugin</artifactId> - <versionRange>[3.1,)</versionRange> - <goals> - <goal>testCompile</goal> - </goals> - </pluginExecutionFilter> - <action> - <ignore/> - </action> - </pluginExecution> - </pluginExecutions> - </lifecycleMappingMetadata> - </configuration> - </plugin> - </plugins> - </pluginManagement> - --> - - </build> -</project> http://git-wip-us.apache.org/repos/asf/flink/blob/8b8dfc16/flink-quickstart/flink-tez-quickstart/src/main/resources/archetype-resources/src/assembly/flink-fat-jar.xml ---------------------------------------------------------------------- diff --git a/flink-quickstart/flink-tez-quickstart/src/main/resources/archetype-resources/src/assembly/flink-fat-jar.xml b/flink-quickstart/flink-tez-quickstart/src/main/resources/archetype-resources/src/assembly/flink-fat-jar.xml deleted file mode 100644 index f9f27e5..0000000 --- a/flink-quickstart/flink-tez-quickstart/src/main/resources/archetype-resources/src/assembly/flink-fat-jar.xml +++ /dev/null @@ -1,40 +0,0 @@ -<!-- - ~ Licensed to the Apache Software Foundation (ASF) under one - ~ or more contributor license agreements. See the NOTICE file - ~ distributed with this work for additional information - ~ regarding copyright ownership. The ASF licenses this file - ~ to you under the Apache License, Version 2.0 (the - ~ "License"); you may not use this file except in compliance - ~ with the License. You may obtain a copy of the License at - ~ - ~ http://www.apache.org/licenses/LICENSE-2.0 - ~ - ~ Unless required by applicable law or agreed to in writing, software - ~ distributed under the License is distributed on an "AS IS" BASIS, - ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - ~ See the License for the specific language governing permissions and - ~ limitations under the License. - --> - -<assembly xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2"; - xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; - xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2 http://maven.apache.org/xsd/assembly-1.1.2.xsd";> - <id>flink-fat-jar</id> - <formats> - <format>jar</format> - </formats> - <includeBaseDirectory>false</includeBaseDirectory> - <dependencySets> - <dependencySet> - <outputDirectory>/</outputDirectory> - <useProjectArtifact>true</useProjectArtifact> - <excludes> - <exclude>org.apache.tez:*</exclude> - <exclude>org.apache.hadoop:*</exclude> - </excludes> - <useTransitiveFiltering>true</useTransitiveFiltering> - <unpack>true</unpack> - <scope>runtime</scope> - </dependencySet> - </dependencySets> -</assembly> \ No newline at end of file http://git-wip-us.apache.org/repos/asf/flink/blob/8b8dfc16/flink-quickstart/flink-tez-quickstart/src/main/resources/archetype-resources/src/main/java/Driver.java ---------------------------------------------------------------------- diff --git a/flink-quickstart/flink-tez-quickstart/src/main/resources/archetype-resources/src/main/java/Driver.java b/flink-quickstart/flink-tez-quickstart/src/main/resources/archetype-resources/src/main/java/Driver.java deleted file mode 100644 index f72eb22..0000000 --- a/flink-quickstart/flink-tez-quickstart/src/main/resources/archetype-resources/src/main/java/Driver.java +++ /dev/null @@ -1,113 +0,0 @@ -package ${package}; - -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import org.apache.hadoop.util.ProgramDriver; -import org.apache.tez.common.counters.TezCounters; -import org.apache.tez.dag.api.TezException; -import org.apache.tez.dag.api.client.DAGClient; -import org.apache.tez.dag.api.client.DAGStatus; -import org.apache.tez.dag.api.client.Progress; -import org.apache.tez.dag.api.client.StatusGetOpts; -import org.apache.tez.dag.api.client.VertexStatus; - -import java.io.IOException; -import java.text.DecimalFormat; -import java.util.EnumSet; -import java.util.Set; - -public class Driver { - - private static final DecimalFormat formatter = new DecimalFormat("###.##%"); - - public static void main(String [] args){ - int exitCode = -1; - ProgramDriver pgd = new ProgramDriver(); - try { - pgd.addClass("yarnjob", YarnJob.class, - "Yarn Job"); - pgd.addClass("wc", YarnWordCount.class, - "Word Count"); - exitCode = pgd.run(args); - } catch(Throwable e){ - e.printStackTrace(); - } - System.exit(exitCode); - } - - public static void printDAGStatus(DAGClient dagClient, String[] vertexNames) - throws IOException, TezException { - printDAGStatus(dagClient, vertexNames, false, false); - } - - public static void printDAGStatus(DAGClient dagClient, String[] vertexNames, boolean displayDAGCounters, boolean displayVertexCounters) - throws IOException, TezException { - Set<StatusGetOpts> opts = EnumSet.of(StatusGetOpts.GET_COUNTERS); - DAGStatus dagStatus = dagClient.getDAGStatus( - (displayDAGCounters ? opts : null)); - Progress progress = dagStatus.getDAGProgress(); - double vProgressFloat = 0.0f; - if (progress != null) { - System.out.println(""); - System.out.println("DAG: State: " - + dagStatus.getState() - + " Progress: " - + (progress.getTotalTaskCount() < 0 ? formatter.format(0.0f) : - formatter.format((double)(progress.getSucceededTaskCount()) - /progress.getTotalTaskCount()))); - for (String vertexName : vertexNames) { - VertexStatus vStatus = dagClient.getVertexStatus(vertexName, - (displayVertexCounters ? opts : null)); - if (vStatus == null) { - System.out.println("Could not retrieve status for vertex: " - + vertexName); - continue; - } - Progress vProgress = vStatus.getProgress(); - if (vProgress != null) { - vProgressFloat = 0.0f; - if (vProgress.getTotalTaskCount() == 0) { - vProgressFloat = 1.0f; - } else if (vProgress.getTotalTaskCount() > 0) { - vProgressFloat = (double)vProgress.getSucceededTaskCount() - /vProgress.getTotalTaskCount(); - } - System.out.println("VertexStatus:" - + " VertexName: " - + (vertexName.equals("ivertex1") ? "intermediate-reducer" - : vertexName) - + " Progress: " + formatter.format(vProgressFloat)); - } - if (displayVertexCounters) { - TezCounters counters = vStatus.getVertexCounters(); - if (counters != null) { - System.out.println("Vertex Counters for " + vertexName + ": " - + counters); - } - } - } - } - if (displayDAGCounters) { - TezCounters counters = dagStatus.getDAGCounters(); - if (counters != null) { - System.out.println("DAG Counters: " + counters); - } - } - } -} http://git-wip-us.apache.org/repos/asf/flink/blob/8b8dfc16/flink-quickstart/flink-tez-quickstart/src/main/resources/archetype-resources/src/main/java/LocalJob.java ---------------------------------------------------------------------- diff --git a/flink-quickstart/flink-tez-quickstart/src/main/resources/archetype-resources/src/main/java/LocalJob.java b/flink-quickstart/flink-tez-quickstart/src/main/resources/archetype-resources/src/main/java/LocalJob.java deleted file mode 100644 index cf7474e..0000000 --- a/flink-quickstart/flink-tez-quickstart/src/main/resources/archetype-resources/src/main/java/LocalJob.java +++ /dev/null @@ -1,72 +0,0 @@ -package ${package}; - -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import org.apache.flink.tez.client.LocalTezEnvironment; - -/** - * Skeleton for a Flink on Tez Job running using Tez local mode. - * - * For a full example of a Flink on TezJob, see the WordCountJob.java file in the - * same package/directory or have a look at the website. - * - * You can also generate a .jar file that you can submit on your Flink - * cluster. - * Just type - * mvn clean package - * in the projects root directory. - * You will find the jar in - * target/flink-quickstart-0.1-SNAPSHOT-Sample.jar - * - */ -public class LocalJob { - - public static void main(String[] args) throws Exception { - // set up the execution environment - - // To use Tez YARN execution, use - final LocalTezEnvironment env = LocalTezEnvironment.create(); - - /** - * Here, you can start creating your execution plan for Flink. - * - * Start with getting some data from the environment, like - * env.readTextFile(textPath); - * - * then, transform the resulting DataSet<String> using operations - * like - * .filter() - * .flatMap() - * .join() - * .coGroup() - * and many more. - * Have a look at the programming guide for the Java API: - * - * http://flink.apache.org/docs/latest/programming_guide.html - * - * and the examples - * - * http://flink.apache.org/docs/latest/examples.html - * - */ - - // execute program - env.execute("Flink Java API Skeleton"); - } -} http://git-wip-us.apache.org/repos/asf/flink/blob/8b8dfc16/flink-quickstart/flink-tez-quickstart/src/main/resources/archetype-resources/src/main/java/LocalWordCount.java ---------------------------------------------------------------------- diff --git a/flink-quickstart/flink-tez-quickstart/src/main/resources/archetype-resources/src/main/java/LocalWordCount.java b/flink-quickstart/flink-tez-quickstart/src/main/resources/archetype-resources/src/main/java/LocalWordCount.java deleted file mode 100644 index dbe81a7..0000000 --- a/flink-quickstart/flink-tez-quickstart/src/main/resources/archetype-resources/src/main/java/LocalWordCount.java +++ /dev/null @@ -1,96 +0,0 @@ -package ${package}; - -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import org.apache.flink.api.java.DataSet; -import org.apache.flink.tez.client.LocalTezEnvironment; -import org.apache.flink.api.common.functions.FlatMapFunction; -import org.apache.flink.api.java.tuple.Tuple2; -import org.apache.flink.util.Collector; - -/** - * Implements the "WordCount" program that computes a simple word occurrence histogram - * over some sample data - * - * <p> - * This example shows how to: - * <ul> - * <li>write a simple Flink program. - * <li>use Tuple data types. - * <li>write and use user-defined functions. - * </ul> - * - */ -public class LocalWordCount { - - // - // Program - // - - public static void main(String[] args) throws Exception { - - // set up the execution environment - final LocalTezEnvironment env = LocalTezEnvironment.create(); - - // get input data - DataSet<String> text = env.fromElements( - "To be, or not to be,--that is the question:--", - "Whether 'tis nobler in the mind to suffer", - "The slings and arrows of outrageous fortune", - "Or to take arms against a sea of troubles," - ); - - DataSet<Tuple2<String, Integer>> counts = - // split up the lines in pairs (2-tuples) containing: (word,1) - text.flatMap(new LineSplitter()) - // group by the tuple field "0" and sum up tuple field "1" - .groupBy(0) - .sum(1); - - // emit result - counts.print(); - - env.execute("WordCount Example"); - } - - // - // User Functions - // - - /** - * Implements the string tokenizer that splits sentences into words as a user-defined - * FlatMapFunction. The function takes a line (String) and splits it into - * multiple pairs in the form of "(word,1)" (Tuple2<String, Integer>). - */ - public static final class LineSplitter implements FlatMapFunction<String, Tuple2<String, Integer>> { - - @Override - public void flatMap(String value, Collector<Tuple2<String, Integer>> out) { - // normalize and split the line - String[] tokens = value.toLowerCase().split("\\W+"); - - // emit the pairs - for (String token : tokens) { - if (token.length() > 0) { - out.collect(new Tuple2<String, Integer>(token, 1)); - } - } - } - } -} http://git-wip-us.apache.org/repos/asf/flink/blob/8b8dfc16/flink-quickstart/flink-tez-quickstart/src/main/resources/archetype-resources/src/main/java/YarnJob.java ---------------------------------------------------------------------- diff --git a/flink-quickstart/flink-tez-quickstart/src/main/resources/archetype-resources/src/main/java/YarnJob.java b/flink-quickstart/flink-tez-quickstart/src/main/resources/archetype-resources/src/main/java/YarnJob.java deleted file mode 100644 index 30885f1..0000000 --- a/flink-quickstart/flink-tez-quickstart/src/main/resources/archetype-resources/src/main/java/YarnJob.java +++ /dev/null @@ -1,75 +0,0 @@ -package ${package}; - -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import org.apache.flink.tez.client.RemoteTezEnvironment; - -/** - * Skeleton for a Flink on Tez program running on Yarn. - * - * For a full example of a Flink on Tez program, see the WordCountJob.java file in the - * same package/directory or have a look at the website. - * - * You can also generate a .jar file that you can submit on your Flink - * cluster. - * Just type - * mvn clean package - * in the projects root directory. - * You will find the jar in - * target/flink-quickstart-0.1-SNAPSHOT-Sample.jar - * - */ -public class YarnJob { - - public static void main(String[] args) throws Exception { - // set up the execution environment - - // To use Tez YARN execution, use - final RemoteTezEnvironment env = RemoteTezEnvironment.create(); - env.setParallelism(8); - - /** - * Here, you can start creating your execution plan for Flink. - * - * Start with getting some data from the environment, like - * env.readTextFile(textPath); - * - * then, transform the resulting DataSet<String> using operations - * like - * .filter() - * .flatMap() - * .join() - * .coGroup() - * and many more. - * Have a look at the programming guide for the Java API: - * - * http://flink.apache.org/docs/latest/programming_guide.html - * - * and the examples - * - * http://flink.apache.org/docs/latest/examples.html - * - */ - - - // execute program - env.registerMainClass(YarnJob.class); - env.execute("Flink Java API Skeleton"); - } -} http://git-wip-us.apache.org/repos/asf/flink/blob/8b8dfc16/flink-quickstart/flink-tez-quickstart/src/main/resources/archetype-resources/src/main/java/YarnWordCount.java ---------------------------------------------------------------------- diff --git a/flink-quickstart/flink-tez-quickstart/src/main/resources/archetype-resources/src/main/java/YarnWordCount.java b/flink-quickstart/flink-tez-quickstart/src/main/resources/archetype-resources/src/main/java/YarnWordCount.java deleted file mode 100644 index 5f9ef74..0000000 --- a/flink-quickstart/flink-tez-quickstart/src/main/resources/archetype-resources/src/main/java/YarnWordCount.java +++ /dev/null @@ -1,124 +0,0 @@ -package ${package}; - -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - - import org.apache.flink.api.java.DataSet; - import org.apache.flink.tez.client.RemoteTezEnvironment; - import org.apache.flink.api.common.functions.FlatMapFunction; - import org.apache.flink.api.java.tuple.Tuple2; - import org.apache.flink.util.Collector; - -/** - * Implements the "WordCount" program that computes a simple word occurrence histogram - * over some sample data - * - * <p> - * This example shows how to: - * <ul> - * <li>write a simple Flink program. - * <li>use Tuple data types. - * <li>write and use user-defined functions. - * </ul> - * - */ -public class YarnWordCount { - - // - // Program - // - - public static void main(String[] args) throws Exception { - - if(!parseParameters(args)) { - return; - } - - // set up the execution environment - final RemoteTezEnvironment env = RemoteTezEnvironment.create(); - env.setParallelism(parallelism); - - // get input data - DataSet<String> text = env.readTextFile(textPath); - - DataSet<Tuple2<String, Integer>> counts = - // split up the lines in pairs (2-tuples) containing: (word,1) - text.flatMap(new LineSplitter()) - // group by the tuple field "0" and sum up tuple field "1" - .groupBy(0) - .sum(1); - - // emit result - counts.writeAsCsv(outputPath, "\n", " "); - - // execute program - env.registerMainClass (YarnWordCount.class); - env.execute("WordCount Example"); - } - - // - // User Functions - // - - /** - * Implements the string tokenizer that splits sentences into words as a user-defined - * FlatMapFunction. The function takes a line (String) and splits it into - * multiple pairs in the form of "(word,1)" (Tuple2<String, Integer>). - */ - public static final class LineSplitter implements FlatMapFunction<String, Tuple2<String, Integer>> { - - @Override - public void flatMap(String value, Collector<Tuple2<String, Integer>> out) { - // normalize and split the line - String[] tokens = value.toLowerCase().split("\\W+"); - - // emit the pairs - for (String token : tokens) { - if (token.length() > 0) { - out.collect(new Tuple2<String, Integer>(token, 1)); - } - } - } - } - - private static boolean fileOutput = false; - private static String textPath; - private static String outputPath; - private static int parallelism; - - private static boolean parseParameters(String[] args) { - - if(args.length > 0) { - // parse input arguments - fileOutput = true; - if(args.length == 3) { - textPath = args[0]; - outputPath = args[1]; - parallelism = Integer.parseInt(args[2]); - } else { - System.err.println("Usage: YarnWordCount <text path> <result path> <parallelism>"); - return false; - } - } else { - System.out.println("Executing WordCount example with built-in default data."); - System.out.println(" Provide parameters to read input data from a file."); - System.out.println(" Usage: YarnWordCount <text path> <result path> <parallelism>"); - } - return true; - } -} http://git-wip-us.apache.org/repos/asf/flink/blob/8b8dfc16/flink-quickstart/flink-tez-quickstart/src/test/resources/projects/testArtifact/archetype.properties ---------------------------------------------------------------------- diff --git a/flink-quickstart/flink-tez-quickstart/src/test/resources/projects/testArtifact/archetype.properties b/flink-quickstart/flink-tez-quickstart/src/test/resources/projects/testArtifact/archetype.properties deleted file mode 100644 index bfce480..0000000 --- a/flink-quickstart/flink-tez-quickstart/src/test/resources/projects/testArtifact/archetype.properties +++ /dev/null @@ -1,21 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -groupId=org.apache.flink.archetypetest -artifactId=testArtifact -version=0.1 -package=org.apache.flink.archetypetest http://git-wip-us.apache.org/repos/asf/flink/blob/8b8dfc16/flink-quickstart/flink-tez-quickstart/src/test/resources/projects/testArtifact/goal.txt ---------------------------------------------------------------------- diff --git a/flink-quickstart/flink-tez-quickstart/src/test/resources/projects/testArtifact/goal.txt b/flink-quickstart/flink-tez-quickstart/src/test/resources/projects/testArtifact/goal.txt deleted file mode 100644 index f8808ba..0000000 --- a/flink-quickstart/flink-tez-quickstart/src/test/resources/projects/testArtifact/goal.txt +++ /dev/null @@ -1 +0,0 @@ -compile \ No newline at end of file http://git-wip-us.apache.org/repos/asf/flink/blob/8b8dfc16/flink-quickstart/pom.xml ---------------------------------------------------------------------- diff --git a/flink-quickstart/pom.xml b/flink-quickstart/pom.xml index cb12b1d..5d5cea3 100644 --- a/flink-quickstart/pom.xml +++ b/flink-quickstart/pom.xml @@ -74,14 +74,4 @@ under the License. </plugin> </plugins> </build> - - <!-- See main pom.xml for explanation of profiles --> - <profiles> - <profile> - <id>include-tez</id> - <modules> - <module>flink-tez-quickstart</module> - </modules> - </profile> - </profiles> </project> http://git-wip-us.apache.org/repos/asf/flink/blob/8b8dfc16/pom.xml ---------------------------------------------------------------------- diff --git a/pom.xml b/pom.xml index e503a11..8ae24dd 100644 --- a/pom.xml +++ b/pom.xml @@ -98,7 +98,6 @@ under the License. <scala.binary.version>2.10</scala.binary.version> <chill.version>0.5.2</chill.version> <asm.version>5.0.4</asm.version> - <tez.version>0.6.1</tez.version> <zookeeper.version>3.4.6</zookeeper.version> <curator.version>2.8.0</curator.version> <jackson.version>2.4.2</jackson.version> @@ -443,14 +442,6 @@ under the License. <module>flink-yarn-tests</module> </modules> </profile> - - <!-- Profile to set the Hadoop 2 version properly for TEZ --> - <profile> - <id>include-tez</id> - <properties> - <hadoop.version>2.6.0</hadoop.version> - </properties> - </profile> <profile> <id>vendor-repos</id>
