Add a Bare-bones ReferenceRunner Job Service This will eventually accept Job API calls and execute a Pipeline using the ReferenceRunner backend.
This change exists primarily to create the appropriate module and POM. Project: http://git-wip-us.apache.org/repos/asf/beam/repo Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/f0394a64 Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/f0394a64 Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/f0394a64 Branch: refs/heads/master Commit: f0394a64c22199b6a15f7eb84ab9ebde8ce069ae Parents: 1259ee9 Author: Thomas Groh <[email protected]> Authored: Fri Aug 4 10:58:32 2017 -0700 Committer: Thomas Groh <[email protected]> Committed: Thu Oct 5 09:15:29 2017 -0700 ---------------------------------------------------------------------- pom.xml | 7 ++ runners/pom.xml | 1 + runners/reference/pom.xml | 75 +++++++++++++++++++ .../reference/job/ReferenceRunnerJobServer.java | 67 +++++++++++++++++ .../job/ReferenceRunnerJobService.java | 77 ++++++++++++++++++++ .../runners/reference/job/package-info.java | 23 ++++++ .../job/ReferenceRunnerJobServiceTest.java | 34 +++++++++ 7 files changed, 284 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/beam/blob/f0394a64/pom.xml ---------------------------------------------------------------------- diff --git a/pom.xml b/pom.xml index 3ccd8d8..d9c2e6d 100644 --- a/pom.xml +++ b/pom.xml @@ -106,6 +106,7 @@ <apache.commons.text.version>1.1</apache.commons.text.version> <apex.kryo.version>2.24.0</apex.kryo.version> <api-common.version>1.0.0-rc2</api-common.version> + <args4j.version>2.33</args4j.version> <avro.version>1.8.2</avro.version> <bigquery.version>v2-rev355-1.22.0</bigquery.version> <bigtable.version>1.0.0-pre3</bigtable.version> @@ -1132,6 +1133,12 @@ </dependency> <dependency> + <groupId>args4j</groupId> + <artifactId>args4j</artifactId> + <version>${args4j.version}</version> + </dependency> + + <dependency> <groupId>org.slf4j</groupId> <artifactId>slf4j-api</artifactId> <version>${slf4j.version}</version> http://git-wip-us.apache.org/repos/asf/beam/blob/f0394a64/runners/pom.xml ---------------------------------------------------------------------- diff --git a/runners/pom.xml b/runners/pom.xml index 4f06748..e0a47bd 100644 --- a/runners/pom.xml +++ b/runners/pom.xml @@ -36,6 +36,7 @@ <module>core-construction-java</module> <module>core-java</module> <module>local-artifact-service-java</module> + <module>reference</module> <module>direct-java</module> <module>flink</module> <module>google-cloud-dataflow-java</module> http://git-wip-us.apache.org/repos/asf/beam/blob/f0394a64/runners/reference/pom.xml ---------------------------------------------------------------------- diff --git a/runners/reference/pom.xml b/runners/reference/pom.xml new file mode 100644 index 0000000..d421786 --- /dev/null +++ b/runners/reference/pom.xml @@ -0,0 +1,75 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<project xmlns="http://maven.apache.org/POM/4.0.0"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd";> + + <modelVersion>4.0.0</modelVersion> + + <parent> + <groupId>org.apache.beam</groupId> + <artifactId>beam-runners-parent</artifactId> + <version>2.2.0-SNAPSHOT</version> + <relativePath>../pom.xml</relativePath> + </parent> + + <artifactId>beam-runners-reference</artifactId> + <name>Apache Beam :: Runners :: Reference</name> + <description>A Pipeline Runner which executes on the local machine using the + Beam portability framework to execute an arbitrary Pipeline.</description> + + <packaging>jar</packaging> + + <dependencies> + <dependency> + <groupId>org.apache.beam</groupId> + <artifactId>beam-sdks-common-runner-api</artifactId> + </dependency> + + <dependency> + <groupId>io.grpc</groupId> + <artifactId>grpc-core</artifactId> + </dependency> + + <dependency> + <groupId>io.grpc</groupId> + <artifactId>grpc-stub</artifactId> + </dependency> + + <dependency> + <groupId>args4j</groupId> + <artifactId>args4j</artifactId> + </dependency> + + <dependency> + <groupId>org.slf4j</groupId> + <artifactId>slf4j-api</artifactId> + </dependency> + + <!-- test dependencies --> + <dependency> + <groupId>junit</groupId> + <artifactId>junit</artifactId> + <scope>test</scope> + </dependency> + + <dependency> + <groupId>org.slf4j</groupId> + <artifactId>slf4j-jdk14</artifactId> + <scope>test</scope> + </dependency> + </dependencies> +</project> http://git-wip-us.apache.org/repos/asf/beam/blob/f0394a64/runners/reference/src/main/java/org/apache/beam/runners/reference/job/ReferenceRunnerJobServer.java ---------------------------------------------------------------------- diff --git a/runners/reference/src/main/java/org/apache/beam/runners/reference/job/ReferenceRunnerJobServer.java b/runners/reference/src/main/java/org/apache/beam/runners/reference/job/ReferenceRunnerJobServer.java new file mode 100644 index 0000000..1dcc2b3 --- /dev/null +++ b/runners/reference/src/main/java/org/apache/beam/runners/reference/job/ReferenceRunnerJobServer.java @@ -0,0 +1,67 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.beam.runners.reference.job; + +import io.grpc.Server; +import io.grpc.ServerBuilder; +import java.io.IOException; +import org.kohsuke.args4j.CmdLineException; +import org.kohsuke.args4j.CmdLineParser; +import org.kohsuke.args4j.Option; + +/** A program that runs a {@link ReferenceRunnerJobService}. */ +public class ReferenceRunnerJobServer { + public static void main(String[] args) throws IOException, InterruptedException { + ServerConfiguration configuration = new ServerConfiguration(); + CmdLineParser parser = new CmdLineParser(configuration); + try { + parser.parseArgument(args); + } catch (CmdLineException e) { + System.err.println(e); + printUsage(parser); + return; + } + runServer(configuration); + } + + private static void printUsage(CmdLineParser parser) { + System.err.println( + String.format( + "Usage: java %s arguments...", ReferenceRunnerJobService.class.getSimpleName())); + parser.printUsage(System.err); + System.err.println(); + } + + private static void runServer(ServerConfiguration configuration) throws IOException, InterruptedException { + ReferenceRunnerJobService service = ReferenceRunnerJobService.create(); + Server server = ServerBuilder.forPort(configuration.port).addService(service).build(); + server.start(); + server.awaitTermination(); + } + + private static class ServerConfiguration { + @Option( + name = "p", + aliases = {"port"}, + required = true, + usage = "The local port to expose the server on" + ) + private int port = -1; + } +} http://git-wip-us.apache.org/repos/asf/beam/blob/f0394a64/runners/reference/src/main/java/org/apache/beam/runners/reference/job/ReferenceRunnerJobService.java ---------------------------------------------------------------------- diff --git a/runners/reference/src/main/java/org/apache/beam/runners/reference/job/ReferenceRunnerJobService.java b/runners/reference/src/main/java/org/apache/beam/runners/reference/job/ReferenceRunnerJobService.java new file mode 100644 index 0000000..9084bdf --- /dev/null +++ b/runners/reference/src/main/java/org/apache/beam/runners/reference/job/ReferenceRunnerJobService.java @@ -0,0 +1,77 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.beam.runners.reference.job; + +import io.grpc.Status; +import io.grpc.stub.StreamObserver; +import org.apache.beam.sdk.common.runner.v1.JobApi; +import org.apache.beam.sdk.common.runner.v1.JobApi.CancelJobRequest; +import org.apache.beam.sdk.common.runner.v1.JobApi.CancelJobResponse; +import org.apache.beam.sdk.common.runner.v1.JobApi.GetJobStateRequest; +import org.apache.beam.sdk.common.runner.v1.JobApi.GetJobStateResponse; +import org.apache.beam.sdk.common.runner.v1.JobApi.PrepareJobResponse; +import org.apache.beam.sdk.common.runner.v1.JobApi.RunJobRequest; +import org.apache.beam.sdk.common.runner.v1.JobServiceGrpc.JobServiceImplBase; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** The ReferenceRunner uses the portability framework to execute a Pipeline on a single machine. */ +public class ReferenceRunnerJobService extends JobServiceImplBase { + private static final Logger LOG = LoggerFactory.getLogger(ReferenceRunnerJobService.class); + + public static ReferenceRunnerJobService create() { + return new ReferenceRunnerJobService(); + } + + private ReferenceRunnerJobService() {} + + @Override + public void prepare( + JobApi.PrepareJobRequest request, + StreamObserver<JobApi.PrepareJobResponse> responseObserver) { + LOG.trace("{} {}", PrepareJobResponse.class.getSimpleName(), request); + responseObserver.onError(Status.UNIMPLEMENTED.asException()); + } + + @Override + public void run( + JobApi.RunJobRequest request, StreamObserver<JobApi.RunJobResponse> responseObserver) { + LOG.trace("{} {}", RunJobRequest.class.getSimpleName(), request); + responseObserver.onError(Status.UNIMPLEMENTED.asException()); + } + + @Override + public void getState( + GetJobStateRequest request, StreamObserver<GetJobStateResponse> responseObserver) { + LOG.trace("{} {}", GetJobStateRequest.class.getSimpleName(), request); + responseObserver.onError( + Status.NOT_FOUND + .withDescription(String.format("Unknown Job ID %s", request.getJobId())) + .asException()); + } + + @Override + public void cancel(CancelJobRequest request, StreamObserver<CancelJobResponse> responseObserver) { + LOG.trace("{} {}", CancelJobRequest.class.getSimpleName(), request); + responseObserver.onError( + Status.NOT_FOUND + .withDescription(String.format("Unknown Job ID %s", request.getJobId())) + .asException()); + } +} http://git-wip-us.apache.org/repos/asf/beam/blob/f0394a64/runners/reference/src/main/java/org/apache/beam/runners/reference/job/package-info.java ---------------------------------------------------------------------- diff --git a/runners/reference/src/main/java/org/apache/beam/runners/reference/job/package-info.java b/runners/reference/src/main/java/org/apache/beam/runners/reference/job/package-info.java new file mode 100644 index 0000000..b6022d9 --- /dev/null +++ b/runners/reference/src/main/java/org/apache/beam/runners/reference/job/package-info.java @@ -0,0 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * An execution engine for Beam Pipelines that uses the Java Runner harness and the Fn API to + * execute. + */ +package org.apache.beam.runners.reference.job; http://git-wip-us.apache.org/repos/asf/beam/blob/f0394a64/runners/reference/src/test/java/org/apache/beam/runners/reference/job/ReferenceRunnerJobServiceTest.java ---------------------------------------------------------------------- diff --git a/runners/reference/src/test/java/org/apache/beam/runners/reference/job/ReferenceRunnerJobServiceTest.java b/runners/reference/src/test/java/org/apache/beam/runners/reference/job/ReferenceRunnerJobServiceTest.java new file mode 100644 index 0000000..16cde11 --- /dev/null +++ b/runners/reference/src/test/java/org/apache/beam/runners/reference/job/ReferenceRunnerJobServiceTest.java @@ -0,0 +1,34 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.beam.runners.reference.job; + +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +/** + * Tests for {@link ReferenceRunnerJobService}. + */ +@RunWith(JUnit4.class) +public class ReferenceRunnerJobServiceTest { + @Test + public void testPrepareJob() { + // TODO: Implement when PrepareJob is implemented. + } +}
