[
https://issues.apache.org/jira/browse/BEAM-4376?focusedWorklogId=142485&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-142485
]
ASF GitHub Bot logged work on BEAM-4376:
----------------------------------------
Author: ASF GitHub Bot
Created on: 08/Sep/18 23:46
Start Date: 08/Sep/18 23:46
Worklog Time Spent: 10m
Work Description: stale[bot] closed pull request #5549: [BEAM-4376] Add
Nexmark events generation based on ParDo
URL: https://github.com/apache/beam/pull/5549
This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:
As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):
diff --git
a/sdks/java/nexmark/src/main/java/org/apache/beam/sdk/nexmark/NexmarkLauncher.java
b/sdks/java/nexmark/src/main/java/org/apache/beam/sdk/nexmark/NexmarkLauncher.java
index bbb7496bf8c..f2e620ffd23 100644
---
a/sdks/java/nexmark/src/main/java/org/apache/beam/sdk/nexmark/NexmarkLauncher.java
+++
b/sdks/java/nexmark/src/main/java/org/apache/beam/sdk/nexmark/NexmarkLauncher.java
@@ -87,9 +87,12 @@
import org.apache.beam.sdk.nexmark.queries.sql.SqlQuery3;
import org.apache.beam.sdk.nexmark.queries.sql.SqlQuery5;
import org.apache.beam.sdk.nexmark.queries.sql.SqlQuery7;
+import org.apache.beam.sdk.nexmark.sources.EventGeneratorDoFn;
import org.apache.beam.sdk.testing.PAssert;
+import org.apache.beam.sdk.transforms.Create;
import org.apache.beam.sdk.transforms.DoFn;
import org.apache.beam.sdk.transforms.ParDo;
+import org.apache.beam.sdk.transforms.Reshuffle;
import org.apache.beam.sdk.util.CoderUtils;
import org.apache.beam.sdk.values.KV;
import org.apache.beam.sdk.values.PCollection;
@@ -163,11 +166,6 @@ public NexmarkLauncher(OptionT options) {
this.options = options;
}
- /** Is this query running in streaming mode? */
- private boolean isStreaming() {
- return options.isStreaming();
- }
-
/** Return maximum number of workers. */
private int maxNumWorkers() {
return 5;
@@ -760,7 +758,15 @@ private String logsDir(long now) {
/** Return a source of synthetic events. */
private PCollection<Event> sourceEventsFromSynthetic(Pipeline p) {
- if (isStreaming()) {
+ if (options.getUseParDoGenerator()) {
+ return sourceEventsFromSyntheticParDo(p);
+ }
+ return sourceEventsFromSyntheticSource(p);
+ }
+
+ /** Return a PCollection of synthetic events using the Source API. */
+ private PCollection<Event> sourceEventsFromSyntheticSource(Pipeline p) {
+ if (options.isStreaming()) {
NexmarkUtils.console("Generating %d events in streaming mode",
configuration.numEvents);
return p.apply(queryName + ".ReadUnbounded",
NexmarkUtils.streamEventsSource(configuration));
} else {
@@ -769,6 +775,19 @@ private String logsDir(long now) {
}
}
+ /** Return a PCollection of synthetic events using the ParDo API. */
+ private PCollection<Event> sourceEventsFromSyntheticParDo(Pipeline p) {
+ if (options.isStreaming()) {
+ throw new UnsupportedOperationException("Can't generate unbounded events
via ParDo.");
+ } else {
+ return p.apply(queryName + ".Create", Create.of((Void) null))
+ .apply(
+ queryName + ".Generate",
+ ParDo.of(new
EventGeneratorDoFn(NexmarkUtils.standardGeneratorConfig(configuration))))
+ .apply(queryName + ".Reshuffle", Reshuffle.viaRandomKey());
+ }
+ }
+
/** Return source of events from Pubsub. */
private PCollection<Event> sourceEventsFromPubsub(Pipeline p) {
NexmarkUtils.console("Reading events from Pubsub %s", pubsubSubscription);
diff --git
a/sdks/java/nexmark/src/main/java/org/apache/beam/sdk/nexmark/NexmarkOptions.java
b/sdks/java/nexmark/src/main/java/org/apache/beam/sdk/nexmark/NexmarkOptions.java
index c287bae4a25..e082b8385ec 100644
---
a/sdks/java/nexmark/src/main/java/org/apache/beam/sdk/nexmark/NexmarkOptions.java
+++
b/sdks/java/nexmark/src/main/java/org/apache/beam/sdk/nexmark/NexmarkOptions.java
@@ -452,4 +452,9 @@
int getMaxNumWorkers();
void setMaxNumWorkers(int value);
+
+ @Description("Use source API for synthetic generation.")
+ boolean getUseParDoGenerator();
+
+ void setUseParDoGenerator(boolean value);
}
diff --git
a/sdks/java/nexmark/src/main/java/org/apache/beam/sdk/nexmark/NexmarkUtils.java
b/sdks/java/nexmark/src/main/java/org/apache/beam/sdk/nexmark/NexmarkUtils.java
index 796e08a19c0..d49ceea550c 100644
---
a/sdks/java/nexmark/src/main/java/org/apache/beam/sdk/nexmark/NexmarkUtils.java
+++
b/sdks/java/nexmark/src/main/java/org/apache/beam/sdk/nexmark/NexmarkUtils.java
@@ -351,7 +351,7 @@ public static void setupPipeline(CoderStrategy
coderStrategy, Pipeline p) {
}
/** Return a generator config to match the given {@code options}. */
- private static GeneratorConfig standardGeneratorConfig(NexmarkConfiguration
configuration) {
+ static GeneratorConfig standardGeneratorConfig(NexmarkConfiguration
configuration) {
return new GeneratorConfig(
configuration,
configuration.useWallclockEventTime ? System.currentTimeMillis() :
BASE_TIME,
diff --git
a/sdks/java/nexmark/src/main/java/org/apache/beam/sdk/nexmark/sources/EventGeneratorDoFn.java
b/sdks/java/nexmark/src/main/java/org/apache/beam/sdk/nexmark/sources/EventGeneratorDoFn.java
new file mode 100644
index 00000000000..bc454e1f44e
--- /dev/null
+++
b/sdks/java/nexmark/src/main/java/org/apache/beam/sdk/nexmark/sources/EventGeneratorDoFn.java
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.nexmark.sources;
+
+import org.apache.beam.sdk.nexmark.model.Event;
+import org.apache.beam.sdk.nexmark.sources.generator.Generator;
+import org.apache.beam.sdk.nexmark.sources.generator.GeneratorConfig;
+import org.apache.beam.sdk.transforms.DoFn;
+import org.apache.beam.sdk.values.TimestampedValue;
+
+/** A DoFn to generate bounded event records. */
+public class EventGeneratorDoFn extends DoFn<Void, Event> {
+ /** Configuration we generate events against. */
+ private final GeneratorConfig config;
+ /** Generator we are reading from. */
+ private transient Generator generator;
+
+ public EventGeneratorDoFn(GeneratorConfig config) {
+ this.config = config;
+ }
+
+ @Setup
+ public void setup() {
+ generator = new Generator(config);
+ }
+
+ @ProcessElement
+ public void processElement(ProcessContext c) {
+ while (generator.hasNext()) {
+ final TimestampedValue<Event> next = generator.next();
+ c.outputWithTimestamp(next.getValue(), next.getTimestamp());
+ }
+ }
+}
diff --git
a/sdks/java/nexmark/src/test/java/org/apache/beam/sdk/nexmark/sources/EventGeneratorDoFnTest.java
b/sdks/java/nexmark/src/test/java/org/apache/beam/sdk/nexmark/sources/EventGeneratorDoFnTest.java
new file mode 100644
index 00000000000..32c4402c980
--- /dev/null
+++
b/sdks/java/nexmark/src/test/java/org/apache/beam/sdk/nexmark/sources/EventGeneratorDoFnTest.java
@@ -0,0 +1,87 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.nexmark.sources;
+
+import java.util.ArrayList;
+import java.util.List;
+import org.apache.beam.sdk.io.Read;
+import org.apache.beam.sdk.nexmark.NexmarkConfiguration;
+import org.apache.beam.sdk.nexmark.model.Event;
+import org.apache.beam.sdk.nexmark.sources.generator.Generator;
+import org.apache.beam.sdk.nexmark.sources.generator.GeneratorConfig;
+import org.apache.beam.sdk.testing.PAssert;
+import org.apache.beam.sdk.testing.TestPipeline;
+import org.apache.beam.sdk.transforms.Count;
+import org.apache.beam.sdk.transforms.Create;
+import org.apache.beam.sdk.transforms.ParDo;
+import org.apache.beam.sdk.values.PCollection;
+import org.apache.beam.sdk.values.TimestampedValue;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+/** Test {@link BoundedEventSource}. */
+@RunWith(JUnit4.class)
+public class EventGeneratorDoFnTest {
+ @Rule public final transient TestPipeline p = TestPipeline.create();
+ @Rule public final transient TestPipeline sourcePipeline =
TestPipeline.create();
+
+ @Test
+ public void testNumGeneratedEvents() {
+ final int numEvents = 100;
+ GeneratorConfig config =
+ new GeneratorConfig(
+ NexmarkConfiguration.DEFAULT, System.currentTimeMillis(), 0,
numEvents, 0);
+ PCollection<Event> events =
+ p.apply(Create.of((Void) null)).apply(ParDo.of(new
EventGeneratorDoFn(config)));
+ PAssert.thatSingleton(events.apply("Count All",
Count.globally())).isEqualTo((long) numEvents);
+ p.run().waitUntilFinish();
+ }
+
+ @Test
+ public void testGeneratesSameEvents() {
+ final int numEvents = 100;
+ GeneratorConfig config =
+ new GeneratorConfig(
+ NexmarkConfiguration.DEFAULT, System.currentTimeMillis(), 0,
numEvents, 0);
+ List<Event> eventsFromGenerator = generateEvents(config);
+
+ PCollection<Event> eventsFromParDo =
+ p.apply(Create.of((Void) null)).apply(ParDo.of(new
EventGeneratorDoFn(config)));
+ PAssert.that(eventsFromParDo).containsInAnyOrder(eventsFromGenerator);
+ p.run().waitUntilFinish();
+
+ PCollection<Event> eventsFromSource =
+ sourcePipeline.apply(Read.from(new BoundedEventSource(config, 1)));
+ PAssert.that(eventsFromSource).containsInAnyOrder(eventsFromGenerator);
+ sourcePipeline.run().waitUntilFinish();
+
+ p.run().waitUntilFinish();
+ }
+
+ private static List<Event> generateEvents(GeneratorConfig config) {
+ Generator generator = new Generator(config);
+ List<Event> events = new ArrayList<>();
+ while (generator.hasNext()) {
+ final TimestampedValue<Event> timestampedEvent = generator.next();
+ events.add(timestampedEvent.getValue());
+ }
+ return events;
+ }
+}
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
Issue Time Tracking
-------------------
Worklog Id: (was: 142485)
Time Spent: 1h 40m (was: 1.5h)
> Add Nexmark events generation based on ParDo
> --------------------------------------------
>
> Key: BEAM-4376
> URL: https://issues.apache.org/jira/browse/BEAM-4376
> Project: Beam
> Issue Type: Improvement
> Components: examples-nexmark
> Reporter: Ismaël Mejía
> Assignee: Ismaël Mejía
> Priority: Minor
> Time Spent: 1h 40m
> Remaining Estimate: 0h
>
> Currently Nexmark generates events using the Source API. Having a ParDo based
> generator will allow us to use Nexmark quickly with the Portable runners.
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
