subject:"\[jira\] \[Work logged\] \(BEAM\-1240\) Create RabbitMqIO"

[jira] [Work logged] (BEAM-1240) Create RabbitMqIO

2018-10-11 Thread ASF GitHub Bot (JIRA)



 [ 
https://issues.apache.org/jira/browse/BEAM-1240?focusedWorklogId=153433&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-153433
 ]

ASF GitHub Bot logged work on BEAM-1240:


Author: ASF GitHub Bot
Created on: 11/Oct/18 08:28
Start Date: 11/Oct/18 08:28
Worklog Time Spent: 10m 
  Work Description: echauchot commented on a change in pull request #1729: 
[BEAM-1240] Create RabbitMqIO
URL: https://github.com/apache/beam/pull/1729#discussion_r224358502
 
 

 ##
 File path: 
sdks/java/io/rabbitmq/src/test/java/org/apache/beam/sdk/io/rabbitmq/RabbitMqIOTest.java
 ##
 @@ -0,0 +1,269 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.io.rabbitmq;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+import com.rabbitmq.client.AMQP;
+import com.rabbitmq.client.Channel;
+import com.rabbitmq.client.Connection;
+import com.rabbitmq.client.ConnectionFactory;
+import com.rabbitmq.client.Consumer;
+import com.rabbitmq.client.DefaultConsumer;
+import com.rabbitmq.client.Envelope;
+import java.io.IOException;
+import java.io.Serializable;
+import java.net.ServerSocket;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.stream.Collectors;
+import java.util.stream.IntStream;
+import org.apache.beam.sdk.testing.PAssert;
+import org.apache.beam.sdk.testing.TestPipeline;
+import org.apache.beam.sdk.transforms.Create;
+import org.apache.beam.sdk.transforms.DoFn;
+import org.apache.beam.sdk.transforms.ParDo;
+import org.apache.beam.sdk.values.PCollection;
+import org.apache.qpid.server.Broker;
+import org.apache.qpid.server.BrokerOptions;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.ClassRule;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/** Test of {@link RabbitMqIO}. */
+@RunWith(JUnit4.class)
+public class RabbitMqIOTest implements Serializable {
+  private static final Logger LOG = 
LoggerFactory.getLogger(RabbitMqIOTest.class);
+
+  private static int port;
+  @ClassRule public static TemporaryFolder temporaryFolder = new 
TemporaryFolder();
+
+  @Rule public transient TestPipeline p = TestPipeline.create();
+
+  private static transient Broker broker;
+
+  @BeforeClass
+  public static void startBroker() throws Exception {
+try (ServerSocket serverSocket = new ServerSocket(0)) {
+  port = serverSocket.getLocalPort();
+}
+
+System.setProperty("derby.stream.error.field", "MyApp.DEV_NULL");
+broker = new Broker();
+BrokerOptions options = new BrokerOptions();
+options.setConfigProperty(BrokerOptions.QPID_AMQP_PORT, 
String.valueOf(port));
+options.setConfigProperty(BrokerOptions.QPID_WORK_DIR, 
temporaryFolder.newFolder().toString());
+options.setConfigProperty(BrokerOptions.QPID_HOME_DIR, "src/test/qpid");
+broker.startup(options);
+  }
+
+  @AfterClass
+  public static void stopBroker() {
+broker.shutdown();
+  }
+
+  @Test
+  public void testReadQueue() throws Exception {
+final int maxNumRecords = 10;
+PCollection raw =
+p.apply(
+RabbitMqIO.read()
+.withUri("amqp://guest:guest@localhost:" + port)
+.withQueue("READ")
+.withMaxNumRecords(maxNumRecords));
+PCollection output = raw.apply(ParDo.of(new ConverterFn()));
+
+List records = generateRecords(maxNumRecords);
+PAssert.that(output).containsInAnyOrder(records);
+
+ConnectionFactory connectionFactory = new ConnectionFactory();
+connectionFactory.setUri("amqp://guest:guest@localhost:" + port);
+Connection connection = connectionFactory.newConnection();
+Channel channel = connection.createChannel();
+channel.queueDeclare("READ", false, false, false, null);
+for (byte[] record : records) {
+  channel.basicPublish("", "READ", null, record);
+}
+
+p.run();
+
+channel.close(

[jira] [Work logged] (BEAM-1240) Create RabbitMqIO

2018-10-11 Thread ASF GitHub Bot (JIRA)



 [ 
https://issues.apache.org/jira/browse/BEAM-1240?focusedWorklogId=153432&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-153432
 ]

ASF GitHub Bot logged work on BEAM-1240:


Author: ASF GitHub Bot
Created on: 11/Oct/18 08:22
Start Date: 11/Oct/18 08:22
Worklog Time Spent: 10m 
  Work Description: echauchot commented on a change in pull request #1729: 
[BEAM-1240] Create RabbitMqIO
URL: https://github.com/apache/beam/pull/1729#discussion_r224356560
 
 

 ##
 File path: 
sdks/java/io/rabbitmq/src/main/java/org/apache/beam/sdk/io/rabbitmq/RabbitMqMessage.java
 ##
 @@ -0,0 +1,297 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.io.rabbitmq;
+
+import com.rabbitmq.client.AMQP;
+import java.io.Serializable;
+import java.util.Arrays;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Objects;
+import javax.annotation.Nullable;
+
+/**
+ * It contains the message payload, and additional metadata like routing key 
or attributes. The main
+ * reason of this class is that AMQP.BasicProperties doesn't provide a 
serializable public API.
+ */
+public class RabbitMqMessage implements Serializable {
+
+  @Nullable private final String routingKey;
+  private final byte[] body;
+  private final String contentType;
+  private final String contentEncoding;
+  private final Map headers;
+  private final Integer deliveryMode;
+  private final Integer priority;
+  @Nullable private final String correlationId;
+  @Nullable private final String replyTo;
+  private final String expiration;
+  private final String messageId;
+  private final Date timestamp;
+  @Nullable private final String type;
+  @Nullable private final String userId;
+  @Nullable private final String appId;
+  @Nullable private final String clusterId;
+
+  public RabbitMqMessage(byte[] body) {
+this.body = body;
+routingKey = "";
+contentType = null;
+contentEncoding = null;
+headers = new HashMap<>();
+deliveryMode = 1;
+priority = 1;
+correlationId = null;
+replyTo = null;
+expiration = null;
+messageId = null;
+timestamp = new Date();
+type = null;
+userId = null;
+appId = null;
+clusterId = null;
+  }
+
+  public RabbitMqMessage(
+  String routingKey,
+  byte[] body,
+  String contentType,
+  String contentEncoding,
+  Map headers,
+  Integer deliveryMode,
+  Integer priority,
+  String correlationId,
+  String replyTo,
+  String expiration,
+  String messageId,
+  Date timestamp,
+  String type,
+  String userId,
+  String appId,
+  String clusterId) {
+this.routingKey = routingKey;
+this.body = body;
+this.contentType = contentType;
+this.contentEncoding = contentEncoding;
+this.headers = headers;
+this.deliveryMode = deliveryMode;
+this.priority = priority;
+this.correlationId = correlationId;
+this.replyTo = replyTo;
+this.expiration = expiration;
+this.messageId = messageId;
+this.timestamp = timestamp;
+this.type = type;
+this.userId = userId;
+this.appId = appId;
+this.clusterId = clusterId;
+  }
+
+  public String getRoutingKey() {
+return routingKey;
+  }
+
+  public byte[] getBody() {
+return body;
+  }
+
+  public String getContentType() {
+return contentType;
+  }
+
+  public String getContentEncoding() {
+return contentEncoding;
+  }
+
+  public Map getHeaders() {
+return headers;
+  }
+
+  public Integer getDeliveryMode() {
+return deliveryMode;
+  }
+
+  public Integer getPriority() {
+return priority;
+  }
+
+  public String getCorrelationId() {
+return correlationId;
+  }
+
+  public String getReplyTo() {
+return replyTo;
+  }
+
+  public String getExpiration() {
+return expiration;
+  }
+
+  public String getMessageId() {
+return messageId;
+  }
+
+  public Date getTimestamp() {
+return timestamp;
+  }
+
+  public String getType() {
+return type;
+  }
+
+  public String getUserId() {
+return userId;
+  }
+
+  public String ge

[jira] [Work logged] (BEAM-1240) Create RabbitMqIO

2018-10-11 Thread ASF GitHub Bot (JIRA)



 [ 
https://issues.apache.org/jira/browse/BEAM-1240?focusedWorklogId=153429&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-153429
 ]

ASF GitHub Bot logged work on BEAM-1240:


Author: ASF GitHub Bot
Created on: 11/Oct/18 08:14
Start Date: 11/Oct/18 08:14
Worklog Time Spent: 10m 
  Work Description: echauchot commented on a change in pull request #1729: 
[BEAM-1240] Create RabbitMqIO
URL: https://github.com/apache/beam/pull/1729#discussion_r224353317
 
 

 ##
 File path: 
sdks/java/io/rabbitmq/src/main/java/org/apache/beam/sdk/io/rabbitmq/RabbitMqIO.java
 ##
 @@ -0,0 +1,655 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.io.rabbitmq;
+
+import static com.google.common.base.Preconditions.checkArgument;
+
+import com.google.auto.value.AutoValue;
+import com.rabbitmq.client.Channel;
+import com.rabbitmq.client.Connection;
+import com.rabbitmq.client.ConnectionFactory;
+import com.rabbitmq.client.MessageProperties;
+import com.rabbitmq.client.QueueingConsumer;
+import java.io.IOException;
+import java.io.Serializable;
+import java.net.URISyntaxException;
+import java.nio.charset.StandardCharsets;
+import java.security.KeyManagementException;
+import java.security.NoSuchAlgorithmException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.NoSuchElementException;
+import java.util.concurrent.TimeoutException;
+import javax.annotation.Nullable;
+import org.apache.beam.sdk.annotations.Experimental;
+import org.apache.beam.sdk.coders.Coder;
+import org.apache.beam.sdk.coders.SerializableCoder;
+import org.apache.beam.sdk.io.UnboundedSource;
+import org.apache.beam.sdk.options.PipelineOptions;
+import org.apache.beam.sdk.transforms.DoFn;
+import org.apache.beam.sdk.transforms.PTransform;
+import org.apache.beam.sdk.transforms.ParDo;
+import org.apache.beam.sdk.values.PBegin;
+import org.apache.beam.sdk.values.PCollection;
+import org.joda.time.Duration;
+import org.joda.time.Instant;
+
+/**
+ * A IO to publish or consume messages with a RabbitMQ broker.
+ *
+ * Consuming messages from RabbitMQ server
+ *
+ * {@link RabbitMqIO} {@link Read} returns an unbounded {@link PCollection} 
containing RabbitMQ
+ * messages body (as {@code byte[]}) wrapped as {@link RabbitMqMessage}.
+ *
+ * To configure a RabbitMQ source, you have to provide a RabbitMQ {@code 
URI} to connect to a
+ * RabbitMQ broker. The following example illustrates various options for 
configuring the source,
+ * reading from the queue:
+ *
+ * {@code
+ * pipeline.apply(
+ *   
RabbitMqIO.read().withUri("amqp://user:password@localhost:5672").withQueue("QUEUE")
+ *
+ * }
+ *
+ * It's also possible to read from an exchange (providing the exchange type 
and routing key)
+ * instead of directly from a queue:
+ *
+ * {@code
+ * pipeline.apply(
+ *   
RabbitMqIO.read().withUri("amqp://user:password@localhost:5672").withExchange("EXCHANGE",
 "fanout", "QUEUE");
+ * }
+ *
+ * Publishing messages to RabbitMQ server
+ *
+ * {@link RabbitMqIO} {@link Write} can send {@code byte[]} to a RabbitMQ 
server queue or
+ * exchange.
+ *
+ * As for the {@link Read}, the {@link Write} is configured with a RabbitMQ 
URI.
+ *
+ * For instance, you can write to an exchange (providing the exchange type):
+ *
+ * {@code
+ * pipeline
+ *   .apply(...) // provide PCollection
+ *   
.apply(RabbitMqIO.write().withUri("amqp://user:password@localhost:5672").withExchange("EXCHANGE",
 "fanout"));
+ * }
+ *
+ * For instance, you can write to a queue:
+ *
+ * {@code
+ * pipeline
+ *   .apply(...) // provide PCollection
+ *   
.apply(RabbitMqIO.write().withUri("amqp://user:password@localhost:5672").withQueue("QUEUE"));
+ *
+ * }
+ */
+@Experimental(Experimental.Kind.SOURCE_SINK)
+public class RabbitMqIO {
+  public static Read read() {
+return new AutoValue_RabbitMqIO_Read.Builder()
+.setQueueDeclare(false)
+.setMaxReadTime(null)
+.setMaxNumRecords(Long.MAX_VALUE)
+.setUseCorrelationId(false)
+.build();
+  }
+
+  public static Write write() {
+return new AutoValue_RabbitMqIO_Write.Builder()
+

[jira] [Work logged] (BEAM-1240) Create RabbitMqIO

2018-10-11 Thread ASF GitHub Bot (JIRA)



 [ 
https://issues.apache.org/jira/browse/BEAM-1240?focusedWorklogId=153431&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-153431
 ]

ASF GitHub Bot logged work on BEAM-1240:


Author: ASF GitHub Bot
Created on: 11/Oct/18 08:14
Start Date: 11/Oct/18 08:14
Worklog Time Spent: 10m 
  Work Description: echauchot commented on a change in pull request #1729: 
[BEAM-1240] Create RabbitMqIO
URL: https://github.com/apache/beam/pull/1729#discussion_r224354306
 
 

 ##
 File path: 
sdks/java/io/rabbitmq/src/main/java/org/apache/beam/sdk/io/rabbitmq/RabbitMqIO.java
 ##
 @@ -0,0 +1,530 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.io.rabbitmq;
+
+import static com.google.common.base.Preconditions.checkArgument;
+
+import com.google.auto.value.AutoValue;
+import com.rabbitmq.client.Channel;
+import com.rabbitmq.client.Connection;
+import com.rabbitmq.client.ConnectionFactory;
+import com.rabbitmq.client.QueueingConsumer;
+import java.io.IOException;
+import java.io.Serializable;
+import java.net.URISyntaxException;
+import java.nio.charset.StandardCharsets;
+import java.security.KeyManagementException;
+import java.security.NoSuchAlgorithmException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.NoSuchElementException;
+import javax.annotation.Nullable;
+import org.apache.beam.sdk.annotations.Experimental;
+import org.apache.beam.sdk.coders.Coder;
+import org.apache.beam.sdk.coders.SerializableCoder;
+import org.apache.beam.sdk.io.UnboundedSource;
+import org.apache.beam.sdk.options.PipelineOptions;
+import org.apache.beam.sdk.transforms.DoFn;
+import org.apache.beam.sdk.transforms.PTransform;
+import org.apache.beam.sdk.transforms.ParDo;
+import org.apache.beam.sdk.values.PBegin;
+import org.apache.beam.sdk.values.PCollection;
+import org.apache.beam.sdk.values.PDone;
+import org.joda.time.Duration;
+import org.joda.time.Instant;
+
+/**
+ * A IO to publish or consume messages with a RabbitMQ broker.
+ *
+ * Consuming messages from RabbitMQ server
+ *
+ * {@link RabbitMqIO} {@link Read} returns an unbounded {@link PCollection} 
containing RabbitMQ
+ * messages body (as {@code byte[]}).
+ *
+ * To configure a RabbitMQ source, you have to provide a RabbitMQ {@code 
URI} to connect
+ * to a RabbitMQ broker. The following example illustrates various options for 
configuring the
+ * source:
+ *
+ * {@code
+ *
+ *  pipeline.apply(
+ *
RabbitMqIO.read().withUri("amqp://user:password@localhost:5672").withQueue("QUEUE")
+ *
+ * }
+ *
+ * Publishing messages to RabbitMQ server
+ *
+ * {@link RabbitMqIO} {@link Write} can send {@code byte[]} to a RabbitMQ 
server queue.
+ *
+ * As for the {@link Read}, the {@link Write} is configured with a RabbitMQ 
URI.
+ *
+ * For instance:
+ *
+ * {@code
+ *
+ *  pipeline
+ *.apply(...) // provide PCollection
+ *
.apply(RabbitMqIO.write().withUri("amqp://user:password@localhost:5672").withQueue("QUEUE"));
+ *
+ * }
+ */
+@Experimental(Experimental.Kind.SOURCE_SINK)
+public class RabbitMqIO {
+  public static Read read() {
+return new AutoValue_RabbitMqIO_Read.Builder().setQueueDeclare(false)
+
.setMaxReadTime(null).setMaxNumRecords(Long.MAX_VALUE).setUseCorrelationId(false).build();
+  }
+
+  public static Write write() {
+return new 
AutoValue_RabbitMqIO_Write.Builder().setExchangeDeclare(false).build();
+  }
+
+  private RabbitMqIO() {
+  }
+
+  private static ConnectionFactory createConnectionFactory(String uri) throws 
URISyntaxException,
+  NoSuchAlgorithmException, KeyManagementException {
+ConnectionFactory connectionFactory = new ConnectionFactory();
+connectionFactory.setUri(uri);
+connectionFactory.setAutomaticRecoveryEnabled(true);
+connectionFactory.setConnectionTimeout(6);
+connectionFactory.setNetworkRecoveryInterval(5000);
+connectionFactory.setRequestedHeartbeat(60);
+connectionFactory.setTopologyRecoveryEnabled(true);
+connectionFactory.setRequestedChannelMax(0);
+connectionFactory.setRequestedFrameMax(0);
+return connectionFactory;
+  }
+
+

[jira] [Work logged] (BEAM-1240) Create RabbitMqIO

2018-10-11 Thread ASF GitHub Bot (JIRA)



 [ 
https://issues.apache.org/jira/browse/BEAM-1240?focusedWorklogId=153430&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-153430
 ]

ASF GitHub Bot logged work on BEAM-1240:


Author: ASF GitHub Bot
Created on: 11/Oct/18 08:14
Start Date: 11/Oct/18 08:14
Worklog Time Spent: 10m 
  Work Description: echauchot commented on a change in pull request #1729: 
[BEAM-1240] Create RabbitMqIO
URL: https://github.com/apache/beam/pull/1729#discussion_r224352528
 
 

 ##
 File path: 
sdks/java/io/rabbitmq/src/main/java/org/apache/beam/sdk/io/rabbitmq/RabbitMqIO.java
 ##
 @@ -0,0 +1,530 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.io.rabbitmq;
+
+import static com.google.common.base.Preconditions.checkArgument;
+
+import com.google.auto.value.AutoValue;
+import com.rabbitmq.client.Channel;
+import com.rabbitmq.client.Connection;
+import com.rabbitmq.client.ConnectionFactory;
+import com.rabbitmq.client.QueueingConsumer;
+import java.io.IOException;
+import java.io.Serializable;
+import java.net.URISyntaxException;
+import java.nio.charset.StandardCharsets;
+import java.security.KeyManagementException;
+import java.security.NoSuchAlgorithmException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.NoSuchElementException;
+import javax.annotation.Nullable;
+import org.apache.beam.sdk.annotations.Experimental;
+import org.apache.beam.sdk.coders.Coder;
+import org.apache.beam.sdk.coders.SerializableCoder;
+import org.apache.beam.sdk.io.UnboundedSource;
+import org.apache.beam.sdk.options.PipelineOptions;
+import org.apache.beam.sdk.transforms.DoFn;
+import org.apache.beam.sdk.transforms.PTransform;
+import org.apache.beam.sdk.transforms.ParDo;
+import org.apache.beam.sdk.values.PBegin;
+import org.apache.beam.sdk.values.PCollection;
+import org.apache.beam.sdk.values.PDone;
+import org.joda.time.Duration;
+import org.joda.time.Instant;
+
+/**
+ * A IO to publish or consume messages with a RabbitMQ broker.
+ *
+ * Consuming messages from RabbitMQ server
+ *
+ * {@link RabbitMqIO} {@link Read} returns an unbounded {@link PCollection} 
containing RabbitMQ
+ * messages body (as {@code byte[]}).
+ *
+ * To configure a RabbitMQ source, you have to provide a RabbitMQ {@code 
URI} to connect
+ * to a RabbitMQ broker. The following example illustrates various options for 
configuring the
+ * source:
+ *
+ * {@code
+ *
+ *  pipeline.apply(
+ *
RabbitMqIO.read().withUri("amqp://user:password@localhost:5672").withQueue("QUEUE")
+ *
+ * }
+ *
+ * Publishing messages to RabbitMQ server
+ *
+ * {@link RabbitMqIO} {@link Write} can send {@code byte[]} to a RabbitMQ 
server queue.
+ *
+ * As for the {@link Read}, the {@link Write} is configured with a RabbitMQ 
URI.
+ *
+ * For instance:
+ *
+ * {@code
+ *
+ *  pipeline
+ *.apply(...) // provide PCollection
+ *
.apply(RabbitMqIO.write().withUri("amqp://user:password@localhost:5672").withQueue("QUEUE"));
+ *
+ * }
+ */
+@Experimental(Experimental.Kind.SOURCE_SINK)
+public class RabbitMqIO {
+  public static Read read() {
+return new AutoValue_RabbitMqIO_Read.Builder().setQueueDeclare(false)
+
.setMaxReadTime(null).setMaxNumRecords(Long.MAX_VALUE).setUseCorrelationId(false).build();
+  }
+
+  public static Write write() {
+return new 
AutoValue_RabbitMqIO_Write.Builder().setExchangeDeclare(false).build();
+  }
+
+  private RabbitMqIO() {
+  }
+
+  private static ConnectionFactory createConnectionFactory(String uri) throws 
URISyntaxException,
+  NoSuchAlgorithmException, KeyManagementException {
+ConnectionFactory connectionFactory = new ConnectionFactory();
+connectionFactory.setUri(uri);
+connectionFactory.setAutomaticRecoveryEnabled(true);
+connectionFactory.setConnectionTimeout(6);
+connectionFactory.setNetworkRecoveryInterval(5000);
+connectionFactory.setRequestedHeartbeat(60);
+connectionFactory.setTopologyRecoveryEnabled(true);
+connectionFactory.setRequestedChannelMax(0);
+connectionFactory.setRequestedFrameMax(0);
+return connectionFactory;
+  }
+
+

[jira] [Work logged] (BEAM-1240) Create RabbitMqIO

2018-09-25 Thread ASF GitHub Bot (JIRA)



 [ 
https://issues.apache.org/jira/browse/BEAM-1240?focusedWorklogId=147626&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-147626
 ]

ASF GitHub Bot logged work on BEAM-1240:


Author: ASF GitHub Bot
Created on: 25/Sep/18 15:50
Start Date: 25/Sep/18 15:50
Worklog Time Spent: 10m 
  Work Description: jbonofre commented on issue #1729: [BEAM-1240] Create 
RabbitMqIO
URL: https://github.com/apache/beam/pull/1729#issuecomment-424398592
 
 
   @akankshajain18 that's probably related to the watermark behavior which is 
not correct for now. I'm fixing that and other comments. I'm moving forward on 
this PR. Sorry for the delay.


This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
 
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org


Issue Time Tracking
---

Worklog Id: (was: 147626)
Time Spent: 12h  (was: 11h 50m)

> Create RabbitMqIO
> -
>
> Key: BEAM-1240
> URL: https://issues.apache.org/jira/browse/BEAM-1240
> Project: Beam
>  Issue Type: New Feature
>  Components: io-ideas
>Reporter: Jean-Baptiste Onofré
>Assignee: Jean-Baptiste Onofré
>Priority: Major
> Fix For: 2.8.0
>
>  Time Spent: 12h
>  Remaining Estimate: 0h
>




--
This message was sent by Atlassian JIRA
(v7.6.3#76005)

[jira] [Work logged] (BEAM-1240) Create RabbitMqIO

2018-08-03 Thread ASF GitHub Bot (JIRA)



 [ 
https://issues.apache.org/jira/browse/BEAM-1240?focusedWorklogId=130861&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-130861
 ]

ASF GitHub Bot logged work on BEAM-1240:


Author: ASF GitHub Bot
Created on: 03/Aug/18 13:07
Start Date: 03/Aug/18 13:07
Worklog Time Spent: 10m 
  Work Description: akankshajain18 commented on issue #1729: [BEAM-1240] 
Create RabbitMqIO
URL: https://github.com/apache/beam/pull/1729#issuecomment-410247802
 
 
   
   Hello Folks
   
   While working with RmqIO and Flink local runner of the beam, I observed, 
Even if Rmq has data to consume, Beam pipeline Job execution switched to status 
FINISHED without any exception.
   In logs,  RabbitMqIO$UnboundedRabbitMqReader close() method get called, 
which led to stopping the connection between RMQ and my beam job(running on 
Flink runner).
   
   Code : 
   [@Override
   public void close() throws IOException {
   if (connectionHandler != null) {
   connectionHandler.stop();
   }
   }]
   
   It will be great if someone can help me out about this random behavior.
   


This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
 
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org


Issue Time Tracking
---

Worklog Id: (was: 130861)
Time Spent: 11h 50m  (was: 11h 40m)

> Create RabbitMqIO
> -
>
> Key: BEAM-1240
> URL: https://issues.apache.org/jira/browse/BEAM-1240
> Project: Beam
>  Issue Type: New Feature
>  Components: io-ideas
>Reporter: Jean-Baptiste Onofré
>Assignee: Jean-Baptiste Onofré
>Priority: Major
> Fix For: 2.6.0
>
>  Time Spent: 11h 50m
>  Remaining Estimate: 0h
>




--
This message was sent by Atlassian JIRA
(v7.6.3#76005)

[jira] [Work logged] (BEAM-1240) Create RabbitMqIO

2018-07-22 Thread ASF GitHub Bot (JIRA)



 [ 
https://issues.apache.org/jira/browse/BEAM-1240?focusedWorklogId=125894&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-125894
 ]

ASF GitHub Bot logged work on BEAM-1240:


Author: ASF GitHub Bot
Created on: 23/Jul/18 06:49
Start Date: 23/Jul/18 06:49
Worklog Time Spent: 10m 
  Work Description: akankshajain18 commented on issue #1729: [BEAM-1240] 
Create RabbitMqIO
URL: https://github.com/apache/beam/pull/1729#issuecomment-406957175
 
 
   Thanks @jbonofre 


This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
 
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org


Issue Time Tracking
---

Worklog Id: (was: 125894)
Time Spent: 11h 40m  (was: 11.5h)

> Create RabbitMqIO
> -
>
> Key: BEAM-1240
> URL: https://issues.apache.org/jira/browse/BEAM-1240
> Project: Beam
>  Issue Type: New Feature
>  Components: io-ideas
>Reporter: Jean-Baptiste Onofré
>Assignee: Jean-Baptiste Onofré
>Priority: Major
> Fix For: 2.6.0
>
>  Time Spent: 11h 40m
>  Remaining Estimate: 0h
>




--
This message was sent by Atlassian JIRA
(v7.6.3#76005)

[jira] [Work logged] (BEAM-1240) Create RabbitMqIO

2018-07-21 Thread ASF GitHub Bot (JIRA)



 [ 
https://issues.apache.org/jira/browse/BEAM-1240?focusedWorklogId=125743&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-125743
 ]

ASF GitHub Bot logged work on BEAM-1240:


Author: ASF GitHub Bot
Created on: 21/Jul/18 18:53
Start Date: 21/Jul/18 18:53
Worklog Time Spent: 10m 
  Work Description: jbonofre commented on issue #1729: [BEAM-1240] Create 
RabbitMqIO
URL: https://github.com/apache/beam/pull/1729#issuecomment-406816396
 
 
   retest this please


This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
 
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org


Issue Time Tracking
---

Worklog Id: (was: 125743)
Time Spent: 11.5h  (was: 11h 20m)

> Create RabbitMqIO
> -
>
> Key: BEAM-1240
> URL: https://issues.apache.org/jira/browse/BEAM-1240
> Project: Beam
>  Issue Type: New Feature
>  Components: io-ideas
>Reporter: Jean-Baptiste Onofré
>Assignee: Jean-Baptiste Onofré
>Priority: Major
> Fix For: 2.6.0
>
>  Time Spent: 11.5h
>  Remaining Estimate: 0h
>




--
This message was sent by Atlassian JIRA
(v7.6.3#76005)

[jira] [Work logged] (BEAM-1240) Create RabbitMqIO

2018-07-21 Thread ASF GitHub Bot (JIRA)



 [ 
https://issues.apache.org/jira/browse/BEAM-1240?focusedWorklogId=125723&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-125723
 ]

ASF GitHub Bot logged work on BEAM-1240:


Author: ASF GitHub Bot
Created on: 21/Jul/18 15:41
Start Date: 21/Jul/18 15:41
Worklog Time Spent: 10m 
  Work Description: jbonofre edited a comment on issue #1729: [BEAM-1240] 
Create RabbitMqIO
URL: https://github.com/apache/beam/pull/1729#issuecomment-406804793
 
 
   I exposed `durable` to end users for queue declaration (as requested by 
@akankshajain18).


This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
 
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org


Issue Time Tracking
---

Worklog Id: (was: 125723)
Time Spent: 11h 20m  (was: 11h 10m)

> Create RabbitMqIO
> -
>
> Key: BEAM-1240
> URL: https://issues.apache.org/jira/browse/BEAM-1240
> Project: Beam
>  Issue Type: New Feature
>  Components: io-ideas
>Reporter: Jean-Baptiste Onofré
>Assignee: Jean-Baptiste Onofré
>Priority: Major
> Fix For: 2.6.0
>
>  Time Spent: 11h 20m
>  Remaining Estimate: 0h
>




--
This message was sent by Atlassian JIRA
(v7.6.3#76005)

[jira] [Work logged] (BEAM-1240) Create RabbitMqIO

2018-07-21 Thread ASF GitHub Bot (JIRA)



 [ 
https://issues.apache.org/jira/browse/BEAM-1240?focusedWorklogId=125722&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-125722
 ]

ASF GitHub Bot logged work on BEAM-1240:


Author: ASF GitHub Bot
Created on: 21/Jul/18 15:40
Start Date: 21/Jul/18 15:40
Worklog Time Spent: 10m 
  Work Description: jbonofre commented on issue #1729: [BEAM-1240] Create 
RabbitMqIO
URL: https://github.com/apache/beam/pull/1729#issuecomment-406804793
 
 
   I exposed `durable` to end users for queue declaration (as requested by 
@akankshajain18 


This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
 
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org


Issue Time Tracking
---

Worklog Id: (was: 125722)
Time Spent: 11h 10m  (was: 11h)

> Create RabbitMqIO
> -
>
> Key: BEAM-1240
> URL: https://issues.apache.org/jira/browse/BEAM-1240
> Project: Beam
>  Issue Type: New Feature
>  Components: io-ideas
>Reporter: Jean-Baptiste Onofré
>Assignee: Jean-Baptiste Onofré
>Priority: Major
> Fix For: 2.6.0
>
>  Time Spent: 11h 10m
>  Remaining Estimate: 0h
>




--
This message was sent by Atlassian JIRA
(v7.6.3#76005)

[jira] [Work logged] (BEAM-1240) Create RabbitMqIO

2018-07-20 Thread ASF GitHub Bot (JIRA)



 [ 
https://issues.apache.org/jira/browse/BEAM-1240?focusedWorklogId=125569&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-125569
 ]

ASF GitHub Bot logged work on BEAM-1240:


Author: ASF GitHub Bot
Created on: 20/Jul/18 16:43
Start Date: 20/Jul/18 16:43
Worklog Time Spent: 10m 
  Work Description: jbonofre commented on a change in pull request #1729: 
[BEAM-1240] Create RabbitMqIO
URL: https://github.com/apache/beam/pull/1729#discussion_r204103403
 
 

 ##
 File path: 
sdks/java/io/rabbitmq/src/test/java/org/apache/beam/sdk/io/rabbitmq/RabbitMqIOTest.java
 ##
 @@ -0,0 +1,269 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.io.rabbitmq;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+import com.rabbitmq.client.AMQP;
+import com.rabbitmq.client.Channel;
+import com.rabbitmq.client.Connection;
+import com.rabbitmq.client.ConnectionFactory;
+import com.rabbitmq.client.Consumer;
+import com.rabbitmq.client.DefaultConsumer;
+import com.rabbitmq.client.Envelope;
+import java.io.IOException;
+import java.io.Serializable;
+import java.net.ServerSocket;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.stream.Collectors;
+import java.util.stream.IntStream;
+import org.apache.beam.sdk.testing.PAssert;
+import org.apache.beam.sdk.testing.TestPipeline;
+import org.apache.beam.sdk.transforms.Create;
+import org.apache.beam.sdk.transforms.DoFn;
+import org.apache.beam.sdk.transforms.ParDo;
+import org.apache.beam.sdk.values.PCollection;
+import org.apache.qpid.server.Broker;
 
 Review comment:
   Unfortunately no as RabbitMQ is not Java based. So, not easy to have a 
portable solution for the test. That's why I use QPid in the test (same 
protocol supported).


This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
 
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org


Issue Time Tracking
---

Worklog Id: (was: 125569)
Time Spent: 11h  (was: 10h 50m)

> Create RabbitMqIO
> -
>
> Key: BEAM-1240
> URL: https://issues.apache.org/jira/browse/BEAM-1240
> Project: Beam
>  Issue Type: New Feature
>  Components: io-ideas
>Reporter: Jean-Baptiste Onofré
>Assignee: Jean-Baptiste Onofré
>Priority: Major
> Fix For: 2.6.0
>
>  Time Spent: 11h
>  Remaining Estimate: 0h
>




--
This message was sent by Atlassian JIRA
(v7.6.3#76005)

[jira] [Work logged] (BEAM-1240) Create RabbitMqIO

2018-07-20 Thread ASF GitHub Bot (JIRA)



 [ 
https://issues.apache.org/jira/browse/BEAM-1240?focusedWorklogId=125567&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-125567
 ]

ASF GitHub Bot logged work on BEAM-1240:


Author: ASF GitHub Bot
Created on: 20/Jul/18 16:42
Start Date: 20/Jul/18 16:42
Worklog Time Spent: 10m 
  Work Description: jbonofre commented on a change in pull request #1729: 
[BEAM-1240] Create RabbitMqIO
URL: https://github.com/apache/beam/pull/1729#discussion_r204103403
 
 

 ##
 File path: 
sdks/java/io/rabbitmq/src/test/java/org/apache/beam/sdk/io/rabbitmq/RabbitMqIOTest.java
 ##
 @@ -0,0 +1,269 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.io.rabbitmq;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+import com.rabbitmq.client.AMQP;
+import com.rabbitmq.client.Channel;
+import com.rabbitmq.client.Connection;
+import com.rabbitmq.client.ConnectionFactory;
+import com.rabbitmq.client.Consumer;
+import com.rabbitmq.client.DefaultConsumer;
+import com.rabbitmq.client.Envelope;
+import java.io.IOException;
+import java.io.Serializable;
+import java.net.ServerSocket;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.stream.Collectors;
+import java.util.stream.IntStream;
+import org.apache.beam.sdk.testing.PAssert;
+import org.apache.beam.sdk.testing.TestPipeline;
+import org.apache.beam.sdk.transforms.Create;
+import org.apache.beam.sdk.transforms.DoFn;
+import org.apache.beam.sdk.transforms.ParDo;
+import org.apache.beam.sdk.values.PCollection;
+import org.apache.qpid.server.Broker;
 
 Review comment:
   Unfortunately no as RabbitMQ is not Java based. So, not easy to have a 
portable solution for the test. That's why I use ActiveMQ in the test (same 
protocol supported).


This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
 
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org


Issue Time Tracking
---

Worklog Id: (was: 125567)
Time Spent: 10h 50m  (was: 10h 40m)

> Create RabbitMqIO
> -
>
> Key: BEAM-1240
> URL: https://issues.apache.org/jira/browse/BEAM-1240
> Project: Beam
>  Issue Type: New Feature
>  Components: io-ideas
>Reporter: Jean-Baptiste Onofré
>Assignee: Jean-Baptiste Onofré
>Priority: Major
> Fix For: 2.6.0
>
>  Time Spent: 10h 50m
>  Remaining Estimate: 0h
>




--
This message was sent by Atlassian JIRA
(v7.6.3#76005)

[jira] [Work logged] (BEAM-1240) Create RabbitMqIO

2018-07-20 Thread ASF GitHub Bot (JIRA)



 [ 
https://issues.apache.org/jira/browse/BEAM-1240?focusedWorklogId=125436&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-125436
 ]

ASF GitHub Bot logged work on BEAM-1240:


Author: ASF GitHub Bot
Created on: 20/Jul/18 10:24
Start Date: 20/Jul/18 10:24
Worklog Time Spent: 10m 
  Work Description: echauchot commented on a change in pull request #1729: 
[BEAM-1240] Create RabbitMqIO
URL: https://github.com/apache/beam/pull/1729#discussion_r203992442
 
 

 ##
 File path: 
sdks/java/io/rabbitmq/src/test/java/org/apache/beam/sdk/io/rabbitmq/RabbitMqIOTest.java
 ##
 @@ -0,0 +1,269 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.io.rabbitmq;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+import com.rabbitmq.client.AMQP;
+import com.rabbitmq.client.Channel;
+import com.rabbitmq.client.Connection;
+import com.rabbitmq.client.ConnectionFactory;
+import com.rabbitmq.client.Consumer;
+import com.rabbitmq.client.DefaultConsumer;
+import com.rabbitmq.client.Envelope;
+import java.io.IOException;
+import java.io.Serializable;
+import java.net.ServerSocket;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.stream.Collectors;
+import java.util.stream.IntStream;
+import org.apache.beam.sdk.testing.PAssert;
+import org.apache.beam.sdk.testing.TestPipeline;
+import org.apache.beam.sdk.transforms.Create;
+import org.apache.beam.sdk.transforms.DoFn;
+import org.apache.beam.sdk.transforms.ParDo;
+import org.apache.beam.sdk.values.PCollection;
+import org.apache.qpid.server.Broker;
+import org.apache.qpid.server.BrokerOptions;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.ClassRule;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/** Test of {@link RabbitMqIO}. */
+@RunWith(JUnit4.class)
+public class RabbitMqIOTest implements Serializable {
+  private static final Logger LOG = 
LoggerFactory.getLogger(RabbitMqIOTest.class);
+
+  private static int port;
+  @ClassRule public static TemporaryFolder temporaryFolder = new 
TemporaryFolder();
+
+  @Rule public transient TestPipeline p = TestPipeline.create();
+
+  private static transient Broker broker;
+
+  @BeforeClass
+  public static void startBroker() throws Exception {
+try (ServerSocket serverSocket = new ServerSocket(0)) {
+  port = serverSocket.getLocalPort();
+}
+
+System.setProperty("derby.stream.error.field", "MyApp.DEV_NULL");
+broker = new Broker();
+BrokerOptions options = new BrokerOptions();
+options.setConfigProperty(BrokerOptions.QPID_AMQP_PORT, 
String.valueOf(port));
+options.setConfigProperty(BrokerOptions.QPID_WORK_DIR, 
temporaryFolder.newFolder().toString());
+options.setConfigProperty(BrokerOptions.QPID_HOME_DIR, "src/test/qpid");
+broker.startup(options);
+  }
+
+  @AfterClass
+  public static void stopBroker() {
+broker.shutdown();
+  }
+
+  @Test
+  public void testReadQueue() throws Exception {
+final int maxNumRecords = 10;
+PCollection raw =
+p.apply(
+RabbitMqIO.read()
+.withUri("amqp://guest:guest@localhost:" + port)
+.withQueue("READ")
+.withMaxNumRecords(maxNumRecords));
+PCollection output = raw.apply(ParDo.of(new ConverterFn()));
+
+List records = generateRecords(maxNumRecords);
+PAssert.that(output).containsInAnyOrder(records);
+
+ConnectionFactory connectionFactory = new ConnectionFactory();
+connectionFactory.setUri("amqp://guest:guest@localhost:" + port);
+Connection connection = connectionFactory.newConnection();
+Channel channel = connection.createChannel();
+channel.queueDeclare("READ", false, false, false, null);
+for (byte[] record : records) {
+  channel.basicPublish("", "READ", null, record);
+}
+
+p.run();
+
+channel.close(

[jira] [Work logged] (BEAM-1240) Create RabbitMqIO

2018-07-20 Thread ASF GitHub Bot (JIRA)



 [ 
https://issues.apache.org/jira/browse/BEAM-1240?focusedWorklogId=125437&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-125437
 ]

ASF GitHub Bot logged work on BEAM-1240:


Author: ASF GitHub Bot
Created on: 20/Jul/18 10:24
Start Date: 20/Jul/18 10:24
Worklog Time Spent: 10m 
  Work Description: echauchot commented on a change in pull request #1729: 
[BEAM-1240] Create RabbitMqIO
URL: https://github.com/apache/beam/pull/1729#discussion_r204000498
 
 

 ##
 File path: 
sdks/java/io/rabbitmq/src/main/java/org/apache/beam/sdk/io/rabbitmq/RabbitMqIO.java
 ##
 @@ -0,0 +1,655 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.io.rabbitmq;
+
+import static com.google.common.base.Preconditions.checkArgument;
+
+import com.google.auto.value.AutoValue;
+import com.rabbitmq.client.Channel;
+import com.rabbitmq.client.Connection;
+import com.rabbitmq.client.ConnectionFactory;
+import com.rabbitmq.client.MessageProperties;
+import com.rabbitmq.client.QueueingConsumer;
+import java.io.IOException;
+import java.io.Serializable;
+import java.net.URISyntaxException;
+import java.nio.charset.StandardCharsets;
+import java.security.KeyManagementException;
+import java.security.NoSuchAlgorithmException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.NoSuchElementException;
+import java.util.concurrent.TimeoutException;
+import javax.annotation.Nullable;
+import org.apache.beam.sdk.annotations.Experimental;
+import org.apache.beam.sdk.coders.Coder;
+import org.apache.beam.sdk.coders.SerializableCoder;
+import org.apache.beam.sdk.io.UnboundedSource;
+import org.apache.beam.sdk.options.PipelineOptions;
+import org.apache.beam.sdk.transforms.DoFn;
+import org.apache.beam.sdk.transforms.PTransform;
+import org.apache.beam.sdk.transforms.ParDo;
+import org.apache.beam.sdk.values.PBegin;
+import org.apache.beam.sdk.values.PCollection;
+import org.joda.time.Duration;
+import org.joda.time.Instant;
+
+/**
+ * A IO to publish or consume messages with a RabbitMQ broker.
+ *
+ * Consuming messages from RabbitMQ server
+ *
+ * {@link RabbitMqIO} {@link Read} returns an unbounded {@link PCollection} 
containing RabbitMQ
+ * messages body (as {@code byte[]}) wrapped as {@link RabbitMqMessage}.
+ *
+ * To configure a RabbitMQ source, you have to provide a RabbitMQ {@code 
URI} to connect to a
+ * RabbitMQ broker. The following example illustrates various options for 
configuring the source,
+ * reading from the queue:
+ *
+ * {@code
+ * pipeline.apply(
+ *   
RabbitMqIO.read().withUri("amqp://user:password@localhost:5672").withQueue("QUEUE")
+ *
+ * }
+ *
+ * It's also possible to read from an exchange (providing the exchange type 
and routing key)
+ * instead of directly from a queue:
+ *
+ * {@code
+ * pipeline.apply(
+ *   
RabbitMqIO.read().withUri("amqp://user:password@localhost:5672").withExchange("EXCHANGE",
 "fanout", "QUEUE");
+ * }
+ *
+ * Publishing messages to RabbitMQ server
+ *
+ * {@link RabbitMqIO} {@link Write} can send {@code byte[]} to a RabbitMQ 
server queue or
+ * exchange.
+ *
+ * As for the {@link Read}, the {@link Write} is configured with a RabbitMQ 
URI.
+ *
+ * For instance, you can write to an exchange (providing the exchange type):
+ *
+ * {@code
+ * pipeline
+ *   .apply(...) // provide PCollection
+ *   
.apply(RabbitMqIO.write().withUri("amqp://user:password@localhost:5672").withExchange("EXCHANGE",
 "fanout"));
+ * }
+ *
+ * For instance, you can write to a queue:
+ *
+ * {@code
+ * pipeline
+ *   .apply(...) // provide PCollection
+ *   
.apply(RabbitMqIO.write().withUri("amqp://user:password@localhost:5672").withQueue("QUEUE"));
+ *
+ * }
+ */
+@Experimental(Experimental.Kind.SOURCE_SINK)
+public class RabbitMqIO {
+  public static Read read() {
+return new AutoValue_RabbitMqIO_Read.Builder()
+.setQueueDeclare(false)
+.setMaxReadTime(null)
+.setMaxNumRecords(Long.MAX_VALUE)
+.setUseCorrelationId(false)
+.build();
+  }
+
+  public static Write write() {
+return new AutoValue_RabbitMqIO_Write.Builder()
+

[jira] [Work logged] (BEAM-1240) Create RabbitMqIO

2018-07-20 Thread ASF GitHub Bot (JIRA)



 [ 
https://issues.apache.org/jira/browse/BEAM-1240?focusedWorklogId=125432&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-125432
 ]

ASF GitHub Bot logged work on BEAM-1240:


Author: ASF GitHub Bot
Created on: 20/Jul/18 10:23
Start Date: 20/Jul/18 10:23
Worklog Time Spent: 10m 
  Work Description: echauchot commented on a change in pull request #1729: 
[BEAM-1240] Create RabbitMqIO
URL: https://github.com/apache/beam/pull/1729#discussion_r203990418
 
 

 ##
 File path: 
sdks/java/io/rabbitmq/src/test/java/org/apache/beam/sdk/io/rabbitmq/RabbitMqIOTest.java
 ##
 @@ -0,0 +1,269 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.io.rabbitmq;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+import com.rabbitmq.client.AMQP;
+import com.rabbitmq.client.Channel;
+import com.rabbitmq.client.Connection;
+import com.rabbitmq.client.ConnectionFactory;
+import com.rabbitmq.client.Consumer;
+import com.rabbitmq.client.DefaultConsumer;
+import com.rabbitmq.client.Envelope;
+import java.io.IOException;
+import java.io.Serializable;
+import java.net.ServerSocket;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.stream.Collectors;
+import java.util.stream.IntStream;
+import org.apache.beam.sdk.testing.PAssert;
+import org.apache.beam.sdk.testing.TestPipeline;
+import org.apache.beam.sdk.transforms.Create;
+import org.apache.beam.sdk.transforms.DoFn;
+import org.apache.beam.sdk.transforms.ParDo;
+import org.apache.beam.sdk.values.PCollection;
+import org.apache.qpid.server.Broker;
+import org.apache.qpid.server.BrokerOptions;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.ClassRule;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/** Test of {@link RabbitMqIO}. */
+@RunWith(JUnit4.class)
+public class RabbitMqIOTest implements Serializable {
+  private static final Logger LOG = 
LoggerFactory.getLogger(RabbitMqIOTest.class);
+
+  private static int port;
+  @ClassRule public static TemporaryFolder temporaryFolder = new 
TemporaryFolder();
+
+  @Rule public transient TestPipeline p = TestPipeline.create();
+
+  private static transient Broker broker;
+
+  @BeforeClass
+  public static void startBroker() throws Exception {
+try (ServerSocket serverSocket = new ServerSocket(0)) {
+  port = serverSocket.getLocalPort();
+}
+
+System.setProperty("derby.stream.error.field", "MyApp.DEV_NULL");
+broker = new Broker();
+BrokerOptions options = new BrokerOptions();
+options.setConfigProperty(BrokerOptions.QPID_AMQP_PORT, 
String.valueOf(port));
+options.setConfigProperty(BrokerOptions.QPID_WORK_DIR, 
temporaryFolder.newFolder().toString());
+options.setConfigProperty(BrokerOptions.QPID_HOME_DIR, "src/test/qpid");
+broker.startup(options);
+  }
+
+  @AfterClass
+  public static void stopBroker() {
+broker.shutdown();
+  }
+
+  @Test
+  public void testReadQueue() throws Exception {
+final int maxNumRecords = 10;
+PCollection raw =
+p.apply(
+RabbitMqIO.read()
+.withUri("amqp://guest:guest@localhost:" + port)
+.withQueue("READ")
+.withMaxNumRecords(maxNumRecords));
+PCollection output = raw.apply(ParDo.of(new ConverterFn()));
+
+List records = generateRecords(maxNumRecords);
+PAssert.that(output).containsInAnyOrder(records);
+
+ConnectionFactory connectionFactory = new ConnectionFactory();
+connectionFactory.setUri("amqp://guest:guest@localhost:" + port);
+Connection connection = connectionFactory.newConnection();
+Channel channel = connection.createChannel();
+channel.queueDeclare("READ", false, false, false, null);
+for (byte[] record : records) {
+  channel.basicPublish("", "READ", null, record);
+}
+
+p.run();
+
+channel.close(

[jira] [Work logged] (BEAM-1240) Create RabbitMqIO

2018-07-20 Thread ASF GitHub Bot (JIRA)



 [ 
https://issues.apache.org/jira/browse/BEAM-1240?focusedWorklogId=125431&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-125431
 ]

ASF GitHub Bot logged work on BEAM-1240:


Author: ASF GitHub Bot
Created on: 20/Jul/18 10:23
Start Date: 20/Jul/18 10:23
Worklog Time Spent: 10m 
  Work Description: echauchot commented on a change in pull request #1729: 
[BEAM-1240] Create RabbitMqIO
URL: https://github.com/apache/beam/pull/1729#discussion_r203788804
 
 

 ##
 File path: 
sdks/java/io/rabbitmq/src/test/java/org/apache/beam/sdk/io/rabbitmq/RabbitMqIOTest.java
 ##
 @@ -0,0 +1,269 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.io.rabbitmq;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+import com.rabbitmq.client.AMQP;
+import com.rabbitmq.client.Channel;
+import com.rabbitmq.client.Connection;
+import com.rabbitmq.client.ConnectionFactory;
+import com.rabbitmq.client.Consumer;
+import com.rabbitmq.client.DefaultConsumer;
+import com.rabbitmq.client.Envelope;
+import java.io.IOException;
+import java.io.Serializable;
+import java.net.ServerSocket;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.stream.Collectors;
+import java.util.stream.IntStream;
+import org.apache.beam.sdk.testing.PAssert;
+import org.apache.beam.sdk.testing.TestPipeline;
+import org.apache.beam.sdk.transforms.Create;
+import org.apache.beam.sdk.transforms.DoFn;
+import org.apache.beam.sdk.transforms.ParDo;
+import org.apache.beam.sdk.values.PCollection;
+import org.apache.qpid.server.Broker;
 
 Review comment:
   Glad to see no mock here ! 
   Is org.apache.qpid.server.Broker representative of RabbitMQ broker as this 
PR targets RabbitMQ and not generic AMQP broker ? Is there any RabbitMQ 
embedded broker usable for tests ?


This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
 
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org


Issue Time Tracking
---

Worklog Id: (was: 125431)
Time Spent: 9h 50m  (was: 9h 40m)

> Create RabbitMqIO
> -
>
> Key: BEAM-1240
> URL: https://issues.apache.org/jira/browse/BEAM-1240
> Project: Beam
>  Issue Type: New Feature
>  Components: io-ideas
>Reporter: Jean-Baptiste Onofré
>Assignee: Jean-Baptiste Onofré
>Priority: Major
> Fix For: 2.6.0
>
>  Time Spent: 9h 50m
>  Remaining Estimate: 0h
>




--
This message was sent by Atlassian JIRA
(v7.6.3#76005)

[jira] [Work logged] (BEAM-1240) Create RabbitMqIO

2018-07-20 Thread ASF GitHub Bot (JIRA)



 [ 
https://issues.apache.org/jira/browse/BEAM-1240?focusedWorklogId=125434&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-125434
 ]

ASF GitHub Bot logged work on BEAM-1240:


Author: ASF GitHub Bot
Created on: 20/Jul/18 10:23
Start Date: 20/Jul/18 10:23
Worklog Time Spent: 10m 
  Work Description: echauchot commented on a change in pull request #1729: 
[BEAM-1240] Create RabbitMqIO
URL: https://github.com/apache/beam/pull/1729#discussion_r204000498
 
 

 ##
 File path: 
sdks/java/io/rabbitmq/src/main/java/org/apache/beam/sdk/io/rabbitmq/RabbitMqIO.java
 ##
 @@ -0,0 +1,655 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.io.rabbitmq;
+
+import static com.google.common.base.Preconditions.checkArgument;
+
+import com.google.auto.value.AutoValue;
+import com.rabbitmq.client.Channel;
+import com.rabbitmq.client.Connection;
+import com.rabbitmq.client.ConnectionFactory;
+import com.rabbitmq.client.MessageProperties;
+import com.rabbitmq.client.QueueingConsumer;
+import java.io.IOException;
+import java.io.Serializable;
+import java.net.URISyntaxException;
+import java.nio.charset.StandardCharsets;
+import java.security.KeyManagementException;
+import java.security.NoSuchAlgorithmException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.NoSuchElementException;
+import java.util.concurrent.TimeoutException;
+import javax.annotation.Nullable;
+import org.apache.beam.sdk.annotations.Experimental;
+import org.apache.beam.sdk.coders.Coder;
+import org.apache.beam.sdk.coders.SerializableCoder;
+import org.apache.beam.sdk.io.UnboundedSource;
+import org.apache.beam.sdk.options.PipelineOptions;
+import org.apache.beam.sdk.transforms.DoFn;
+import org.apache.beam.sdk.transforms.PTransform;
+import org.apache.beam.sdk.transforms.ParDo;
+import org.apache.beam.sdk.values.PBegin;
+import org.apache.beam.sdk.values.PCollection;
+import org.joda.time.Duration;
+import org.joda.time.Instant;
+
+/**
+ * A IO to publish or consume messages with a RabbitMQ broker.
+ *
+ * Consuming messages from RabbitMQ server
+ *
+ * {@link RabbitMqIO} {@link Read} returns an unbounded {@link PCollection} 
containing RabbitMQ
+ * messages body (as {@code byte[]}) wrapped as {@link RabbitMqMessage}.
+ *
+ * To configure a RabbitMQ source, you have to provide a RabbitMQ {@code 
URI} to connect to a
+ * RabbitMQ broker. The following example illustrates various options for 
configuring the source,
+ * reading from the queue:
+ *
+ * {@code
+ * pipeline.apply(
+ *   
RabbitMqIO.read().withUri("amqp://user:password@localhost:5672").withQueue("QUEUE")
+ *
+ * }
+ *
+ * It's also possible to read from an exchange (providing the exchange type 
and routing key)
+ * instead of directly from a queue:
+ *
+ * {@code
+ * pipeline.apply(
+ *   
RabbitMqIO.read().withUri("amqp://user:password@localhost:5672").withExchange("EXCHANGE",
 "fanout", "QUEUE");
+ * }
+ *
+ * Publishing messages to RabbitMQ server
+ *
+ * {@link RabbitMqIO} {@link Write} can send {@code byte[]} to a RabbitMQ 
server queue or
+ * exchange.
+ *
+ * As for the {@link Read}, the {@link Write} is configured with a RabbitMQ 
URI.
+ *
+ * For instance, you can write to an exchange (providing the exchange type):
+ *
+ * {@code
+ * pipeline
+ *   .apply(...) // provide PCollection
+ *   
.apply(RabbitMqIO.write().withUri("amqp://user:password@localhost:5672").withExchange("EXCHANGE",
 "fanout"));
+ * }
+ *
+ * For instance, you can write to a queue:
+ *
+ * {@code
+ * pipeline
+ *   .apply(...) // provide PCollection
+ *   
.apply(RabbitMqIO.write().withUri("amqp://user:password@localhost:5672").withQueue("QUEUE"));
+ *
+ * }
+ */
+@Experimental(Experimental.Kind.SOURCE_SINK)
+public class RabbitMqIO {
+  public static Read read() {
+return new AutoValue_RabbitMqIO_Read.Builder()
+.setQueueDeclare(false)
+.setMaxReadTime(null)
+.setMaxNumRecords(Long.MAX_VALUE)
+.setUseCorrelationId(false)
+.build();
+  }
+
+  public static Write write() {
+return new AutoValue_RabbitMqIO_Write.Builder()
+

[jira] [Work logged] (BEAM-1240) Create RabbitMqIO

2018-07-20 Thread ASF GitHub Bot (JIRA)



 [ 
https://issues.apache.org/jira/browse/BEAM-1240?focusedWorklogId=125433&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-125433
 ]

ASF GitHub Bot logged work on BEAM-1240:


Author: ASF GitHub Bot
Created on: 20/Jul/18 10:23
Start Date: 20/Jul/18 10:23
Worklog Time Spent: 10m 
  Work Description: echauchot commented on a change in pull request #1729: 
[BEAM-1240] Create RabbitMqIO
URL: https://github.com/apache/beam/pull/1729#discussion_r203992442
 
 

 ##
 File path: 
sdks/java/io/rabbitmq/src/test/java/org/apache/beam/sdk/io/rabbitmq/RabbitMqIOTest.java
 ##
 @@ -0,0 +1,269 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.io.rabbitmq;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+import com.rabbitmq.client.AMQP;
+import com.rabbitmq.client.Channel;
+import com.rabbitmq.client.Connection;
+import com.rabbitmq.client.ConnectionFactory;
+import com.rabbitmq.client.Consumer;
+import com.rabbitmq.client.DefaultConsumer;
+import com.rabbitmq.client.Envelope;
+import java.io.IOException;
+import java.io.Serializable;
+import java.net.ServerSocket;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.stream.Collectors;
+import java.util.stream.IntStream;
+import org.apache.beam.sdk.testing.PAssert;
+import org.apache.beam.sdk.testing.TestPipeline;
+import org.apache.beam.sdk.transforms.Create;
+import org.apache.beam.sdk.transforms.DoFn;
+import org.apache.beam.sdk.transforms.ParDo;
+import org.apache.beam.sdk.values.PCollection;
+import org.apache.qpid.server.Broker;
+import org.apache.qpid.server.BrokerOptions;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.ClassRule;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/** Test of {@link RabbitMqIO}. */
+@RunWith(JUnit4.class)
+public class RabbitMqIOTest implements Serializable {
+  private static final Logger LOG = 
LoggerFactory.getLogger(RabbitMqIOTest.class);
+
+  private static int port;
+  @ClassRule public static TemporaryFolder temporaryFolder = new 
TemporaryFolder();
+
+  @Rule public transient TestPipeline p = TestPipeline.create();
+
+  private static transient Broker broker;
+
+  @BeforeClass
+  public static void startBroker() throws Exception {
+try (ServerSocket serverSocket = new ServerSocket(0)) {
+  port = serverSocket.getLocalPort();
+}
+
+System.setProperty("derby.stream.error.field", "MyApp.DEV_NULL");
+broker = new Broker();
+BrokerOptions options = new BrokerOptions();
+options.setConfigProperty(BrokerOptions.QPID_AMQP_PORT, 
String.valueOf(port));
+options.setConfigProperty(BrokerOptions.QPID_WORK_DIR, 
temporaryFolder.newFolder().toString());
+options.setConfigProperty(BrokerOptions.QPID_HOME_DIR, "src/test/qpid");
+broker.startup(options);
+  }
+
+  @AfterClass
+  public static void stopBroker() {
+broker.shutdown();
+  }
+
+  @Test
+  public void testReadQueue() throws Exception {
+final int maxNumRecords = 10;
+PCollection raw =
+p.apply(
+RabbitMqIO.read()
+.withUri("amqp://guest:guest@localhost:" + port)
+.withQueue("READ")
+.withMaxNumRecords(maxNumRecords));
+PCollection output = raw.apply(ParDo.of(new ConverterFn()));
+
+List records = generateRecords(maxNumRecords);
+PAssert.that(output).containsInAnyOrder(records);
+
+ConnectionFactory connectionFactory = new ConnectionFactory();
+connectionFactory.setUri("amqp://guest:guest@localhost:" + port);
+Connection connection = connectionFactory.newConnection();
+Channel channel = connection.createChannel();
+channel.queueDeclare("READ", false, false, false, null);
+for (byte[] record : records) {
+  channel.basicPublish("", "READ", null, record);
+}
+
+p.run();
+
+channel.close(

[jira] [Work logged] (BEAM-1240) Create RabbitMqIO

2018-07-20 Thread ASF GitHub Bot (JIRA)



 [ 
https://issues.apache.org/jira/browse/BEAM-1240?focusedWorklogId=125374&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-125374
 ]

ASF GitHub Bot logged work on BEAM-1240:


Author: ASF GitHub Bot
Created on: 20/Jul/18 07:40
Start Date: 20/Jul/18 07:40
Worklog Time Spent: 10m 
  Work Description: akankshajain18 edited a comment on issue #1729: 
[BEAM-1240] Create RabbitMqIO
URL: https://github.com/apache/beam/pull/1729#issuecomment-406515489
 
 
   Hi All
   
   Thanks for RmqIO :)
   But It will be great if setDurable(boolean durable) API get exposed to end 
user
   As of now, non-durable queue get created everytime.
   [
// channel.queueDeclare(queueName, durable, exclusive, autoDelete, 
arguments);
   channel.queueDeclare(queueName, false, false, false, null)
   ]


This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
 
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org


Issue Time Tracking
---

Worklog Id: (was: 125374)
Time Spent: 9h 40m  (was: 9.5h)

> Create RabbitMqIO
> -
>
> Key: BEAM-1240
> URL: https://issues.apache.org/jira/browse/BEAM-1240
> Project: Beam
>  Issue Type: New Feature
>  Components: io-ideas
>Reporter: Jean-Baptiste Onofré
>Assignee: Jean-Baptiste Onofré
>Priority: Major
> Fix For: 2.6.0
>
>  Time Spent: 9h 40m
>  Remaining Estimate: 0h
>




--
This message was sent by Atlassian JIRA
(v7.6.3#76005)

[jira] [Work logged] (BEAM-1240) Create RabbitMqIO

2018-07-20 Thread ASF GitHub Bot (JIRA)



 [ 
https://issues.apache.org/jira/browse/BEAM-1240?focusedWorklogId=125373&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-125373
 ]

ASF GitHub Bot logged work on BEAM-1240:


Author: ASF GitHub Bot
Created on: 20/Jul/18 07:37
Start Date: 20/Jul/18 07:37
Worklog Time Spent: 10m 
  Work Description: akankshajain18 commented on issue #1729: [BEAM-1240] 
Create RabbitMqIO
URL: https://github.com/apache/beam/pull/1729#issuecomment-406515489
 
 
   Hi All
   It will be great if setDurable(boolean durable) API get exposed to end user
   As of now, false is set while declaring the queue.
   [
// channel.queueDeclare(queueName, durable, exclusive, autoDelete, 
arguments);
   channel.queueDeclare(queueName, false, false, false, null)
   ]


This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
 
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org


Issue Time Tracking
---

Worklog Id: (was: 125373)
Time Spent: 9.5h  (was: 9h 20m)

> Create RabbitMqIO
> -
>
> Key: BEAM-1240
> URL: https://issues.apache.org/jira/browse/BEAM-1240
> Project: Beam
>  Issue Type: New Feature
>  Components: io-ideas
>Reporter: Jean-Baptiste Onofré
>Assignee: Jean-Baptiste Onofré
>Priority: Major
> Fix For: 2.6.0
>
>  Time Spent: 9.5h
>  Remaining Estimate: 0h
>




--
This message was sent by Atlassian JIRA
(v7.6.3#76005)

[jira] [Work logged] (BEAM-1240) Create RabbitMqIO

2018-07-19 Thread ASF GitHub Bot (JIRA)



 [ 
https://issues.apache.org/jira/browse/BEAM-1240?focusedWorklogId=125212&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-125212
 ]

ASF GitHub Bot logged work on BEAM-1240:


Author: ASF GitHub Bot
Created on: 19/Jul/18 19:48
Start Date: 19/Jul/18 19:48
Worklog Time Spent: 10m 
  Work Description: jbonofre commented on issue #1729: [BEAM-1240] Create 
RabbitMqIO
URL: https://github.com/apache/beam/pull/1729#issuecomment-406393148
 
 
   @jkff thanks for the update, I will address your comments !


This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
 
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org


Issue Time Tracking
---

Worklog Id: (was: 125212)
Time Spent: 9h 20m  (was: 9h 10m)

> Create RabbitMqIO
> -
>
> Key: BEAM-1240
> URL: https://issues.apache.org/jira/browse/BEAM-1240
> Project: Beam
>  Issue Type: New Feature
>  Components: io-ideas
>Reporter: Jean-Baptiste Onofré
>Assignee: Jean-Baptiste Onofré
>Priority: Major
> Fix For: 2.6.0
>
>  Time Spent: 9h 20m
>  Remaining Estimate: 0h
>




--
This message was sent by Atlassian JIRA
(v7.6.3#76005)

[jira] [Work logged] (BEAM-1240) Create RabbitMqIO

2018-07-19 Thread ASF GitHub Bot (JIRA)



 [ 
https://issues.apache.org/jira/browse/BEAM-1240?focusedWorklogId=125193&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-125193
 ]

ASF GitHub Bot logged work on BEAM-1240:


Author: ASF GitHub Bot
Created on: 19/Jul/18 19:03
Start Date: 19/Jul/18 19:03
Worklog Time Spent: 10m 
  Work Description: jkff commented on a change in pull request #1729: 
[BEAM-1240] Create RabbitMqIO
URL: https://github.com/apache/beam/pull/1729#discussion_r203837280
 
 

 ##
 File path: 
sdks/java/io/rabbitmq/src/main/java/org/apache/beam/sdk/io/rabbitmq/RabbitMqIO.java
 ##
 @@ -0,0 +1,655 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.io.rabbitmq;
+
+import static com.google.common.base.Preconditions.checkArgument;
+
+import com.google.auto.value.AutoValue;
+import com.rabbitmq.client.Channel;
+import com.rabbitmq.client.Connection;
+import com.rabbitmq.client.ConnectionFactory;
+import com.rabbitmq.client.MessageProperties;
+import com.rabbitmq.client.QueueingConsumer;
+import java.io.IOException;
+import java.io.Serializable;
+import java.net.URISyntaxException;
+import java.nio.charset.StandardCharsets;
+import java.security.KeyManagementException;
+import java.security.NoSuchAlgorithmException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.NoSuchElementException;
+import java.util.concurrent.TimeoutException;
+import javax.annotation.Nullable;
+import org.apache.beam.sdk.annotations.Experimental;
+import org.apache.beam.sdk.coders.Coder;
+import org.apache.beam.sdk.coders.SerializableCoder;
+import org.apache.beam.sdk.io.UnboundedSource;
+import org.apache.beam.sdk.options.PipelineOptions;
+import org.apache.beam.sdk.transforms.DoFn;
+import org.apache.beam.sdk.transforms.PTransform;
+import org.apache.beam.sdk.transforms.ParDo;
+import org.apache.beam.sdk.values.PBegin;
+import org.apache.beam.sdk.values.PCollection;
+import org.joda.time.Duration;
+import org.joda.time.Instant;
+
+/**
+ * A IO to publish or consume messages with a RabbitMQ broker.
+ *
+ * Consuming messages from RabbitMQ server
+ *
+ * {@link RabbitMqIO} {@link Read} returns an unbounded {@link PCollection} 
containing RabbitMQ
+ * messages body (as {@code byte[]}) wrapped as {@link RabbitMqMessage}.
+ *
+ * To configure a RabbitMQ source, you have to provide a RabbitMQ {@code 
URI} to connect to a
+ * RabbitMQ broker. The following example illustrates various options for 
configuring the source,
+ * reading from the queue:
+ *
+ * {@code
+ * pipeline.apply(
+ *   
RabbitMqIO.read().withUri("amqp://user:password@localhost:5672").withQueue("QUEUE")
+ *
+ * }
+ *
+ * It's also possible to read from an exchange (providing the exchange type 
and routing key)
+ * instead of directly from a queue:
+ *
+ * {@code
+ * pipeline.apply(
+ *   
RabbitMqIO.read().withUri("amqp://user:password@localhost:5672").withExchange("EXCHANGE",
 "fanout", "QUEUE");
+ * }
+ *
+ * Publishing messages to RabbitMQ server
+ *
+ * {@link RabbitMqIO} {@link Write} can send {@code byte[]} to a RabbitMQ 
server queue or
+ * exchange.
+ *
+ * As for the {@link Read}, the {@link Write} is configured with a RabbitMQ 
URI.
+ *
+ * For instance, you can write to an exchange (providing the exchange type):
+ *
+ * {@code
+ * pipeline
+ *   .apply(...) // provide PCollection
+ *   
.apply(RabbitMqIO.write().withUri("amqp://user:password@localhost:5672").withExchange("EXCHANGE",
 "fanout"));
+ * }
+ *
+ * For instance, you can write to a queue:
+ *
+ * {@code
+ * pipeline
+ *   .apply(...) // provide PCollection
+ *   
.apply(RabbitMqIO.write().withUri("amqp://user:password@localhost:5672").withQueue("QUEUE"));
+ *
+ * }
+ */
+@Experimental(Experimental.Kind.SOURCE_SINK)
+public class RabbitMqIO {
+  public static Read read() {
+return new AutoValue_RabbitMqIO_Read.Builder()
+.setQueueDeclare(false)
+.setMaxReadTime(null)
+.setMaxNumRecords(Long.MAX_VALUE)
+.setUseCorrelationId(false)
+.build();
+  }
+
+  public static Write write() {
+return new AutoValue_RabbitMqIO_Write.Builder()
+

[jira] [Work logged] (BEAM-1240) Create RabbitMqIO

2018-07-19 Thread ASF GitHub Bot (JIRA)



 [ 
https://issues.apache.org/jira/browse/BEAM-1240?focusedWorklogId=125185&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-125185
 ]

ASF GitHub Bot logged work on BEAM-1240:


Author: ASF GitHub Bot
Created on: 19/Jul/18 19:03
Start Date: 19/Jul/18 19:03
Worklog Time Spent: 10m 
  Work Description: jkff commented on a change in pull request #1729: 
[BEAM-1240] Create RabbitMqIO
URL: https://github.com/apache/beam/pull/1729#discussion_r203835879
 
 

 ##
 File path: 
sdks/java/io/rabbitmq/src/main/java/org/apache/beam/sdk/io/rabbitmq/RabbitMqIO.java
 ##
 @@ -0,0 +1,655 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.io.rabbitmq;
+
+import static com.google.common.base.Preconditions.checkArgument;
+
+import com.google.auto.value.AutoValue;
+import com.rabbitmq.client.Channel;
+import com.rabbitmq.client.Connection;
+import com.rabbitmq.client.ConnectionFactory;
+import com.rabbitmq.client.MessageProperties;
+import com.rabbitmq.client.QueueingConsumer;
+import java.io.IOException;
+import java.io.Serializable;
+import java.net.URISyntaxException;
+import java.nio.charset.StandardCharsets;
+import java.security.KeyManagementException;
+import java.security.NoSuchAlgorithmException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.NoSuchElementException;
+import java.util.concurrent.TimeoutException;
+import javax.annotation.Nullable;
+import org.apache.beam.sdk.annotations.Experimental;
+import org.apache.beam.sdk.coders.Coder;
+import org.apache.beam.sdk.coders.SerializableCoder;
+import org.apache.beam.sdk.io.UnboundedSource;
+import org.apache.beam.sdk.options.PipelineOptions;
+import org.apache.beam.sdk.transforms.DoFn;
+import org.apache.beam.sdk.transforms.PTransform;
+import org.apache.beam.sdk.transforms.ParDo;
+import org.apache.beam.sdk.values.PBegin;
+import org.apache.beam.sdk.values.PCollection;
+import org.joda.time.Duration;
+import org.joda.time.Instant;
+
+/**
+ * A IO to publish or consume messages with a RabbitMQ broker.
+ *
+ * Consuming messages from RabbitMQ server
+ *
+ * {@link RabbitMqIO} {@link Read} returns an unbounded {@link PCollection} 
containing RabbitMQ
+ * messages body (as {@code byte[]}) wrapped as {@link RabbitMqMessage}.
+ *
+ * To configure a RabbitMQ source, you have to provide a RabbitMQ {@code 
URI} to connect to a
+ * RabbitMQ broker. The following example illustrates various options for 
configuring the source,
+ * reading from the queue:
+ *
+ * {@code
+ * pipeline.apply(
+ *   
RabbitMqIO.read().withUri("amqp://user:password@localhost:5672").withQueue("QUEUE")
 
 Review comment:
   Missing ), and missing "PCollection messages = ..."


This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
 
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org


Issue Time Tracking
---

Worklog Id: (was: 125185)
Time Spent: 8h 10m  (was: 8h)

> Create RabbitMqIO
> -
>
> Key: BEAM-1240
> URL: https://issues.apache.org/jira/browse/BEAM-1240
> Project: Beam
>  Issue Type: New Feature
>  Components: io-ideas
>Reporter: Jean-Baptiste Onofré
>Assignee: Jean-Baptiste Onofré
>Priority: Major
> Fix For: 2.6.0
>
>  Time Spent: 8h 10m
>  Remaining Estimate: 0h
>




--
This message was sent by Atlassian JIRA
(v7.6.3#76005)

[jira] [Work logged] (BEAM-1240) Create RabbitMqIO

2018-07-19 Thread ASF GitHub Bot (JIRA)



 [ 
https://issues.apache.org/jira/browse/BEAM-1240?focusedWorklogId=125188&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-125188
 ]

ASF GitHub Bot logged work on BEAM-1240:


Author: ASF GitHub Bot
Created on: 19/Jul/18 19:03
Start Date: 19/Jul/18 19:03
Worklog Time Spent: 10m 
  Work Description: jkff commented on a change in pull request #1729: 
[BEAM-1240] Create RabbitMqIO
URL: https://github.com/apache/beam/pull/1729#discussion_r203837537
 
 

 ##
 File path: 
sdks/java/io/rabbitmq/src/main/java/org/apache/beam/sdk/io/rabbitmq/RabbitMqMessage.java
 ##
 @@ -0,0 +1,297 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.io.rabbitmq;
+
+import com.rabbitmq.client.AMQP;
+import java.io.Serializable;
+import java.util.Arrays;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Objects;
+import javax.annotation.Nullable;
+
+/**
+ * It contains the message payload, and additional metadata like routing key 
or attributes. The main
+ * reason of this class is that AMQP.BasicProperties doesn't provide a 
serializable public API.
+ */
+public class RabbitMqMessage implements Serializable {
+
+  @Nullable private final String routingKey;
+  private final byte[] body;
+  private final String contentType;
+  private final String contentEncoding;
+  private final Map headers;
+  private final Integer deliveryMode;
+  private final Integer priority;
+  @Nullable private final String correlationId;
+  @Nullable private final String replyTo;
+  private final String expiration;
+  private final String messageId;
+  private final Date timestamp;
+  @Nullable private final String type;
+  @Nullable private final String userId;
+  @Nullable private final String appId;
+  @Nullable private final String clusterId;
+
+  public RabbitMqMessage(byte[] body) {
+this.body = body;
+routingKey = "";
+contentType = null;
+contentEncoding = null;
+headers = new HashMap<>();
+deliveryMode = 1;
+priority = 1;
+correlationId = null;
+replyTo = null;
+expiration = null;
+messageId = null;
+timestamp = new Date();
+type = null;
+userId = null;
+appId = null;
+clusterId = null;
+  }
+
+  public RabbitMqMessage(
+  String routingKey,
+  byte[] body,
+  String contentType,
+  String contentEncoding,
+  Map headers,
+  Integer deliveryMode,
+  Integer priority,
+  String correlationId,
+  String replyTo,
+  String expiration,
+  String messageId,
+  Date timestamp,
+  String type,
+  String userId,
+  String appId,
+  String clusterId) {
+this.routingKey = routingKey;
+this.body = body;
+this.contentType = contentType;
+this.contentEncoding = contentEncoding;
+this.headers = headers;
+this.deliveryMode = deliveryMode;
+this.priority = priority;
+this.correlationId = correlationId;
+this.replyTo = replyTo;
+this.expiration = expiration;
+this.messageId = messageId;
+this.timestamp = timestamp;
+this.type = type;
+this.userId = userId;
+this.appId = appId;
+this.clusterId = clusterId;
+  }
+
+  public String getRoutingKey() {
+return routingKey;
+  }
+
+  public byte[] getBody() {
+return body;
+  }
+
+  public String getContentType() {
+return contentType;
+  }
+
+  public String getContentEncoding() {
+return contentEncoding;
+  }
+
+  public Map getHeaders() {
+return headers;
+  }
+
+  public Integer getDeliveryMode() {
+return deliveryMode;
+  }
+
+  public Integer getPriority() {
+return priority;
+  }
+
+  public String getCorrelationId() {
+return correlationId;
+  }
+
+  public String getReplyTo() {
+return replyTo;
+  }
+
+  public String getExpiration() {
+return expiration;
+  }
+
+  public String getMessageId() {
+return messageId;
+  }
+
+  public Date getTimestamp() {
+return timestamp;
+  }
+
+  public String getType() {
+return type;
+  }
+
+  public String getUserId() {
+return userId;
+  }
+
+  public String getAppI

[jira] [Work logged] (BEAM-1240) Create RabbitMqIO

2018-07-19 Thread ASF GitHub Bot (JIRA)



 [ 
https://issues.apache.org/jira/browse/BEAM-1240?focusedWorklogId=125197&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-125197
 ]

ASF GitHub Bot logged work on BEAM-1240:


Author: ASF GitHub Bot
Created on: 19/Jul/18 19:03
Start Date: 19/Jul/18 19:03
Worklog Time Spent: 10m 
  Work Description: jkff commented on a change in pull request #1729: 
[BEAM-1240] Create RabbitMqIO
URL: https://github.com/apache/beam/pull/1729#discussion_r203837119
 
 

 ##
 File path: 
sdks/java/io/rabbitmq/src/main/java/org/apache/beam/sdk/io/rabbitmq/RabbitMqMessage.java
 ##
 @@ -0,0 +1,297 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.io.rabbitmq;
+
+import com.rabbitmq.client.AMQP;
+import java.io.Serializable;
+import java.util.Arrays;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Objects;
+import javax.annotation.Nullable;
+
+/**
+ * It contains the message payload, and additional metadata like routing key 
or attributes. The main
+ * reason of this class is that AMQP.BasicProperties doesn't provide a 
serializable public API.
+ */
+public class RabbitMqMessage implements Serializable {
+
+  @Nullable private final String routingKey;
+  private final byte[] body;
+  private final String contentType;
+  private final String contentEncoding;
+  private final Map headers;
+  private final Integer deliveryMode;
+  private final Integer priority;
+  @Nullable private final String correlationId;
+  @Nullable private final String replyTo;
+  private final String expiration;
+  private final String messageId;
+  private final Date timestamp;
+  @Nullable private final String type;
+  @Nullable private final String userId;
+  @Nullable private final String appId;
+  @Nullable private final String clusterId;
+
+  public RabbitMqMessage(byte[] body) {
+this.body = body;
+routingKey = "";
+contentType = null;
+contentEncoding = null;
+headers = new HashMap<>();
+deliveryMode = 1;
+priority = 1;
+correlationId = null;
+replyTo = null;
+expiration = null;
+messageId = null;
+timestamp = new Date();
+type = null;
+userId = null;
+appId = null;
+clusterId = null;
+  }
+
+  public RabbitMqMessage(
+  String routingKey,
+  byte[] body,
+  String contentType,
+  String contentEncoding,
+  Map headers,
+  Integer deliveryMode,
+  Integer priority,
+  String correlationId,
+  String replyTo,
+  String expiration,
+  String messageId,
+  Date timestamp,
+  String type,
+  String userId,
+  String appId,
+  String clusterId) {
+this.routingKey = routingKey;
+this.body = body;
+this.contentType = contentType;
+this.contentEncoding = contentEncoding;
+this.headers = headers;
+this.deliveryMode = deliveryMode;
+this.priority = priority;
+this.correlationId = correlationId;
+this.replyTo = replyTo;
+this.expiration = expiration;
+this.messageId = messageId;
+this.timestamp = timestamp;
+this.type = type;
+this.userId = userId;
+this.appId = appId;
+this.clusterId = clusterId;
+  }
+
+  public String getRoutingKey() {
+return routingKey;
+  }
+
+  public byte[] getBody() {
+return body;
+  }
+
+  public String getContentType() {
+return contentType;
+  }
+
+  public String getContentEncoding() {
+return contentEncoding;
+  }
+
+  public Map getHeaders() {
+return headers;
+  }
+
+  public Integer getDeliveryMode() {
+return deliveryMode;
+  }
+
+  public Integer getPriority() {
+return priority;
+  }
+
+  public String getCorrelationId() {
+return correlationId;
+  }
+
+  public String getReplyTo() {
+return replyTo;
+  }
+
+  public String getExpiration() {
+return expiration;
+  }
+
+  public String getMessageId() {
+return messageId;
+  }
+
+  public Date getTimestamp() {
+return timestamp;
+  }
+
+  public String getType() {
+return type;
+  }
+
+  public String getUserId() {
+return userId;
+  }
+
+  public String getAppI

[jira] [Work logged] (BEAM-1240) Create RabbitMqIO

2018-07-19 Thread ASF GitHub Bot (JIRA)



 [ 
https://issues.apache.org/jira/browse/BEAM-1240?focusedWorklogId=125189&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-125189
 ]

ASF GitHub Bot logged work on BEAM-1240:


Author: ASF GitHub Bot
Created on: 19/Jul/18 19:03
Start Date: 19/Jul/18 19:03
Worklog Time Spent: 10m 
  Work Description: jkff commented on a change in pull request #1729: 
[BEAM-1240] Create RabbitMqIO
URL: https://github.com/apache/beam/pull/1729#discussion_r203837965
 
 

 ##
 File path: 
sdks/java/io/rabbitmq/src/test/java/org/apache/beam/sdk/io/rabbitmq/RabbitMqIOTest.java
 ##
 @@ -0,0 +1,269 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.io.rabbitmq;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+import com.rabbitmq.client.AMQP;
+import com.rabbitmq.client.Channel;
+import com.rabbitmq.client.Connection;
+import com.rabbitmq.client.ConnectionFactory;
+import com.rabbitmq.client.Consumer;
+import com.rabbitmq.client.DefaultConsumer;
+import com.rabbitmq.client.Envelope;
+import java.io.IOException;
+import java.io.Serializable;
+import java.net.ServerSocket;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.stream.Collectors;
+import java.util.stream.IntStream;
+import org.apache.beam.sdk.testing.PAssert;
+import org.apache.beam.sdk.testing.TestPipeline;
+import org.apache.beam.sdk.transforms.Create;
+import org.apache.beam.sdk.transforms.DoFn;
+import org.apache.beam.sdk.transforms.ParDo;
+import org.apache.beam.sdk.values.PCollection;
+import org.apache.qpid.server.Broker;
+import org.apache.qpid.server.BrokerOptions;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.ClassRule;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/** Test of {@link RabbitMqIO}. */
+@RunWith(JUnit4.class)
+public class RabbitMqIOTest implements Serializable {
+  private static final Logger LOG = 
LoggerFactory.getLogger(RabbitMqIOTest.class);
+
+  private static int port;
+  @ClassRule public static TemporaryFolder temporaryFolder = new 
TemporaryFolder();
+
+  @Rule public transient TestPipeline p = TestPipeline.create();
+
+  private static transient Broker broker;
+
+  @BeforeClass
+  public static void startBroker() throws Exception {
+try (ServerSocket serverSocket = new ServerSocket(0)) {
+  port = serverSocket.getLocalPort();
+}
+
+System.setProperty("derby.stream.error.field", "MyApp.DEV_NULL");
+broker = new Broker();
+BrokerOptions options = new BrokerOptions();
+options.setConfigProperty(BrokerOptions.QPID_AMQP_PORT, 
String.valueOf(port));
+options.setConfigProperty(BrokerOptions.QPID_WORK_DIR, 
temporaryFolder.newFolder().toString());
+options.setConfigProperty(BrokerOptions.QPID_HOME_DIR, "src/test/qpid");
+broker.startup(options);
+  }
+
+  @AfterClass
+  public static void stopBroker() {
+broker.shutdown();
+  }
+
+  @Test
+  public void testReadQueue() throws Exception {
+final int maxNumRecords = 10;
+PCollection raw =
+p.apply(
+RabbitMqIO.read()
+.withUri("amqp://guest:guest@localhost:" + port)
+.withQueue("READ")
+.withMaxNumRecords(maxNumRecords));
+PCollection output = raw.apply(ParDo.of(new ConverterFn()));
+
+List records = generateRecords(maxNumRecords);
+PAssert.that(output).containsInAnyOrder(records);
+
+ConnectionFactory connectionFactory = new ConnectionFactory();
+connectionFactory.setUri("amqp://guest:guest@localhost:" + port);
+Connection connection = connectionFactory.newConnection();
+Channel channel = connection.createChannel();
+channel.queueDeclare("READ", false, false, false, null);
+for (byte[] record : records) {
+  channel.basicPublish("", "READ", null, record);
+}
+
+p.run();
+
+channel.close();

[jira] [Work logged] (BEAM-1240) Create RabbitMqIO

2018-07-19 Thread ASF GitHub Bot (JIRA)



 [ 
https://issues.apache.org/jira/browse/BEAM-1240?focusedWorklogId=125191&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-125191
 ]

ASF GitHub Bot logged work on BEAM-1240:


Author: ASF GitHub Bot
Created on: 19/Jul/18 19:03
Start Date: 19/Jul/18 19:03
Worklog Time Spent: 10m 
  Work Description: jkff commented on a change in pull request #1729: 
[BEAM-1240] Create RabbitMqIO
URL: https://github.com/apache/beam/pull/1729#discussion_r203836690
 
 

 ##
 File path: 
sdks/java/io/rabbitmq/src/main/java/org/apache/beam/sdk/io/rabbitmq/RabbitMqIO.java
 ##
 @@ -0,0 +1,530 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.io.rabbitmq;
+
+import static com.google.common.base.Preconditions.checkArgument;
+
+import com.google.auto.value.AutoValue;
+import com.rabbitmq.client.Channel;
+import com.rabbitmq.client.Connection;
+import com.rabbitmq.client.ConnectionFactory;
+import com.rabbitmq.client.QueueingConsumer;
+import java.io.IOException;
+import java.io.Serializable;
+import java.net.URISyntaxException;
+import java.nio.charset.StandardCharsets;
+import java.security.KeyManagementException;
+import java.security.NoSuchAlgorithmException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.NoSuchElementException;
+import javax.annotation.Nullable;
+import org.apache.beam.sdk.annotations.Experimental;
+import org.apache.beam.sdk.coders.Coder;
+import org.apache.beam.sdk.coders.SerializableCoder;
+import org.apache.beam.sdk.io.UnboundedSource;
+import org.apache.beam.sdk.options.PipelineOptions;
+import org.apache.beam.sdk.transforms.DoFn;
+import org.apache.beam.sdk.transforms.PTransform;
+import org.apache.beam.sdk.transforms.ParDo;
+import org.apache.beam.sdk.values.PBegin;
+import org.apache.beam.sdk.values.PCollection;
+import org.apache.beam.sdk.values.PDone;
+import org.joda.time.Duration;
+import org.joda.time.Instant;
+
+/**
+ * A IO to publish or consume messages with a RabbitMQ broker.
+ *
+ * Consuming messages from RabbitMQ server
+ *
+ * {@link RabbitMqIO} {@link Read} returns an unbounded {@link PCollection} 
containing RabbitMQ
+ * messages body (as {@code byte[]}).
+ *
+ * To configure a RabbitMQ source, you have to provide a RabbitMQ {@code 
URI} to connect
+ * to a RabbitMQ broker. The following example illustrates various options for 
configuring the
+ * source:
+ *
+ * {@code
+ *
+ *  pipeline.apply(
+ *
RabbitMqIO.read().withUri("amqp://user:password@localhost:5672").withQueue("QUEUE")
+ *
+ * }
+ *
+ * Publishing messages to RabbitMQ server
+ *
+ * {@link RabbitMqIO} {@link Write} can send {@code byte[]} to a RabbitMQ 
server queue.
+ *
+ * As for the {@link Read}, the {@link Write} is configured with a RabbitMQ 
URI.
+ *
+ * For instance:
+ *
+ * {@code
+ *
+ *  pipeline
+ *.apply(...) // provide PCollection
+ *
.apply(RabbitMqIO.write().withUri("amqp://user:password@localhost:5672").withQueue("QUEUE"));
+ *
+ * }
+ */
+@Experimental(Experimental.Kind.SOURCE_SINK)
+public class RabbitMqIO {
+  public static Read read() {
+return new AutoValue_RabbitMqIO_Read.Builder().setQueueDeclare(false)
+
.setMaxReadTime(null).setMaxNumRecords(Long.MAX_VALUE).setUseCorrelationId(false).build();
+  }
+
+  public static Write write() {
+return new 
AutoValue_RabbitMqIO_Write.Builder().setExchangeDeclare(false).build();
+  }
+
+  private RabbitMqIO() {
+  }
+
+  private static ConnectionFactory createConnectionFactory(String uri) throws 
URISyntaxException,
+  NoSuchAlgorithmException, KeyManagementException {
+ConnectionFactory connectionFactory = new ConnectionFactory();
+connectionFactory.setUri(uri);
+connectionFactory.setAutomaticRecoveryEnabled(true);
+connectionFactory.setConnectionTimeout(6);
+connectionFactory.setNetworkRecoveryInterval(5000);
+connectionFactory.setRequestedHeartbeat(60);
+connectionFactory.setTopologyRecoveryEnabled(true);
+connectionFactory.setRequestedChannelMax(0);
+connectionFactory.setRequestedFrameMax(0);
+return connectionFactory;
+  }
+
+  /**

[jira] [Work logged] (BEAM-1240) Create RabbitMqIO

2018-07-19 Thread ASF GitHub Bot (JIRA)



 [ 
https://issues.apache.org/jira/browse/BEAM-1240?focusedWorklogId=125187&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-125187
 ]

ASF GitHub Bot logged work on BEAM-1240:


Author: ASF GitHub Bot
Created on: 19/Jul/18 19:03
Start Date: 19/Jul/18 19:03
Worklog Time Spent: 10m 
  Work Description: jkff commented on a change in pull request #1729: 
[BEAM-1240] Create RabbitMqIO
URL: https://github.com/apache/beam/pull/1729#discussion_r203836361
 
 

 ##
 File path: 
sdks/java/io/rabbitmq/src/main/java/org/apache/beam/sdk/io/rabbitmq/RabbitMqIO.java
 ##
 @@ -0,0 +1,530 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.io.rabbitmq;
+
+import static com.google.common.base.Preconditions.checkArgument;
+
+import com.google.auto.value.AutoValue;
+import com.rabbitmq.client.Channel;
+import com.rabbitmq.client.Connection;
+import com.rabbitmq.client.ConnectionFactory;
+import com.rabbitmq.client.QueueingConsumer;
+import java.io.IOException;
+import java.io.Serializable;
+import java.net.URISyntaxException;
+import java.nio.charset.StandardCharsets;
+import java.security.KeyManagementException;
+import java.security.NoSuchAlgorithmException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.NoSuchElementException;
+import javax.annotation.Nullable;
+import org.apache.beam.sdk.annotations.Experimental;
+import org.apache.beam.sdk.coders.Coder;
+import org.apache.beam.sdk.coders.SerializableCoder;
+import org.apache.beam.sdk.io.UnboundedSource;
+import org.apache.beam.sdk.options.PipelineOptions;
+import org.apache.beam.sdk.transforms.DoFn;
+import org.apache.beam.sdk.transforms.PTransform;
+import org.apache.beam.sdk.transforms.ParDo;
+import org.apache.beam.sdk.values.PBegin;
+import org.apache.beam.sdk.values.PCollection;
+import org.apache.beam.sdk.values.PDone;
+import org.joda.time.Duration;
+import org.joda.time.Instant;
+
+/**
+ * A IO to publish or consume messages with a RabbitMQ broker.
+ *
+ * Consuming messages from RabbitMQ server
+ *
+ * {@link RabbitMqIO} {@link Read} returns an unbounded {@link PCollection} 
containing RabbitMQ
+ * messages body (as {@code byte[]}).
+ *
+ * To configure a RabbitMQ source, you have to provide a RabbitMQ {@code 
URI} to connect
+ * to a RabbitMQ broker. The following example illustrates various options for 
configuring the
+ * source:
+ *
+ * {@code
+ *
+ *  pipeline.apply(
+ *
RabbitMqIO.read().withUri("amqp://user:password@localhost:5672").withQueue("QUEUE")
+ *
+ * }
+ *
+ * Publishing messages to RabbitMQ server
+ *
+ * {@link RabbitMqIO} {@link Write} can send {@code byte[]} to a RabbitMQ 
server queue.
+ *
+ * As for the {@link Read}, the {@link Write} is configured with a RabbitMQ 
URI.
+ *
+ * For instance:
+ *
+ * {@code
+ *
+ *  pipeline
+ *.apply(...) // provide PCollection
+ *
.apply(RabbitMqIO.write().withUri("amqp://user:password@localhost:5672").withQueue("QUEUE"));
+ *
+ * }
+ */
+@Experimental(Experimental.Kind.SOURCE_SINK)
+public class RabbitMqIO {
+  public static Read read() {
+return new AutoValue_RabbitMqIO_Read.Builder().setQueueDeclare(false)
+
.setMaxReadTime(null).setMaxNumRecords(Long.MAX_VALUE).setUseCorrelationId(false).build();
+  }
+
+  public static Write write() {
+return new 
AutoValue_RabbitMqIO_Write.Builder().setExchangeDeclare(false).build();
+  }
+
+  private RabbitMqIO() {
+  }
+
+  private static ConnectionFactory createConnectionFactory(String uri) throws 
URISyntaxException,
+  NoSuchAlgorithmException, KeyManagementException {
+ConnectionFactory connectionFactory = new ConnectionFactory();
+connectionFactory.setUri(uri);
+connectionFactory.setAutomaticRecoveryEnabled(true);
+connectionFactory.setConnectionTimeout(6);
+connectionFactory.setNetworkRecoveryInterval(5000);
+connectionFactory.setRequestedHeartbeat(60);
+connectionFactory.setTopologyRecoveryEnabled(true);
+connectionFactory.setRequestedChannelMax(0);
+connectionFactory.setRequestedFrameMax(0);
+return connectionFactory;
+  }
+
+  /**

[jira] [Work logged] (BEAM-1240) Create RabbitMqIO

2018-07-19 Thread ASF GitHub Bot (JIRA)



 [ 
https://issues.apache.org/jira/browse/BEAM-1240?focusedWorklogId=125195&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-125195
 ]

ASF GitHub Bot logged work on BEAM-1240:


Author: ASF GitHub Bot
Created on: 19/Jul/18 19:03
Start Date: 19/Jul/18 19:03
Worklog Time Spent: 10m 
  Work Description: jkff commented on a change in pull request #1729: 
[BEAM-1240] Create RabbitMqIO
URL: https://github.com/apache/beam/pull/1729#discussion_r203838246
 
 

 ##
 File path: 
sdks/java/io/rabbitmq/src/test/java/org/apache/beam/sdk/io/rabbitmq/RabbitMqIOTest.java
 ##
 @@ -0,0 +1,269 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.io.rabbitmq;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+import com.rabbitmq.client.AMQP;
+import com.rabbitmq.client.Channel;
+import com.rabbitmq.client.Connection;
+import com.rabbitmq.client.ConnectionFactory;
+import com.rabbitmq.client.Consumer;
+import com.rabbitmq.client.DefaultConsumer;
+import com.rabbitmq.client.Envelope;
+import java.io.IOException;
+import java.io.Serializable;
+import java.net.ServerSocket;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.stream.Collectors;
+import java.util.stream.IntStream;
+import org.apache.beam.sdk.testing.PAssert;
+import org.apache.beam.sdk.testing.TestPipeline;
+import org.apache.beam.sdk.transforms.Create;
+import org.apache.beam.sdk.transforms.DoFn;
+import org.apache.beam.sdk.transforms.ParDo;
+import org.apache.beam.sdk.values.PCollection;
+import org.apache.qpid.server.Broker;
+import org.apache.qpid.server.BrokerOptions;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.ClassRule;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/** Test of {@link RabbitMqIO}. */
+@RunWith(JUnit4.class)
+public class RabbitMqIOTest implements Serializable {
+  private static final Logger LOG = 
LoggerFactory.getLogger(RabbitMqIOTest.class);
+
+  private static int port;
+  @ClassRule public static TemporaryFolder temporaryFolder = new 
TemporaryFolder();
+
+  @Rule public transient TestPipeline p = TestPipeline.create();
+
+  private static transient Broker broker;
+
+  @BeforeClass
+  public static void startBroker() throws Exception {
+try (ServerSocket serverSocket = new ServerSocket(0)) {
+  port = serverSocket.getLocalPort();
+}
+
+System.setProperty("derby.stream.error.field", "MyApp.DEV_NULL");
+broker = new Broker();
+BrokerOptions options = new BrokerOptions();
+options.setConfigProperty(BrokerOptions.QPID_AMQP_PORT, 
String.valueOf(port));
+options.setConfigProperty(BrokerOptions.QPID_WORK_DIR, 
temporaryFolder.newFolder().toString());
+options.setConfigProperty(BrokerOptions.QPID_HOME_DIR, "src/test/qpid");
+broker.startup(options);
+  }
+
+  @AfterClass
+  public static void stopBroker() {
+broker.shutdown();
+  }
+
+  @Test
+  public void testReadQueue() throws Exception {
+final int maxNumRecords = 10;
+PCollection raw =
+p.apply(
+RabbitMqIO.read()
+.withUri("amqp://guest:guest@localhost:" + port)
+.withQueue("READ")
+.withMaxNumRecords(maxNumRecords));
+PCollection output = raw.apply(ParDo.of(new ConverterFn()));
+
+List records = generateRecords(maxNumRecords);
+PAssert.that(output).containsInAnyOrder(records);
+
+ConnectionFactory connectionFactory = new ConnectionFactory();
+connectionFactory.setUri("amqp://guest:guest@localhost:" + port);
+Connection connection = connectionFactory.newConnection();
+Channel channel = connection.createChannel();
+channel.queueDeclare("READ", false, false, false, null);
+for (byte[] record : records) {
+  channel.basicPublish("", "READ", null, record);
+}
+
+p.run();
+
+channel.close();
+

[jira] [Work logged] (BEAM-1240) Create RabbitMqIO

2018-07-19 Thread ASF GitHub Bot (JIRA)



 [ 
https://issues.apache.org/jira/browse/BEAM-1240?focusedWorklogId=125184&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-125184
 ]

ASF GitHub Bot logged work on BEAM-1240:


Author: ASF GitHub Bot
Created on: 19/Jul/18 19:03
Start Date: 19/Jul/18 19:03
Worklog Time Spent: 10m 
  Work Description: jkff commented on a change in pull request #1729: 
[BEAM-1240] Create RabbitMqIO
URL: https://github.com/apache/beam/pull/1729#discussion_r203835977
 
 

 ##
 File path: 
sdks/java/io/rabbitmq/src/main/java/org/apache/beam/sdk/io/rabbitmq/RabbitMqIO.java
 ##
 @@ -0,0 +1,655 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.io.rabbitmq;
+
+import static com.google.common.base.Preconditions.checkArgument;
+
+import com.google.auto.value.AutoValue;
+import com.rabbitmq.client.Channel;
+import com.rabbitmq.client.Connection;
+import com.rabbitmq.client.ConnectionFactory;
+import com.rabbitmq.client.MessageProperties;
+import com.rabbitmq.client.QueueingConsumer;
+import java.io.IOException;
+import java.io.Serializable;
+import java.net.URISyntaxException;
+import java.nio.charset.StandardCharsets;
+import java.security.KeyManagementException;
+import java.security.NoSuchAlgorithmException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.NoSuchElementException;
+import java.util.concurrent.TimeoutException;
+import javax.annotation.Nullable;
+import org.apache.beam.sdk.annotations.Experimental;
+import org.apache.beam.sdk.coders.Coder;
+import org.apache.beam.sdk.coders.SerializableCoder;
+import org.apache.beam.sdk.io.UnboundedSource;
+import org.apache.beam.sdk.options.PipelineOptions;
+import org.apache.beam.sdk.transforms.DoFn;
+import org.apache.beam.sdk.transforms.PTransform;
+import org.apache.beam.sdk.transforms.ParDo;
+import org.apache.beam.sdk.values.PBegin;
+import org.apache.beam.sdk.values.PCollection;
+import org.joda.time.Duration;
+import org.joda.time.Instant;
+
+/**
+ * A IO to publish or consume messages with a RabbitMQ broker.
+ *
+ * Consuming messages from RabbitMQ server
+ *
+ * {@link RabbitMqIO} {@link Read} returns an unbounded {@link PCollection} 
containing RabbitMQ
+ * messages body (as {@code byte[]}) wrapped as {@link RabbitMqMessage}.
+ *
+ * To configure a RabbitMQ source, you have to provide a RabbitMQ {@code 
URI} to connect to a
+ * RabbitMQ broker. The following example illustrates various options for 
configuring the source,
+ * reading from the queue:
+ *
+ * {@code
+ * pipeline.apply(
+ *   
RabbitMqIO.read().withUri("amqp://user:password@localhost:5672").withQueue("QUEUE")
+ *
+ * }
+ *
+ * It's also possible to read from an exchange (providing the exchange type 
and routing key)
+ * instead of directly from a queue:
+ *
+ * {@code
+ * pipeline.apply(
+ *   
RabbitMqIO.read().withUri("amqp://user:password@localhost:5672").withExchange("EXCHANGE",
 "fanout", "QUEUE");
+ * }
+ *
+ * Publishing messages to RabbitMQ server
+ *
+ * {@link RabbitMqIO} {@link Write} can send {@code byte[]} to a RabbitMQ 
server queue or
 
 Review comment:
   It's actually RabbitMqMessage


This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
 
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org


Issue Time Tracking
---

Worklog Id: (was: 125184)
Time Spent: 8h 10m  (was: 8h)

> Create RabbitMqIO
> -
>
> Key: BEAM-1240
> URL: https://issues.apache.org/jira/browse/BEAM-1240
> Project: Beam
>  Issue Type: New Feature
>  Components: io-ideas
>Reporter: Jean-Baptiste Onofré
>Assignee: Jean-Baptiste Onofré
>Priority: Major
> Fix For: 2.6.0
>
>  Time Spent: 8h 10m
>  Remaining Estimate: 0h
>




--
This message was sent by Atlassian JIRA
(v7.6.3#76005)

[jira] [Work logged] (BEAM-1240) Create RabbitMqIO

2018-07-19 Thread ASF GitHub Bot (JIRA)



 [ 
https://issues.apache.org/jira/browse/BEAM-1240?focusedWorklogId=125186&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-125186
 ]

ASF GitHub Bot logged work on BEAM-1240:


Author: ASF GitHub Bot
Created on: 19/Jul/18 19:03
Start Date: 19/Jul/18 19:03
Worklog Time Spent: 10m 
  Work Description: jkff commented on a change in pull request #1729: 
[BEAM-1240] Create RabbitMqIO
URL: https://github.com/apache/beam/pull/1729#discussion_r203836628
 
 

 ##
 File path: 
sdks/java/io/rabbitmq/src/main/java/org/apache/beam/sdk/io/rabbitmq/RabbitMqIO.java
 ##
 @@ -0,0 +1,530 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.io.rabbitmq;
+
+import static com.google.common.base.Preconditions.checkArgument;
+
+import com.google.auto.value.AutoValue;
+import com.rabbitmq.client.Channel;
+import com.rabbitmq.client.Connection;
+import com.rabbitmq.client.ConnectionFactory;
+import com.rabbitmq.client.QueueingConsumer;
+import java.io.IOException;
+import java.io.Serializable;
+import java.net.URISyntaxException;
+import java.nio.charset.StandardCharsets;
+import java.security.KeyManagementException;
+import java.security.NoSuchAlgorithmException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.NoSuchElementException;
+import javax.annotation.Nullable;
+import org.apache.beam.sdk.annotations.Experimental;
+import org.apache.beam.sdk.coders.Coder;
+import org.apache.beam.sdk.coders.SerializableCoder;
+import org.apache.beam.sdk.io.UnboundedSource;
+import org.apache.beam.sdk.options.PipelineOptions;
+import org.apache.beam.sdk.transforms.DoFn;
+import org.apache.beam.sdk.transforms.PTransform;
+import org.apache.beam.sdk.transforms.ParDo;
+import org.apache.beam.sdk.values.PBegin;
+import org.apache.beam.sdk.values.PCollection;
+import org.apache.beam.sdk.values.PDone;
+import org.joda.time.Duration;
+import org.joda.time.Instant;
+
+/**
+ * A IO to publish or consume messages with a RabbitMQ broker.
+ *
+ * Consuming messages from RabbitMQ server
+ *
+ * {@link RabbitMqIO} {@link Read} returns an unbounded {@link PCollection} 
containing RabbitMQ
+ * messages body (as {@code byte[]}).
+ *
+ * To configure a RabbitMQ source, you have to provide a RabbitMQ {@code 
URI} to connect
+ * to a RabbitMQ broker. The following example illustrates various options for 
configuring the
+ * source:
+ *
+ * {@code
+ *
+ *  pipeline.apply(
+ *
RabbitMqIO.read().withUri("amqp://user:password@localhost:5672").withQueue("QUEUE")
+ *
+ * }
+ *
+ * Publishing messages to RabbitMQ server
+ *
+ * {@link RabbitMqIO} {@link Write} can send {@code byte[]} to a RabbitMQ 
server queue.
+ *
+ * As for the {@link Read}, the {@link Write} is configured with a RabbitMQ 
URI.
+ *
+ * For instance:
+ *
+ * {@code
+ *
+ *  pipeline
+ *.apply(...) // provide PCollection
+ *
.apply(RabbitMqIO.write().withUri("amqp://user:password@localhost:5672").withQueue("QUEUE"));
+ *
+ * }
+ */
+@Experimental(Experimental.Kind.SOURCE_SINK)
+public class RabbitMqIO {
+  public static Read read() {
+return new AutoValue_RabbitMqIO_Read.Builder().setQueueDeclare(false)
+
.setMaxReadTime(null).setMaxNumRecords(Long.MAX_VALUE).setUseCorrelationId(false).build();
+  }
+
+  public static Write write() {
+return new 
AutoValue_RabbitMqIO_Write.Builder().setExchangeDeclare(false).build();
+  }
+
+  private RabbitMqIO() {
+  }
+
+  private static ConnectionFactory createConnectionFactory(String uri) throws 
URISyntaxException,
+  NoSuchAlgorithmException, KeyManagementException {
+ConnectionFactory connectionFactory = new ConnectionFactory();
+connectionFactory.setUri(uri);
+connectionFactory.setAutomaticRecoveryEnabled(true);
+connectionFactory.setConnectionTimeout(6);
+connectionFactory.setNetworkRecoveryInterval(5000);
+connectionFactory.setRequestedHeartbeat(60);
+connectionFactory.setTopologyRecoveryEnabled(true);
+connectionFactory.setRequestedChannelMax(0);
+connectionFactory.setRequestedFrameMax(0);
+return connectionFactory;
+  }
+
+  /**

[jira] [Work logged] (BEAM-1240) Create RabbitMqIO

2018-07-19 Thread ASF GitHub Bot (JIRA)



 [ 
https://issues.apache.org/jira/browse/BEAM-1240?focusedWorklogId=125196&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-125196
 ]

ASF GitHub Bot logged work on BEAM-1240:


Author: ASF GitHub Bot
Created on: 19/Jul/18 19:03
Start Date: 19/Jul/18 19:03
Worklog Time Spent: 10m 
  Work Description: jkff commented on a change in pull request #1729: 
[BEAM-1240] Create RabbitMqIO
URL: https://github.com/apache/beam/pull/1729#discussion_r203837888
 
 

 ##
 File path: 
sdks/java/io/rabbitmq/src/test/java/org/apache/beam/sdk/io/rabbitmq/RabbitMqIOTest.java
 ##
 @@ -0,0 +1,269 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.io.rabbitmq;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+import com.rabbitmq.client.AMQP;
+import com.rabbitmq.client.Channel;
+import com.rabbitmq.client.Connection;
+import com.rabbitmq.client.ConnectionFactory;
+import com.rabbitmq.client.Consumer;
+import com.rabbitmq.client.DefaultConsumer;
+import com.rabbitmq.client.Envelope;
+import java.io.IOException;
+import java.io.Serializable;
+import java.net.ServerSocket;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.stream.Collectors;
+import java.util.stream.IntStream;
+import org.apache.beam.sdk.testing.PAssert;
+import org.apache.beam.sdk.testing.TestPipeline;
+import org.apache.beam.sdk.transforms.Create;
+import org.apache.beam.sdk.transforms.DoFn;
+import org.apache.beam.sdk.transforms.ParDo;
+import org.apache.beam.sdk.values.PCollection;
+import org.apache.qpid.server.Broker;
+import org.apache.qpid.server.BrokerOptions;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.ClassRule;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/** Test of {@link RabbitMqIO}. */
+@RunWith(JUnit4.class)
+public class RabbitMqIOTest implements Serializable {
+  private static final Logger LOG = 
LoggerFactory.getLogger(RabbitMqIOTest.class);
+
+  private static int port;
+  @ClassRule public static TemporaryFolder temporaryFolder = new 
TemporaryFolder();
+
+  @Rule public transient TestPipeline p = TestPipeline.create();
+
+  private static transient Broker broker;
+
+  @BeforeClass
+  public static void startBroker() throws Exception {
+try (ServerSocket serverSocket = new ServerSocket(0)) {
+  port = serverSocket.getLocalPort();
+}
+
+System.setProperty("derby.stream.error.field", "MyApp.DEV_NULL");
+broker = new Broker();
+BrokerOptions options = new BrokerOptions();
+options.setConfigProperty(BrokerOptions.QPID_AMQP_PORT, 
String.valueOf(port));
+options.setConfigProperty(BrokerOptions.QPID_WORK_DIR, 
temporaryFolder.newFolder().toString());
+options.setConfigProperty(BrokerOptions.QPID_HOME_DIR, "src/test/qpid");
+broker.startup(options);
+  }
+
+  @AfterClass
+  public static void stopBroker() {
+broker.shutdown();
+  }
+
+  @Test
+  public void testReadQueue() throws Exception {
 
 Review comment:
   Please make sure that tests cover all combinations of what to declare and 
what to not declare


This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
 
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org


Issue Time Tracking
---

Worklog Id: (was: 125196)
Time Spent: 9h  (was: 8h 50m)

> Create RabbitMqIO
> -
>
> Key: BEAM-1240
> URL: https://issues.apache.org/jira/browse/BEAM-1240
> Project: Beam
>  Issue Type: New Feature
>  Components: io-ideas
>Reporter: Jean-Baptiste Onofré
>Assignee: Jean-Baptiste Onofré
>Priority:

[jira] [Work logged] (BEAM-1240) Create RabbitMqIO

2018-07-19 Thread ASF GitHub Bot (JIRA)



 [ 
https://issues.apache.org/jira/browse/BEAM-1240?focusedWorklogId=125192&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-125192
 ]

ASF GitHub Bot logged work on BEAM-1240:


Author: ASF GitHub Bot
Created on: 19/Jul/18 19:03
Start Date: 19/Jul/18 19:03
Worklog Time Spent: 10m 
  Work Description: jkff commented on a change in pull request #1729: 
[BEAM-1240] Create RabbitMqIO
URL: https://github.com/apache/beam/pull/1729#discussion_r203837667
 
 

 ##
 File path: 
sdks/java/io/rabbitmq/src/main/java/org/apache/beam/sdk/io/rabbitmq/RabbitMqMessage.java
 ##
 @@ -0,0 +1,297 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.io.rabbitmq;
+
+import com.rabbitmq.client.AMQP;
+import java.io.Serializable;
+import java.util.Arrays;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Objects;
+import javax.annotation.Nullable;
+
+/**
+ * It contains the message payload, and additional metadata like routing key 
or attributes. The main
+ * reason of this class is that AMQP.BasicProperties doesn't provide a 
serializable public API.
+ */
+public class RabbitMqMessage implements Serializable {
+
+  @Nullable private final String routingKey;
+  private final byte[] body;
+  private final String contentType;
+  private final String contentEncoding;
+  private final Map headers;
+  private final Integer deliveryMode;
+  private final Integer priority;
+  @Nullable private final String correlationId;
+  @Nullable private final String replyTo;
+  private final String expiration;
+  private final String messageId;
+  private final Date timestamp;
+  @Nullable private final String type;
+  @Nullable private final String userId;
+  @Nullable private final String appId;
+  @Nullable private final String clusterId;
+
+  public RabbitMqMessage(byte[] body) {
+this.body = body;
+routingKey = "";
+contentType = null;
+contentEncoding = null;
+headers = new HashMap<>();
+deliveryMode = 1;
+priority = 1;
+correlationId = null;
+replyTo = null;
+expiration = null;
+messageId = null;
+timestamp = new Date();
+type = null;
+userId = null;
+appId = null;
+clusterId = null;
+  }
+
+  public RabbitMqMessage(
+  String routingKey,
+  byte[] body,
+  String contentType,
+  String contentEncoding,
+  Map headers,
+  Integer deliveryMode,
+  Integer priority,
+  String correlationId,
+  String replyTo,
+  String expiration,
+  String messageId,
+  Date timestamp,
+  String type,
+  String userId,
+  String appId,
+  String clusterId) {
+this.routingKey = routingKey;
+this.body = body;
+this.contentType = contentType;
+this.contentEncoding = contentEncoding;
+this.headers = headers;
+this.deliveryMode = deliveryMode;
+this.priority = priority;
+this.correlationId = correlationId;
+this.replyTo = replyTo;
+this.expiration = expiration;
+this.messageId = messageId;
+this.timestamp = timestamp;
+this.type = type;
+this.userId = userId;
+this.appId = appId;
+this.clusterId = clusterId;
+  }
+
+  public String getRoutingKey() {
+return routingKey;
+  }
+
+  public byte[] getBody() {
+return body;
+  }
+
+  public String getContentType() {
+return contentType;
+  }
+
+  public String getContentEncoding() {
+return contentEncoding;
+  }
+
+  public Map getHeaders() {
+return headers;
+  }
+
+  public Integer getDeliveryMode() {
+return deliveryMode;
+  }
+
+  public Integer getPriority() {
+return priority;
+  }
+
+  public String getCorrelationId() {
+return correlationId;
+  }
+
+  public String getReplyTo() {
+return replyTo;
+  }
+
+  public String getExpiration() {
+return expiration;
+  }
+
+  public String getMessageId() {
+return messageId;
+  }
+
+  public Date getTimestamp() {
+return timestamp;
+  }
+
+  public String getType() {
+return type;
+  }
+
+  public String getUserId() {
+return userId;
+  }
+
+  public String getAppI

[jira] [Work logged] (BEAM-1240) Create RabbitMqIO

2018-07-19 Thread ASF GitHub Bot (JIRA)



 [ 
https://issues.apache.org/jira/browse/BEAM-1240?focusedWorklogId=125190&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-125190
 ]

ASF GitHub Bot logged work on BEAM-1240:


Author: ASF GitHub Bot
Created on: 19/Jul/18 19:03
Start Date: 19/Jul/18 19:03
Worklog Time Spent: 10m 
  Work Description: jkff commented on a change in pull request #1729: 
[BEAM-1240] Create RabbitMqIO
URL: https://github.com/apache/beam/pull/1729#discussion_r203838287
 
 

 ##
 File path: sdks/java/io/rabbitmq/src/test/qpid/etc/passwd
 ##
 @@ -0,0 +1,19 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+guest:guest
 
 Review comment:
   Ping?


This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
 
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org


Issue Time Tracking
---

Worklog Id: (was: 125190)

> Create RabbitMqIO
> -
>
> Key: BEAM-1240
> URL: https://issues.apache.org/jira/browse/BEAM-1240
> Project: Beam
>  Issue Type: New Feature
>  Components: io-ideas
>Reporter: Jean-Baptiste Onofré
>Assignee: Jean-Baptiste Onofré
>Priority: Major
> Fix For: 2.6.0
>
>  Time Spent: 8h 40m
>  Remaining Estimate: 0h
>




--
This message was sent by Atlassian JIRA
(v7.6.3#76005)

[jira] [Work logged] (BEAM-1240) Create RabbitMqIO

2018-07-19 Thread ASF GitHub Bot (JIRA)



 [ 
https://issues.apache.org/jira/browse/BEAM-1240?focusedWorklogId=125194&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-125194
 ]

ASF GitHub Bot logged work on BEAM-1240:


Author: ASF GitHub Bot
Created on: 19/Jul/18 19:03
Start Date: 19/Jul/18 19:03
Worklog Time Spent: 10m 
  Work Description: jkff commented on a change in pull request #1729: 
[BEAM-1240] Create RabbitMqIO
URL: https://github.com/apache/beam/pull/1729#discussion_r203838041
 
 

 ##
 File path: 
sdks/java/io/rabbitmq/src/test/java/org/apache/beam/sdk/io/rabbitmq/RabbitMqIOTest.java
 ##
 @@ -0,0 +1,269 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.io.rabbitmq;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+import com.rabbitmq.client.AMQP;
+import com.rabbitmq.client.Channel;
+import com.rabbitmq.client.Connection;
+import com.rabbitmq.client.ConnectionFactory;
+import com.rabbitmq.client.Consumer;
+import com.rabbitmq.client.DefaultConsumer;
+import com.rabbitmq.client.Envelope;
+import java.io.IOException;
+import java.io.Serializable;
+import java.net.ServerSocket;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.stream.Collectors;
+import java.util.stream.IntStream;
+import org.apache.beam.sdk.testing.PAssert;
+import org.apache.beam.sdk.testing.TestPipeline;
+import org.apache.beam.sdk.transforms.Create;
+import org.apache.beam.sdk.transforms.DoFn;
+import org.apache.beam.sdk.transforms.ParDo;
+import org.apache.beam.sdk.values.PCollection;
+import org.apache.qpid.server.Broker;
+import org.apache.qpid.server.BrokerOptions;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.ClassRule;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/** Test of {@link RabbitMqIO}. */
+@RunWith(JUnit4.class)
+public class RabbitMqIOTest implements Serializable {
+  private static final Logger LOG = 
LoggerFactory.getLogger(RabbitMqIOTest.class);
+
+  private static int port;
+  @ClassRule public static TemporaryFolder temporaryFolder = new 
TemporaryFolder();
+
+  @Rule public transient TestPipeline p = TestPipeline.create();
+
+  private static transient Broker broker;
+
+  @BeforeClass
+  public static void startBroker() throws Exception {
+try (ServerSocket serverSocket = new ServerSocket(0)) {
+  port = serverSocket.getLocalPort();
+}
+
+System.setProperty("derby.stream.error.field", "MyApp.DEV_NULL");
+broker = new Broker();
+BrokerOptions options = new BrokerOptions();
+options.setConfigProperty(BrokerOptions.QPID_AMQP_PORT, 
String.valueOf(port));
+options.setConfigProperty(BrokerOptions.QPID_WORK_DIR, 
temporaryFolder.newFolder().toString());
+options.setConfigProperty(BrokerOptions.QPID_HOME_DIR, "src/test/qpid");
+broker.startup(options);
+  }
+
+  @AfterClass
+  public static void stopBroker() {
+broker.shutdown();
+  }
+
+  @Test
+  public void testReadQueue() throws Exception {
+final int maxNumRecords = 10;
+PCollection raw =
+p.apply(
+RabbitMqIO.read()
+.withUri("amqp://guest:guest@localhost:" + port)
+.withQueue("READ")
+.withMaxNumRecords(maxNumRecords));
+PCollection output = raw.apply(ParDo.of(new ConverterFn()));
+
+List records = generateRecords(maxNumRecords);
+PAssert.that(output).containsInAnyOrder(records);
+
+ConnectionFactory connectionFactory = new ConnectionFactory();
+connectionFactory.setUri("amqp://guest:guest@localhost:" + port);
+Connection connection = connectionFactory.newConnection();
+Channel channel = connection.createChannel();
+channel.queueDeclare("READ", false, false, false, null);
+for (byte[] record : records) {
+  channel.basicPublish("", "READ", null, record);
+}
+
+p.run();
+
+channel.close();
+

[jira] [Work logged] (BEAM-1240) Create RabbitMqIO

2018-07-19 Thread ASF GitHub Bot (JIRA)



 [ 
https://issues.apache.org/jira/browse/BEAM-1240?focusedWorklogId=124873&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-124873
 ]

ASF GitHub Bot logged work on BEAM-1240:


Author: ASF GitHub Bot
Created on: 19/Jul/18 09:17
Start Date: 19/Jul/18 09:17
Worklog Time Spent: 10m 
  Work Description: jbonofre commented on issue #1729: [BEAM-1240] Create 
RabbitMqIO
URL: https://github.com/apache/beam/pull/1729#issuecomment-406211459
 
 
   @echauchot great thanks !


This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
 
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org


Issue Time Tracking
---

Worklog Id: (was: 124873)
Time Spent: 8h  (was: 7h 50m)

> Create RabbitMqIO
> -
>
> Key: BEAM-1240
> URL: https://issues.apache.org/jira/browse/BEAM-1240
> Project: Beam
>  Issue Type: New Feature
>  Components: io-ideas
>Reporter: Jean-Baptiste Onofré
>Assignee: Jean-Baptiste Onofré
>Priority: Major
> Fix For: 2.6.0
>
>  Time Spent: 8h
>  Remaining Estimate: 0h
>




--
This message was sent by Atlassian JIRA
(v7.6.3#76005)

[jira] [Work logged] (BEAM-1240) Create RabbitMqIO

2018-07-18 Thread ASF GitHub Bot (JIRA)



 [ 
https://issues.apache.org/jira/browse/BEAM-1240?focusedWorklogId=124579&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-124579
 ]

ASF GitHub Bot logged work on BEAM-1240:


Author: ASF GitHub Bot
Created on: 18/Jul/18 19:59
Start Date: 18/Jul/18 19:59
Worklog Time Spent: 10m 
  Work Description: jbonofre commented on issue #1729: [BEAM-1240] Create 
RabbitMqIO
URL: https://github.com/apache/beam/pull/1729#issuecomment-406055489
 
 
   Build is now OK. Can someone review ? Thanks !


This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
 
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org


Issue Time Tracking
---

Worklog Id: (was: 124579)
Time Spent: 7h 50m  (was: 7h 40m)

> Create RabbitMqIO
> -
>
> Key: BEAM-1240
> URL: https://issues.apache.org/jira/browse/BEAM-1240
> Project: Beam
>  Issue Type: New Feature
>  Components: io-ideas
>Reporter: Jean-Baptiste Onofré
>Assignee: Jean-Baptiste Onofré
>Priority: Major
> Fix For: 2.6.0
>
>  Time Spent: 7h 50m
>  Remaining Estimate: 0h
>




--
This message was sent by Atlassian JIRA
(v7.6.3#76005)

[jira] [Work logged] (BEAM-1240) Create RabbitMqIO

2018-07-18 Thread ASF GitHub Bot (JIRA)



 [ 
https://issues.apache.org/jira/browse/BEAM-1240?focusedWorklogId=124407&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-124407
 ]

ASF GitHub Bot logged work on BEAM-1240:


Author: ASF GitHub Bot
Created on: 18/Jul/18 12:53
Start Date: 18/Jul/18 12:53
Worklog Time Spent: 10m 
  Work Description: jbonofre commented on issue #1729: [BEAM-1240] Create 
RabbitMqIO
URL: https://github.com/apache/beam/pull/1729#issuecomment-405919886
 
 
   retest this please


This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
 
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org


Issue Time Tracking
---

Worklog Id: (was: 124407)
Time Spent: 7h 40m  (was: 7.5h)

> Create RabbitMqIO
> -
>
> Key: BEAM-1240
> URL: https://issues.apache.org/jira/browse/BEAM-1240
> Project: Beam
>  Issue Type: New Feature
>  Components: io-ideas
>Reporter: Jean-Baptiste Onofré
>Assignee: Jean-Baptiste Onofré
>Priority: Major
> Fix For: 2.6.0
>
>  Time Spent: 7h 40m
>  Remaining Estimate: 0h
>




--
This message was sent by Atlassian JIRA
(v7.6.3#76005)

[jira] [Work logged] (BEAM-1240) Create RabbitMqIO

2018-07-18 Thread ASF GitHub Bot (JIRA)



 [ 
https://issues.apache.org/jira/browse/BEAM-1240?focusedWorklogId=124406&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-124406
 ]

ASF GitHub Bot logged work on BEAM-1240:


Author: ASF GitHub Bot
Created on: 18/Jul/18 12:53
Start Date: 18/Jul/18 12:53
Worklog Time Spent: 10m 
  Work Description: jbonofre commented on issue #1729: [BEAM-1240] Create 
RabbitMqIO
URL: https://github.com/apache/beam/pull/1729#issuecomment-405919851
 
 
   I'm not able to reproduce the issue on my machine. Let me try a new run in 
order for me to investigate.


This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
 
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org


Issue Time Tracking
---

Worklog Id: (was: 124406)
Time Spent: 7.5h  (was: 7h 20m)

> Create RabbitMqIO
> -
>
> Key: BEAM-1240
> URL: https://issues.apache.org/jira/browse/BEAM-1240
> Project: Beam
>  Issue Type: New Feature
>  Components: io-ideas
>Reporter: Jean-Baptiste Onofré
>Assignee: Jean-Baptiste Onofré
>Priority: Major
> Fix For: 2.6.0
>
>  Time Spent: 7.5h
>  Remaining Estimate: 0h
>




--
This message was sent by Atlassian JIRA
(v7.6.3#76005)

[jira] [Work logged] (BEAM-1240) Create RabbitMqIO

2018-07-18 Thread ASF GitHub Bot (JIRA)



 [ 
https://issues.apache.org/jira/browse/BEAM-1240?focusedWorklogId=124360&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-124360
 ]

ASF GitHub Bot logged work on BEAM-1240:


Author: ASF GitHub Bot
Created on: 18/Jul/18 08:43
Start Date: 18/Jul/18 08:43
Worklog Time Spent: 10m 
  Work Description: echauchot commented on issue #1729: [BEAM-1240] Create 
RabbitMqIO
URL: https://github.com/apache/beam/pull/1729#issuecomment-405856898
 
 
   @jbonofre sure, I'll take a look with pleasure !


This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
 
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org


Issue Time Tracking
---

Worklog Id: (was: 124360)
Time Spent: 7h 20m  (was: 7h 10m)

> Create RabbitMqIO
> -
>
> Key: BEAM-1240
> URL: https://issues.apache.org/jira/browse/BEAM-1240
> Project: Beam
>  Issue Type: New Feature
>  Components: io-ideas
>Reporter: Jean-Baptiste Onofré
>Assignee: Jean-Baptiste Onofré
>Priority: Major
> Fix For: 2.6.0
>
>  Time Spent: 7h 20m
>  Remaining Estimate: 0h
>




--
This message was sent by Atlassian JIRA
(v7.6.3#76005)

[jira] [Work logged] (BEAM-1240) Create RabbitMqIO

2018-07-18 Thread ASF GitHub Bot (JIRA)



 [ 
https://issues.apache.org/jira/browse/BEAM-1240?focusedWorklogId=124355&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-124355
 ]

ASF GitHub Bot logged work on BEAM-1240:


Author: ASF GitHub Bot
Created on: 18/Jul/18 08:35
Start Date: 18/Jul/18 08:35
Worklog Time Spent: 10m 
  Work Description: jbonofre commented on issue #1729: [BEAM-1240] Create 
RabbitMqIO
URL: https://github.com/apache/beam/pull/1729#issuecomment-405854542
 
 
   Checking the build issue.


This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
 
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org


Issue Time Tracking
---

Worklog Id: (was: 124355)
Time Spent: 7h 10m  (was: 7h)

> Create RabbitMqIO
> -
>
> Key: BEAM-1240
> URL: https://issues.apache.org/jira/browse/BEAM-1240
> Project: Beam
>  Issue Type: New Feature
>  Components: io-ideas
>Reporter: Jean-Baptiste Onofré
>Assignee: Jean-Baptiste Onofré
>Priority: Major
> Fix For: 2.6.0
>
>  Time Spent: 7h 10m
>  Remaining Estimate: 0h
>




--
This message was sent by Atlassian JIRA
(v7.6.3#76005)

[jira] [Work logged] (BEAM-1240) Create RabbitMqIO

2018-07-17 Thread ASF GitHub Bot (JIRA)



 [ 
https://issues.apache.org/jira/browse/BEAM-1240?focusedWorklogId=124159&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-124159
 ]

ASF GitHub Bot logged work on BEAM-1240:


Author: ASF GitHub Bot
Created on: 17/Jul/18 18:17
Start Date: 17/Jul/18 18:17
Worklog Time Spent: 10m 
  Work Description: pabloem commented on issue #1729: [BEAM-1240] Create 
RabbitMqIO
URL: https://github.com/apache/beam/pull/1729#issuecomment-405677987
 
 
   There was this failure: 
https://scans.gradle.com/s/ydr6b3ul7phag/tests/zwsoh7b2afk5i-cuey54y3c4yu4


This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
 
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org


Issue Time Tracking
---

Worklog Id: (was: 124159)
Time Spent: 7h  (was: 6h 50m)

> Create RabbitMqIO
> -
>
> Key: BEAM-1240
> URL: https://issues.apache.org/jira/browse/BEAM-1240
> Project: Beam
>  Issue Type: New Feature
>  Components: io-ideas
>Reporter: Jean-Baptiste Onofré
>Assignee: Jean-Baptiste Onofré
>Priority: Major
> Fix For: 2.6.0
>
>  Time Spent: 7h
>  Remaining Estimate: 0h
>




--
This message was sent by Atlassian JIRA
(v7.6.3#76005)

[jira] [Work logged] (BEAM-1240) Create RabbitMqIO

2018-07-17 Thread ASF GitHub Bot (JIRA)



 [ 
https://issues.apache.org/jira/browse/BEAM-1240?focusedWorklogId=124054&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-124054
 ]

ASF GitHub Bot logged work on BEAM-1240:


Author: ASF GitHub Bot
Created on: 17/Jul/18 12:26
Start Date: 17/Jul/18 12:26
Worklog Time Spent: 10m 
  Work Description: jbonofre commented on issue #1729: [BEAM-1240] Create 
RabbitMqIO
URL: https://github.com/apache/beam/pull/1729#issuecomment-405564221
 
 
   I updated all, addressing pending comments. @echauchot do you mind to take a 
look ? Thanks !


This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
 
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org


Issue Time Tracking
---

Worklog Id: (was: 124054)
Time Spent: 6h 50m  (was: 6h 40m)

> Create RabbitMqIO
> -
>
> Key: BEAM-1240
> URL: https://issues.apache.org/jira/browse/BEAM-1240
> Project: Beam
>  Issue Type: New Feature
>  Components: io-ideas
>Reporter: Jean-Baptiste Onofré
>Assignee: Jean-Baptiste Onofré
>Priority: Major
> Fix For: 2.6.0
>
>  Time Spent: 6h 50m
>  Remaining Estimate: 0h
>




--
This message was sent by Atlassian JIRA
(v7.6.3#76005)

[jira] [Work logged] (BEAM-1240) Create RabbitMqIO

2018-07-05 Thread ASF GitHub Bot (JIRA)



 [ 
https://issues.apache.org/jira/browse/BEAM-1240?focusedWorklogId=119481&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-119481
 ]

ASF GitHub Bot logged work on BEAM-1240:


Author: ASF GitHub Bot
Created on: 05/Jul/18 17:06
Start Date: 05/Jul/18 17:06
Worklog Time Spent: 10m 
  Work Description: jbonofre commented on issue #1729: [BEAM-1240] Create 
RabbitMqIO
URL: https://github.com/apache/beam/pull/1729#issuecomment-402790333
 
 
   Resuming my work on this one.


This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
 
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org


Issue Time Tracking
---

Worklog Id: (was: 119481)
Time Spent: 6h 40m  (was: 6.5h)

> Create RabbitMqIO
> -
>
> Key: BEAM-1240
> URL: https://issues.apache.org/jira/browse/BEAM-1240
> Project: Beam
>  Issue Type: New Feature
>  Components: io-ideas
>Reporter: Jean-Baptiste Onofré
>Assignee: Jean-Baptiste Onofré
>Priority: Major
>  Time Spent: 6h 40m
>  Remaining Estimate: 0h
>




--
This message was sent by Atlassian JIRA
(v7.6.3#76005)

[jira] [Work logged] (BEAM-1240) Create RabbitMqIO

2018-05-30 Thread ASF GitHub Bot (JIRA)



 [ 
https://issues.apache.org/jira/browse/BEAM-1240?focusedWorklogId=107368&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-107368
 ]

ASF GitHub Bot logged work on BEAM-1240:


Author: ASF GitHub Bot
Created on: 30/May/18 20:42
Start Date: 30/May/18 20:42
Worklog Time Spent: 10m 
  Work Description: jkff commented on a change in pull request #1729: 
[BEAM-1240] Create RabbitMqIO
URL: https://github.com/apache/beam/pull/1729#discussion_r191914617
 
 

 ##
 File path: 
sdks/java/io/rabbitmq/src/main/java/org/apache/beam/sdk/io/rabbitmq/RabbitMqIO.java
 ##
 @@ -0,0 +1,530 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.io.rabbitmq;
+
+import static com.google.common.base.Preconditions.checkArgument;
+
+import com.google.auto.value.AutoValue;
+import com.rabbitmq.client.Channel;
+import com.rabbitmq.client.Connection;
+import com.rabbitmq.client.ConnectionFactory;
+import com.rabbitmq.client.QueueingConsumer;
+import java.io.IOException;
+import java.io.Serializable;
+import java.net.URISyntaxException;
+import java.nio.charset.StandardCharsets;
+import java.security.KeyManagementException;
+import java.security.NoSuchAlgorithmException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.NoSuchElementException;
+import javax.annotation.Nullable;
+import org.apache.beam.sdk.annotations.Experimental;
+import org.apache.beam.sdk.coders.Coder;
+import org.apache.beam.sdk.coders.SerializableCoder;
+import org.apache.beam.sdk.io.UnboundedSource;
+import org.apache.beam.sdk.options.PipelineOptions;
+import org.apache.beam.sdk.transforms.DoFn;
+import org.apache.beam.sdk.transforms.PTransform;
+import org.apache.beam.sdk.transforms.ParDo;
+import org.apache.beam.sdk.values.PBegin;
+import org.apache.beam.sdk.values.PCollection;
+import org.apache.beam.sdk.values.PDone;
+import org.joda.time.Duration;
+import org.joda.time.Instant;
+
+/**
+ * A IO to publish or consume messages with a RabbitMQ broker.
+ *
+ * Consuming messages from RabbitMQ server
+ *
+ * {@link RabbitMqIO} {@link Read} returns an unbounded {@link PCollection} 
containing RabbitMQ
+ * messages body (as {@code byte[]}).
+ *
+ * To configure a RabbitMQ source, you have to provide a RabbitMQ {@code 
URI} to connect
+ * to a RabbitMQ broker. The following example illustrates various options for 
configuring the
+ * source:
+ *
+ * {@code
+ *
+ *  pipeline.apply(
+ *
RabbitMqIO.read().withUri("amqp://user:password@localhost:5672").withQueue("QUEUE")
+ *
+ * }
+ *
+ * Publishing messages to RabbitMQ server
+ *
+ * {@link RabbitMqIO} {@link Write} can send {@code byte[]} to a RabbitMQ 
server queue.
+ *
+ * As for the {@link Read}, the {@link Write} is configured with a RabbitMQ 
URI.
+ *
+ * For instance:
+ *
+ * {@code
+ *
+ *  pipeline
+ *.apply(...) // provide PCollection
+ *
.apply(RabbitMqIO.write().withUri("amqp://user:password@localhost:5672").withQueue("QUEUE"));
+ *
+ * }
+ */
+@Experimental(Experimental.Kind.SOURCE_SINK)
+public class RabbitMqIO {
+  public static Read read() {
+return new AutoValue_RabbitMqIO_Read.Builder().setQueueDeclare(false)
+
.setMaxReadTime(null).setMaxNumRecords(Long.MAX_VALUE).setUseCorrelationId(false).build();
+  }
+
+  public static Write write() {
+return new 
AutoValue_RabbitMqIO_Write.Builder().setExchangeDeclare(false).build();
+  }
+
+  private RabbitMqIO() {
+  }
+
+  private static ConnectionFactory createConnectionFactory(String uri) throws 
URISyntaxException,
+  NoSuchAlgorithmException, KeyManagementException {
+ConnectionFactory connectionFactory = new ConnectionFactory();
+connectionFactory.setUri(uri);
+connectionFactory.setAutomaticRecoveryEnabled(true);
+connectionFactory.setConnectionTimeout(6);
+connectionFactory.setNetworkRecoveryInterval(5000);
+connectionFactory.setRequestedHeartbeat(60);
+connectionFactory.setTopologyRecoveryEnabled(true);
+connectionFactory.setRequestedChannelMax(0);
+connectionFactory.setRequestedFrameMax(0);
+return connectionFactory;
+  }
+
+  /**

[jira] [Work logged] (BEAM-1240) Create RabbitMqIO

2018-05-30 Thread ASF GitHub Bot (JIRA)



 [ 
https://issues.apache.org/jira/browse/BEAM-1240?focusedWorklogId=107365&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-107365
 ]

ASF GitHub Bot logged work on BEAM-1240:


Author: ASF GitHub Bot
Created on: 30/May/18 20:42
Start Date: 30/May/18 20:42
Worklog Time Spent: 10m 
  Work Description: jkff commented on a change in pull request #1729: 
[BEAM-1240] Create RabbitMqIO
URL: https://github.com/apache/beam/pull/1729#discussion_r191915651
 
 

 ##
 File path: 
sdks/java/io/rabbitmq/src/main/java/org/apache/beam/sdk/io/rabbitmq/RabbitMqIO.java
 ##
 @@ -0,0 +1,530 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.io.rabbitmq;
+
+import static com.google.common.base.Preconditions.checkArgument;
+
+import com.google.auto.value.AutoValue;
+import com.rabbitmq.client.Channel;
+import com.rabbitmq.client.Connection;
+import com.rabbitmq.client.ConnectionFactory;
+import com.rabbitmq.client.QueueingConsumer;
+import java.io.IOException;
+import java.io.Serializable;
+import java.net.URISyntaxException;
+import java.nio.charset.StandardCharsets;
+import java.security.KeyManagementException;
+import java.security.NoSuchAlgorithmException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.NoSuchElementException;
+import javax.annotation.Nullable;
+import org.apache.beam.sdk.annotations.Experimental;
+import org.apache.beam.sdk.coders.Coder;
+import org.apache.beam.sdk.coders.SerializableCoder;
+import org.apache.beam.sdk.io.UnboundedSource;
+import org.apache.beam.sdk.options.PipelineOptions;
+import org.apache.beam.sdk.transforms.DoFn;
+import org.apache.beam.sdk.transforms.PTransform;
+import org.apache.beam.sdk.transforms.ParDo;
+import org.apache.beam.sdk.values.PBegin;
+import org.apache.beam.sdk.values.PCollection;
+import org.apache.beam.sdk.values.PDone;
+import org.joda.time.Duration;
+import org.joda.time.Instant;
+
+/**
+ * A IO to publish or consume messages with a RabbitMQ broker.
+ *
+ * Consuming messages from RabbitMQ server
+ *
+ * {@link RabbitMqIO} {@link Read} returns an unbounded {@link PCollection} 
containing RabbitMQ
+ * messages body (as {@code byte[]}).
+ *
+ * To configure a RabbitMQ source, you have to provide a RabbitMQ {@code 
URI} to connect
+ * to a RabbitMQ broker. The following example illustrates various options for 
configuring the
+ * source:
+ *
+ * {@code
+ *
+ *  pipeline.apply(
+ *
RabbitMqIO.read().withUri("amqp://user:password@localhost:5672").withQueue("QUEUE")
+ *
+ * }
+ *
+ * Publishing messages to RabbitMQ server
+ *
+ * {@link RabbitMqIO} {@link Write} can send {@code byte[]} to a RabbitMQ 
server queue.
+ *
+ * As for the {@link Read}, the {@link Write} is configured with a RabbitMQ 
URI.
+ *
+ * For instance:
+ *
+ * {@code
+ *
+ *  pipeline
+ *.apply(...) // provide PCollection
+ *
.apply(RabbitMqIO.write().withUri("amqp://user:password@localhost:5672").withQueue("QUEUE"));
+ *
+ * }
+ */
+@Experimental(Experimental.Kind.SOURCE_SINK)
+public class RabbitMqIO {
+  public static Read read() {
+return new AutoValue_RabbitMqIO_Read.Builder().setQueueDeclare(false)
+
.setMaxReadTime(null).setMaxNumRecords(Long.MAX_VALUE).setUseCorrelationId(false).build();
+  }
+
+  public static Write write() {
+return new 
AutoValue_RabbitMqIO_Write.Builder().setExchangeDeclare(false).build();
+  }
+
+  private RabbitMqIO() {
+  }
+
+  private static ConnectionFactory createConnectionFactory(String uri) throws 
URISyntaxException,
+  NoSuchAlgorithmException, KeyManagementException {
+ConnectionFactory connectionFactory = new ConnectionFactory();
+connectionFactory.setUri(uri);
+connectionFactory.setAutomaticRecoveryEnabled(true);
+connectionFactory.setConnectionTimeout(6);
+connectionFactory.setNetworkRecoveryInterval(5000);
+connectionFactory.setRequestedHeartbeat(60);
+connectionFactory.setTopologyRecoveryEnabled(true);
+connectionFactory.setRequestedChannelMax(0);
+connectionFactory.setRequestedFrameMax(0);
+return connectionFactory;
+  }
+
+  /**

[jira] [Work logged] (BEAM-1240) Create RabbitMqIO

2018-05-30 Thread ASF GitHub Bot (JIRA)



 [ 
https://issues.apache.org/jira/browse/BEAM-1240?focusedWorklogId=107361&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-107361
 ]

ASF GitHub Bot logged work on BEAM-1240:


Author: ASF GitHub Bot
Created on: 30/May/18 20:42
Start Date: 30/May/18 20:42
Worklog Time Spent: 10m 
  Work Description: jkff commented on a change in pull request #1729: 
[BEAM-1240] Create RabbitMqIO
URL: https://github.com/apache/beam/pull/1729#discussion_r191910644
 
 

 ##
 File path: 
sdks/java/io/rabbitmq/src/main/java/org/apache/beam/sdk/io/rabbitmq/RabbitMqMessage.java
 ##
 @@ -0,0 +1,215 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.io.rabbitmq;
+
+import com.rabbitmq.client.AMQP;
+import java.io.Serializable;
+import java.util.Arrays;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Objects;
+
+/** It contains the message payload, and additional metadata like routing key 
or attributes. */
+public class RabbitMqMessage implements Serializable {
+  private final String routingKey;
+  private final byte[] body;
+  private final String contentType;
+  private final String contentEncoding;
+  private final Map headers;
+  private final Integer deliveryMode;
+  private final Integer priority;
+  private final String correlationId;
+  private final String replyTo;
+  private final String expiration;
+  private final String messageId;
+  private final Date timestamp;
+  private final String type;
+  private final String userId;
+  private final String appId;
+  private final String clusterId;
+
+  public RabbitMqMessage(byte[] body) {
+this.body = body;
+routingKey = "";
+contentType = null;
+contentEncoding = null;
+headers = new HashMap<>();
+deliveryMode = 1;
+priority = 1;
+correlationId = null;
+replyTo = null;
+expiration = null;
+messageId = null;
+timestamp = new Date();
+type = null;
+userId = null;
+appId = null;
+clusterId = null;
+  }
+
+  public RabbitMqMessage(
+  String routingKey,
+  byte[] body,
+  String contentType,
+  String contentEncoding,
+  Map headers,
+  Integer deliveryMode,
+  Integer priority,
+  String correlationId,
+  String replyTo,
+  String expiration,
+  String messageId,
+  Date timestamp,
+  String type,
+  String userId,
+  String appId,
+  String clusterId) {
+this.routingKey = routingKey;
+this.body = body;
+this.contentType = contentType;
+this.contentEncoding = contentEncoding;
+this.headers = headers;
+this.deliveryMode = deliveryMode;
+this.priority = priority;
+this.correlationId = correlationId;
+this.replyTo = replyTo;
+this.expiration = expiration;
+this.messageId = messageId;
+this.timestamp = timestamp;
+this.type = type;
+this.userId = userId;
+this.appId = appId;
+this.clusterId = clusterId;
+  }
+
+  public String getRoutingKey() {
+return routingKey;
+  }
+
+  public byte[] getBody() {
+return body;
+  }
+
+  public String getContentType() {
+return contentType;
+  }
+
+  public String getContentEncoding() {
+return contentEncoding;
+  }
+
+  public Map getHeaders() {
+return headers;
+  }
+
+  public Integer getDeliveryMode() {
+return deliveryMode;
+  }
+
+  public Integer getPriority() {
+return priority;
+  }
+
+  public String getCorrelationId() {
+return correlationId;
+  }
+
+  public String getReplyTo() {
+return replyTo;
+  }
+
+  public String getExpiration() {
+return expiration;
+  }
+
+  public String getMessageId() {
+return messageId;
+  }
+
+  public Date getTimestamp() {
+return timestamp;
+  }
+
+  public String getType() {
+return type;
+  }
+
+  public String getUserId() {
+return userId;
+  }
+
+  public String getAppId() {
+return appId;
+  }
+
+  public String getClusterId() {
+return clusterId;
+  }
+
+  public AMQP.BasicProperties createProperties() {
+return new AMQP.BasicProperties()
+.builder()
+.con

[jira] [Work logged] (BEAM-1240) Create RabbitMqIO

2018-05-30 Thread ASF GitHub Bot (JIRA)



 [ 
https://issues.apache.org/jira/browse/BEAM-1240?focusedWorklogId=107370&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-107370
 ]

ASF GitHub Bot logged work on BEAM-1240:


Author: ASF GitHub Bot
Created on: 30/May/18 20:42
Start Date: 30/May/18 20:42
Worklog Time Spent: 10m 
  Work Description: jkff commented on a change in pull request #1729: 
[BEAM-1240] Create RabbitMqIO
URL: https://github.com/apache/beam/pull/1729#discussion_r191913392
 
 

 ##
 File path: 
sdks/java/io/rabbitmq/src/main/java/org/apache/beam/sdk/io/rabbitmq/RabbitMqIO.java
 ##
 @@ -0,0 +1,530 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.io.rabbitmq;
+
+import static com.google.common.base.Preconditions.checkArgument;
+
+import com.google.auto.value.AutoValue;
+import com.rabbitmq.client.Channel;
+import com.rabbitmq.client.Connection;
+import com.rabbitmq.client.ConnectionFactory;
+import com.rabbitmq.client.QueueingConsumer;
+import java.io.IOException;
+import java.io.Serializable;
+import java.net.URISyntaxException;
+import java.nio.charset.StandardCharsets;
+import java.security.KeyManagementException;
+import java.security.NoSuchAlgorithmException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.NoSuchElementException;
+import javax.annotation.Nullable;
+import org.apache.beam.sdk.annotations.Experimental;
+import org.apache.beam.sdk.coders.Coder;
+import org.apache.beam.sdk.coders.SerializableCoder;
+import org.apache.beam.sdk.io.UnboundedSource;
+import org.apache.beam.sdk.options.PipelineOptions;
+import org.apache.beam.sdk.transforms.DoFn;
+import org.apache.beam.sdk.transforms.PTransform;
+import org.apache.beam.sdk.transforms.ParDo;
+import org.apache.beam.sdk.values.PBegin;
+import org.apache.beam.sdk.values.PCollection;
+import org.apache.beam.sdk.values.PDone;
+import org.joda.time.Duration;
+import org.joda.time.Instant;
+
+/**
+ * A IO to publish or consume messages with a RabbitMQ broker.
+ *
+ * Consuming messages from RabbitMQ server
+ *
+ * {@link RabbitMqIO} {@link Read} returns an unbounded {@link PCollection} 
containing RabbitMQ
+ * messages body (as {@code byte[]}).
+ *
+ * To configure a RabbitMQ source, you have to provide a RabbitMQ {@code 
URI} to connect
+ * to a RabbitMQ broker. The following example illustrates various options for 
configuring the
+ * source:
+ *
+ * {@code
+ *
+ *  pipeline.apply(
+ *
RabbitMqIO.read().withUri("amqp://user:password@localhost:5672").withQueue("QUEUE")
+ *
+ * }
+ *
+ * Publishing messages to RabbitMQ server
+ *
+ * {@link RabbitMqIO} {@link Write} can send {@code byte[]} to a RabbitMQ 
server queue.
+ *
+ * As for the {@link Read}, the {@link Write} is configured with a RabbitMQ 
URI.
+ *
+ * For instance:
+ *
+ * {@code
+ *
+ *  pipeline
+ *.apply(...) // provide PCollection
+ *
.apply(RabbitMqIO.write().withUri("amqp://user:password@localhost:5672").withQueue("QUEUE"));
+ *
+ * }
+ */
+@Experimental(Experimental.Kind.SOURCE_SINK)
+public class RabbitMqIO {
+  public static Read read() {
+return new AutoValue_RabbitMqIO_Read.Builder().setQueueDeclare(false)
+
.setMaxReadTime(null).setMaxNumRecords(Long.MAX_VALUE).setUseCorrelationId(false).build();
+  }
+
+  public static Write write() {
+return new 
AutoValue_RabbitMqIO_Write.Builder().setExchangeDeclare(false).build();
+  }
+
+  private RabbitMqIO() {
+  }
+
+  private static ConnectionFactory createConnectionFactory(String uri) throws 
URISyntaxException,
+  NoSuchAlgorithmException, KeyManagementException {
+ConnectionFactory connectionFactory = new ConnectionFactory();
+connectionFactory.setUri(uri);
+connectionFactory.setAutomaticRecoveryEnabled(true);
+connectionFactory.setConnectionTimeout(6);
+connectionFactory.setNetworkRecoveryInterval(5000);
+connectionFactory.setRequestedHeartbeat(60);
+connectionFactory.setTopologyRecoveryEnabled(true);
+connectionFactory.setRequestedChannelMax(0);
+connectionFactory.setRequestedFrameMax(0);
+return connectionFactory;
+  }
+
+  /**

[jira] [Work logged] (BEAM-1240) Create RabbitMqIO

2018-05-30 Thread ASF GitHub Bot (JIRA)



 [ 
https://issues.apache.org/jira/browse/BEAM-1240?focusedWorklogId=107362&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-107362
 ]

ASF GitHub Bot logged work on BEAM-1240:


Author: ASF GitHub Bot
Created on: 30/May/18 20:42
Start Date: 30/May/18 20:42
Worklog Time Spent: 10m 
  Work Description: jkff commented on a change in pull request #1729: 
[BEAM-1240] Create RabbitMqIO
URL: https://github.com/apache/beam/pull/1729#discussion_r191910965
 
 

 ##
 File path: 
sdks/java/io/rabbitmq/src/main/java/org/apache/beam/sdk/io/rabbitmq/RabbitMqMessage.java
 ##
 @@ -0,0 +1,215 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.io.rabbitmq;
+
+import com.rabbitmq.client.AMQP;
+import java.io.Serializable;
+import java.util.Arrays;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Objects;
+
+/** It contains the message payload, and additional metadata like routing key 
or attributes. */
+public class RabbitMqMessage implements Serializable {
+  private final String routingKey;
+  private final byte[] body;
+  private final String contentType;
+  private final String contentEncoding;
+  private final Map headers;
+  private final Integer deliveryMode;
+  private final Integer priority;
+  private final String correlationId;
+  private final String replyTo;
+  private final String expiration;
+  private final String messageId;
+  private final Date timestamp;
+  private final String type;
+  private final String userId;
+  private final String appId;
+  private final String clusterId;
+
+  public RabbitMqMessage(byte[] body) {
+this.body = body;
+routingKey = "";
+contentType = null;
+contentEncoding = null;
+headers = new HashMap<>();
+deliveryMode = 1;
+priority = 1;
+correlationId = null;
+replyTo = null;
+expiration = null;
+messageId = null;
 
 Review comment:
   Per implementation of equals() below it seems that two default-constructed 
messages with the same body will be non-equal. Is this expected?


This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
 
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org


Issue Time Tracking
---

Worklog Id: (was: 107362)
Time Spent: 5h 20m  (was: 5h 10m)

> Create RabbitMqIO
> -
>
> Key: BEAM-1240
> URL: https://issues.apache.org/jira/browse/BEAM-1240
> Project: Beam
>  Issue Type: New Feature
>  Components: io-ideas
>Reporter: Jean-Baptiste Onofré
>Assignee: Jean-Baptiste Onofré
>Priority: Major
>  Time Spent: 5h 20m
>  Remaining Estimate: 0h
>




--
This message was sent by Atlassian JIRA
(v7.6.3#76005)

[jira] [Work logged] (BEAM-1240) Create RabbitMqIO

2018-05-30 Thread ASF GitHub Bot (JIRA)



 [ 
https://issues.apache.org/jira/browse/BEAM-1240?focusedWorklogId=107369&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-107369
 ]

ASF GitHub Bot logged work on BEAM-1240:


Author: ASF GitHub Bot
Created on: 30/May/18 20:42
Start Date: 30/May/18 20:42
Worklog Time Spent: 10m 
  Work Description: jkff commented on a change in pull request #1729: 
[BEAM-1240] Create RabbitMqIO
URL: https://github.com/apache/beam/pull/1729#discussion_r191914857
 
 

 ##
 File path: 
sdks/java/io/rabbitmq/src/main/java/org/apache/beam/sdk/io/rabbitmq/RabbitMqIO.java
 ##
 @@ -0,0 +1,530 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.io.rabbitmq;
+
+import static com.google.common.base.Preconditions.checkArgument;
+
+import com.google.auto.value.AutoValue;
+import com.rabbitmq.client.Channel;
+import com.rabbitmq.client.Connection;
+import com.rabbitmq.client.ConnectionFactory;
+import com.rabbitmq.client.QueueingConsumer;
+import java.io.IOException;
+import java.io.Serializable;
+import java.net.URISyntaxException;
+import java.nio.charset.StandardCharsets;
+import java.security.KeyManagementException;
+import java.security.NoSuchAlgorithmException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.NoSuchElementException;
+import javax.annotation.Nullable;
+import org.apache.beam.sdk.annotations.Experimental;
+import org.apache.beam.sdk.coders.Coder;
+import org.apache.beam.sdk.coders.SerializableCoder;
+import org.apache.beam.sdk.io.UnboundedSource;
+import org.apache.beam.sdk.options.PipelineOptions;
+import org.apache.beam.sdk.transforms.DoFn;
+import org.apache.beam.sdk.transforms.PTransform;
+import org.apache.beam.sdk.transforms.ParDo;
+import org.apache.beam.sdk.values.PBegin;
+import org.apache.beam.sdk.values.PCollection;
+import org.apache.beam.sdk.values.PDone;
+import org.joda.time.Duration;
+import org.joda.time.Instant;
+
+/**
+ * A IO to publish or consume messages with a RabbitMQ broker.
+ *
+ * Consuming messages from RabbitMQ server
+ *
+ * {@link RabbitMqIO} {@link Read} returns an unbounded {@link PCollection} 
containing RabbitMQ
+ * messages body (as {@code byte[]}).
+ *
+ * To configure a RabbitMQ source, you have to provide a RabbitMQ {@code 
URI} to connect
+ * to a RabbitMQ broker. The following example illustrates various options for 
configuring the
+ * source:
+ *
+ * {@code
+ *
+ *  pipeline.apply(
+ *
RabbitMqIO.read().withUri("amqp://user:password@localhost:5672").withQueue("QUEUE")
+ *
+ * }
+ *
+ * Publishing messages to RabbitMQ server
+ *
+ * {@link RabbitMqIO} {@link Write} can send {@code byte[]} to a RabbitMQ 
server queue.
+ *
+ * As for the {@link Read}, the {@link Write} is configured with a RabbitMQ 
URI.
+ *
+ * For instance:
+ *
+ * {@code
+ *
+ *  pipeline
+ *.apply(...) // provide PCollection
+ *
.apply(RabbitMqIO.write().withUri("amqp://user:password@localhost:5672").withQueue("QUEUE"));
+ *
+ * }
+ */
+@Experimental(Experimental.Kind.SOURCE_SINK)
+public class RabbitMqIO {
+  public static Read read() {
+return new AutoValue_RabbitMqIO_Read.Builder().setQueueDeclare(false)
+
.setMaxReadTime(null).setMaxNumRecords(Long.MAX_VALUE).setUseCorrelationId(false).build();
+  }
+
+  public static Write write() {
+return new 
AutoValue_RabbitMqIO_Write.Builder().setExchangeDeclare(false).build();
+  }
+
+  private RabbitMqIO() {
+  }
+
+  private static ConnectionFactory createConnectionFactory(String uri) throws 
URISyntaxException,
+  NoSuchAlgorithmException, KeyManagementException {
+ConnectionFactory connectionFactory = new ConnectionFactory();
+connectionFactory.setUri(uri);
+connectionFactory.setAutomaticRecoveryEnabled(true);
+connectionFactory.setConnectionTimeout(6);
+connectionFactory.setNetworkRecoveryInterval(5000);
+connectionFactory.setRequestedHeartbeat(60);
+connectionFactory.setTopologyRecoveryEnabled(true);
+connectionFactory.setRequestedChannelMax(0);
+connectionFactory.setRequestedFrameMax(0);
+return connectionFactory;
+  }
+
+  /**

[jira] [Work logged] (BEAM-1240) Create RabbitMqIO

2018-05-30 Thread ASF GitHub Bot (JIRA)



 [ 
https://issues.apache.org/jira/browse/BEAM-1240?focusedWorklogId=107363&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-107363
 ]

ASF GitHub Bot logged work on BEAM-1240:


Author: ASF GitHub Bot
Created on: 30/May/18 20:42
Start Date: 30/May/18 20:42
Worklog Time Spent: 10m 
  Work Description: jkff commented on a change in pull request #1729: 
[BEAM-1240] Create RabbitMqIO
URL: https://github.com/apache/beam/pull/1729#discussion_r191912340
 
 

 ##
 File path: 
sdks/java/io/rabbitmq/src/main/java/org/apache/beam/sdk/io/rabbitmq/RabbitMqMessage.java
 ##
 @@ -0,0 +1,215 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.io.rabbitmq;
+
+import com.rabbitmq.client.AMQP;
+import java.io.Serializable;
+import java.util.Arrays;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Objects;
+
+/** It contains the message payload, and additional metadata like routing key 
or attributes. */
+public class RabbitMqMessage implements Serializable {
 
 Review comment:
   This class looks so bad boilerplate-wise that I think it's worth adding an 
explanatory note. AFAICT the reason is that AMQP.BasicProperties provides no 
public APIs to convert it into anything serializable except by invoking each 
individual method.


This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
 
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org


Issue Time Tracking
---

Worklog Id: (was: 107363)

> Create RabbitMqIO
> -
>
> Key: BEAM-1240
> URL: https://issues.apache.org/jira/browse/BEAM-1240
> Project: Beam
>  Issue Type: New Feature
>  Components: io-ideas
>Reporter: Jean-Baptiste Onofré
>Assignee: Jean-Baptiste Onofré
>Priority: Major
>  Time Spent: 5h 20m
>  Remaining Estimate: 0h
>




--
This message was sent by Atlassian JIRA
(v7.6.3#76005)

[jira] [Work logged] (BEAM-1240) Create RabbitMqIO

2018-05-30 Thread ASF GitHub Bot (JIRA)



 [ 
https://issues.apache.org/jira/browse/BEAM-1240?focusedWorklogId=107359&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-107359
 ]

ASF GitHub Bot logged work on BEAM-1240:


Author: ASF GitHub Bot
Created on: 30/May/18 20:42
Start Date: 30/May/18 20:42
Worklog Time Spent: 10m 
  Work Description: jkff commented on a change in pull request #1729: 
[BEAM-1240] Create RabbitMqIO
URL: https://github.com/apache/beam/pull/1729#discussion_r191914088
 
 

 ##
 File path: 
sdks/java/io/rabbitmq/src/main/java/org/apache/beam/sdk/io/rabbitmq/RabbitMqIO.java
 ##
 @@ -0,0 +1,530 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.io.rabbitmq;
+
+import static com.google.common.base.Preconditions.checkArgument;
+
+import com.google.auto.value.AutoValue;
+import com.rabbitmq.client.Channel;
+import com.rabbitmq.client.Connection;
+import com.rabbitmq.client.ConnectionFactory;
+import com.rabbitmq.client.QueueingConsumer;
+import java.io.IOException;
+import java.io.Serializable;
+import java.net.URISyntaxException;
+import java.nio.charset.StandardCharsets;
+import java.security.KeyManagementException;
+import java.security.NoSuchAlgorithmException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.NoSuchElementException;
+import javax.annotation.Nullable;
+import org.apache.beam.sdk.annotations.Experimental;
+import org.apache.beam.sdk.coders.Coder;
+import org.apache.beam.sdk.coders.SerializableCoder;
+import org.apache.beam.sdk.io.UnboundedSource;
+import org.apache.beam.sdk.options.PipelineOptions;
+import org.apache.beam.sdk.transforms.DoFn;
+import org.apache.beam.sdk.transforms.PTransform;
+import org.apache.beam.sdk.transforms.ParDo;
+import org.apache.beam.sdk.values.PBegin;
+import org.apache.beam.sdk.values.PCollection;
+import org.apache.beam.sdk.values.PDone;
+import org.joda.time.Duration;
+import org.joda.time.Instant;
+
+/**
+ * A IO to publish or consume messages with a RabbitMQ broker.
+ *
+ * Consuming messages from RabbitMQ server
+ *
+ * {@link RabbitMqIO} {@link Read} returns an unbounded {@link PCollection} 
containing RabbitMQ
+ * messages body (as {@code byte[]}).
+ *
+ * To configure a RabbitMQ source, you have to provide a RabbitMQ {@code 
URI} to connect
+ * to a RabbitMQ broker. The following example illustrates various options for 
configuring the
+ * source:
+ *
+ * {@code
+ *
+ *  pipeline.apply(
+ *
RabbitMqIO.read().withUri("amqp://user:password@localhost:5672").withQueue("QUEUE")
+ *
+ * }
+ *
+ * Publishing messages to RabbitMQ server
+ *
+ * {@link RabbitMqIO} {@link Write} can send {@code byte[]} to a RabbitMQ 
server queue.
+ *
+ * As for the {@link Read}, the {@link Write} is configured with a RabbitMQ 
URI.
+ *
+ * For instance:
+ *
+ * {@code
+ *
+ *  pipeline
+ *.apply(...) // provide PCollection
+ *
.apply(RabbitMqIO.write().withUri("amqp://user:password@localhost:5672").withQueue("QUEUE"));
+ *
+ * }
+ */
+@Experimental(Experimental.Kind.SOURCE_SINK)
+public class RabbitMqIO {
+  public static Read read() {
+return new AutoValue_RabbitMqIO_Read.Builder().setQueueDeclare(false)
+
.setMaxReadTime(null).setMaxNumRecords(Long.MAX_VALUE).setUseCorrelationId(false).build();
+  }
+
+  public static Write write() {
+return new 
AutoValue_RabbitMqIO_Write.Builder().setExchangeDeclare(false).build();
+  }
+
+  private RabbitMqIO() {
+  }
+
+  private static ConnectionFactory createConnectionFactory(String uri) throws 
URISyntaxException,
+  NoSuchAlgorithmException, KeyManagementException {
+ConnectionFactory connectionFactory = new ConnectionFactory();
+connectionFactory.setUri(uri);
+connectionFactory.setAutomaticRecoveryEnabled(true);
+connectionFactory.setConnectionTimeout(6);
+connectionFactory.setNetworkRecoveryInterval(5000);
+connectionFactory.setRequestedHeartbeat(60);
+connectionFactory.setTopologyRecoveryEnabled(true);
+connectionFactory.setRequestedChannelMax(0);
+connectionFactory.setRequestedFrameMax(0);
+return connectionFactory;
+  }
+
+  /**

[jira] [Work logged] (BEAM-1240) Create RabbitMqIO

2018-05-30 Thread ASF GitHub Bot (JIRA)



 [ 
https://issues.apache.org/jira/browse/BEAM-1240?focusedWorklogId=107366&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-107366
 ]

ASF GitHub Bot logged work on BEAM-1240:


Author: ASF GitHub Bot
Created on: 30/May/18 20:42
Start Date: 30/May/18 20:42
Worklog Time Spent: 10m 
  Work Description: jkff commented on a change in pull request #1729: 
[BEAM-1240] Create RabbitMqIO
URL: https://github.com/apache/beam/pull/1729#discussion_r191915183
 
 

 ##
 File path: 
sdks/java/io/rabbitmq/src/main/java/org/apache/beam/sdk/io/rabbitmq/RabbitMqIO.java
 ##
 @@ -0,0 +1,748 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.io.rabbitmq;
+
+import static com.google.common.base.Preconditions.checkArgument;
+
+import com.google.auto.value.AutoValue;
+import com.rabbitmq.client.Channel;
+import com.rabbitmq.client.Connection;
+import com.rabbitmq.client.ConnectionFactory;
+import com.rabbitmq.client.QueueingConsumer;
+
+import java.io.IOException;
+import java.io.Serializable;
+import java.net.URISyntaxException;
+import java.security.KeyManagementException;
+import java.security.NoSuchAlgorithmException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.NoSuchElementException;
+
+import javax.annotation.Nullable;
+
+import org.apache.beam.sdk.annotations.Experimental;
+import org.apache.beam.sdk.coders.ByteArrayCoder;
+import org.apache.beam.sdk.coders.Coder;
+import org.apache.beam.sdk.coders.SerializableCoder;
+import org.apache.beam.sdk.io.UnboundedSource;
+import org.apache.beam.sdk.options.PipelineOptions;
+import org.apache.beam.sdk.transforms.DoFn;
+import org.apache.beam.sdk.transforms.PTransform;
+import org.apache.beam.sdk.transforms.ParDo;
+import org.apache.beam.sdk.values.PBegin;
+import org.apache.beam.sdk.values.PCollection;
+import org.apache.beam.sdk.values.PDone;
+import org.joda.time.Duration;
+import org.joda.time.Instant;
+
+/**
+ * A IO to publish or consume messages with a RabbitMQ broker.
+ *
+ * Consuming messages from RabbitMQ server
+ *
+ * {@link RabbitMqIO} {@link Read} returns an unbounded {@link PCollection} 
containing RabbitMQ
+ * messages body (as {@code byte[]}).
+ *
+ * To configure a RabbitMQ source, you have to provide a RabbitMQ {@code 
URI} to connect
+ * to a RabbitMQ broker. The following example illustrates various options for 
configuring the
+ * source:
+ *
+ * {@code
+ *
+ *  pipeline.apply(
+ *
RabbitMqIO.read().withUri("amqp://user:password@localhost:5672").withQueue("QUEUE")
+ *
+ * }
+ *
+ * Publishing messages to RabbitMQ server
+ *
+ * {@link RabbitMqIO} {@link Write} can send {@code byte[]} to a RabbitMQ 
server queue.
+ *
+ * As for the {@link Read}, the {@link Write} is configured with a RabbitMQ
+ * {@link ConnectionConfig}.
+ *
+ * For instance:
+ *
+ * {@code
+ *
+ *  pipeline
+ *.apply(...) // provide PCollection
+ *
.apply(RabbitMqIO.write().withUri("amqp://user:password@localhost:5672").withQueue("QUEUE"));
+ *
+ * }
+ */
+@Experimental(Experimental.Kind.SOURCE_SINK)
+public class RabbitMqIO {
+
+  public static Read read() {
+return new AutoValue_RabbitMqIO_Read.Builder()
+.setConnectionConfig(ConnectionConfig.create())
+
.setMaxReadTime(null).setMaxNumRecords(Long.MAX_VALUE).setUseCorrelationId(false).build();
+  }
+
+  public static Write write() {
+return new AutoValue_RabbitMqIO_Write.Builder()
+.setConnectionConfig(ConnectionConfig.create()).build();
+  }
+
+  private RabbitMqIO() {
+  }
+
+  /**
+   * Describe a connection configuration to a RabbitMQ server.
+   */
+  @AutoValue
+  public abstract static class ConnectionConfig implements Serializable {
+
+@Nullable abstract String uri();
+@Nullable abstract String queue();
+@Nullable abstract String exchange();
+@Nullable abstract String exchangeType();
+
+abstract int networkRecoveryInterval();
+abstract boolean automaticRecovery();
+abstract boolean topologyRecovery();
+
+abstract int connectionTimeout();
+abstract int requestedChannelMax();
+abstract int requestedF

[jira] [Work logged] (BEAM-1240) Create RabbitMqIO

2018-05-30 Thread ASF GitHub Bot (JIRA)



 [ 
https://issues.apache.org/jira/browse/BEAM-1240?focusedWorklogId=107367&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-107367
 ]

ASF GitHub Bot logged work on BEAM-1240:


Author: ASF GitHub Bot
Created on: 30/May/18 20:42
Start Date: 30/May/18 20:42
Worklog Time Spent: 10m 
  Work Description: jkff commented on a change in pull request #1729: 
[BEAM-1240] Create RabbitMqIO
URL: https://github.com/apache/beam/pull/1729#discussion_r191915907
 
 

 ##
 File path: 
sdks/java/io/rabbitmq/src/main/java/org/apache/beam/sdk/io/rabbitmq/RabbitMqIO.java
 ##
 @@ -0,0 +1,748 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.io.rabbitmq;
+
+import static com.google.common.base.Preconditions.checkArgument;
+
+import com.google.auto.value.AutoValue;
+import com.rabbitmq.client.Channel;
+import com.rabbitmq.client.Connection;
+import com.rabbitmq.client.ConnectionFactory;
+import com.rabbitmq.client.QueueingConsumer;
+
+import java.io.IOException;
+import java.io.Serializable;
+import java.net.URISyntaxException;
+import java.security.KeyManagementException;
+import java.security.NoSuchAlgorithmException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.NoSuchElementException;
+
+import javax.annotation.Nullable;
+
+import org.apache.beam.sdk.annotations.Experimental;
+import org.apache.beam.sdk.coders.ByteArrayCoder;
+import org.apache.beam.sdk.coders.Coder;
+import org.apache.beam.sdk.coders.SerializableCoder;
+import org.apache.beam.sdk.io.UnboundedSource;
+import org.apache.beam.sdk.options.PipelineOptions;
+import org.apache.beam.sdk.transforms.DoFn;
+import org.apache.beam.sdk.transforms.PTransform;
+import org.apache.beam.sdk.transforms.ParDo;
+import org.apache.beam.sdk.values.PBegin;
+import org.apache.beam.sdk.values.PCollection;
+import org.apache.beam.sdk.values.PDone;
+import org.joda.time.Duration;
+import org.joda.time.Instant;
+
+/**
+ * A IO to publish or consume messages with a RabbitMQ broker.
+ *
+ * Consuming messages from RabbitMQ server
+ *
+ * {@link RabbitMqIO} {@link Read} returns an unbounded {@link PCollection} 
containing RabbitMQ
+ * messages body (as {@code byte[]}).
+ *
+ * To configure a RabbitMQ source, you have to provide a RabbitMQ {@code 
URI} to connect
+ * to a RabbitMQ broker. The following example illustrates various options for 
configuring the
+ * source:
+ *
+ * {@code
+ *
+ *  pipeline.apply(
+ *
RabbitMqIO.read().withUri("amqp://user:password@localhost:5672").withQueue("QUEUE")
+ *
+ * }
+ *
+ * Publishing messages to RabbitMQ server
+ *
+ * {@link RabbitMqIO} {@link Write} can send {@code byte[]} to a RabbitMQ 
server queue.
+ *
+ * As for the {@link Read}, the {@link Write} is configured with a RabbitMQ
+ * {@link ConnectionConfig}.
+ *
+ * For instance:
+ *
+ * {@code
+ *
+ *  pipeline
+ *.apply(...) // provide PCollection
+ *
.apply(RabbitMqIO.write().withUri("amqp://user:password@localhost:5672").withQueue("QUEUE"));
 
 Review comment:
   This is not valid anymore, please update the javadoc


This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
 
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org


Issue Time Tracking
---

Worklog Id: (was: 107367)
Time Spent: 6h  (was: 5h 50m)

> Create RabbitMqIO
> -
>
> Key: BEAM-1240
> URL: https://issues.apache.org/jira/browse/BEAM-1240
> Project: Beam
>  Issue Type: New Feature
>  Components: io-ideas
>Reporter: Jean-Baptiste Onofré
>Assignee: Jean-Baptiste Onofré
>Priority: Major
>  Time Spent: 6h
>  Remaining Estimate: 0h
>




--
This message was sent by Atlassian JIRA
(v7.6.3#76005)

[jira] [Work logged] (BEAM-1240) Create RabbitMqIO

2018-05-30 Thread ASF GitHub Bot (JIRA)



 [ 
https://issues.apache.org/jira/browse/BEAM-1240?focusedWorklogId=107371&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-107371
 ]

ASF GitHub Bot logged work on BEAM-1240:


Author: ASF GitHub Bot
Created on: 30/May/18 20:42
Start Date: 30/May/18 20:42
Worklog Time Spent: 10m 
  Work Description: jkff commented on a change in pull request #1729: 
[BEAM-1240] Create RabbitMqIO
URL: https://github.com/apache/beam/pull/1729#discussion_r191915237
 
 

 ##
 File path: 
sdks/java/io/rabbitmq/src/main/java/org/apache/beam/sdk/io/rabbitmq/RabbitMqIO.java
 ##
 @@ -0,0 +1,748 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.io.rabbitmq;
+
+import static com.google.common.base.Preconditions.checkArgument;
+
+import com.google.auto.value.AutoValue;
+import com.rabbitmq.client.Channel;
+import com.rabbitmq.client.Connection;
+import com.rabbitmq.client.ConnectionFactory;
+import com.rabbitmq.client.QueueingConsumer;
+
+import java.io.IOException;
+import java.io.Serializable;
+import java.net.URISyntaxException;
+import java.security.KeyManagementException;
+import java.security.NoSuchAlgorithmException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.NoSuchElementException;
+
+import javax.annotation.Nullable;
+
+import org.apache.beam.sdk.annotations.Experimental;
+import org.apache.beam.sdk.coders.ByteArrayCoder;
+import org.apache.beam.sdk.coders.Coder;
+import org.apache.beam.sdk.coders.SerializableCoder;
+import org.apache.beam.sdk.io.UnboundedSource;
+import org.apache.beam.sdk.options.PipelineOptions;
+import org.apache.beam.sdk.transforms.DoFn;
+import org.apache.beam.sdk.transforms.PTransform;
+import org.apache.beam.sdk.transforms.ParDo;
+import org.apache.beam.sdk.values.PBegin;
+import org.apache.beam.sdk.values.PCollection;
+import org.apache.beam.sdk.values.PDone;
+import org.joda.time.Duration;
+import org.joda.time.Instant;
+
+/**
+ * A IO to publish or consume messages with a RabbitMQ broker.
+ *
+ * Consuming messages from RabbitMQ server
+ *
+ * {@link RabbitMqIO} {@link Read} returns an unbounded {@link PCollection} 
containing RabbitMQ
+ * messages body (as {@code byte[]}).
+ *
+ * To configure a RabbitMQ source, you have to provide a RabbitMQ {@code 
URI} to connect
+ * to a RabbitMQ broker. The following example illustrates various options for 
configuring the
+ * source:
+ *
+ * {@code
+ *
+ *  pipeline.apply(
+ *
RabbitMqIO.read().withUri("amqp://user:password@localhost:5672").withQueue("QUEUE")
+ *
+ * }
+ *
+ * Publishing messages to RabbitMQ server
+ *
+ * {@link RabbitMqIO} {@link Write} can send {@code byte[]} to a RabbitMQ 
server queue.
+ *
+ * As for the {@link Read}, the {@link Write} is configured with a RabbitMQ
+ * {@link ConnectionConfig}.
+ *
+ * For instance:
+ *
+ * {@code
+ *
+ *  pipeline
+ *.apply(...) // provide PCollection
+ *
.apply(RabbitMqIO.write().withUri("amqp://user:password@localhost:5672").withQueue("QUEUE"));
+ *
+ * }
+ */
+@Experimental(Experimental.Kind.SOURCE_SINK)
+public class RabbitMqIO {
+
+  public static Read read() {
+return new AutoValue_RabbitMqIO_Read.Builder()
+.setConnectionConfig(ConnectionConfig.create())
+
.setMaxReadTime(null).setMaxNumRecords(Long.MAX_VALUE).setUseCorrelationId(false).build();
+  }
+
+  public static Write write() {
+return new AutoValue_RabbitMqIO_Write.Builder()
+.setConnectionConfig(ConnectionConfig.create()).build();
+  }
+
+  private RabbitMqIO() {
+  }
+
+  /**
+   * Describe a connection configuration to a RabbitMQ server.
+   */
+  @AutoValue
+  public abstract static class ConnectionConfig implements Serializable {
+
+@Nullable abstract String uri();
+@Nullable abstract String queue();
+@Nullable abstract String exchange();
+@Nullable abstract String exchangeType();
+
+abstract int networkRecoveryInterval();
+abstract boolean automaticRecovery();
+abstract boolean topologyRecovery();
+
+abstract int connectionTimeout();
+abstract int requestedChannelMax();
+abstract int requestedF

[jira] [Work logged] (BEAM-1240) Create RabbitMqIO

2018-05-30 Thread ASF GitHub Bot (JIRA)



 [ 
https://issues.apache.org/jira/browse/BEAM-1240?focusedWorklogId=107372&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-107372
 ]

ASF GitHub Bot logged work on BEAM-1240:


Author: ASF GitHub Bot
Created on: 30/May/18 20:42
Start Date: 30/May/18 20:42
Worklog Time Spent: 10m 
  Work Description: jkff commented on a change in pull request #1729: 
[BEAM-1240] Create RabbitMqIO
URL: https://github.com/apache/beam/pull/1729#discussion_r191913570
 
 

 ##
 File path: 
sdks/java/io/rabbitmq/src/main/java/org/apache/beam/sdk/io/rabbitmq/RabbitMqIO.java
 ##
 @@ -0,0 +1,530 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.io.rabbitmq;
+
+import static com.google.common.base.Preconditions.checkArgument;
+
+import com.google.auto.value.AutoValue;
+import com.rabbitmq.client.Channel;
+import com.rabbitmq.client.Connection;
+import com.rabbitmq.client.ConnectionFactory;
+import com.rabbitmq.client.QueueingConsumer;
+import java.io.IOException;
+import java.io.Serializable;
+import java.net.URISyntaxException;
+import java.nio.charset.StandardCharsets;
+import java.security.KeyManagementException;
+import java.security.NoSuchAlgorithmException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.NoSuchElementException;
+import javax.annotation.Nullable;
+import org.apache.beam.sdk.annotations.Experimental;
+import org.apache.beam.sdk.coders.Coder;
+import org.apache.beam.sdk.coders.SerializableCoder;
+import org.apache.beam.sdk.io.UnboundedSource;
+import org.apache.beam.sdk.options.PipelineOptions;
+import org.apache.beam.sdk.transforms.DoFn;
+import org.apache.beam.sdk.transforms.PTransform;
+import org.apache.beam.sdk.transforms.ParDo;
+import org.apache.beam.sdk.values.PBegin;
+import org.apache.beam.sdk.values.PCollection;
+import org.apache.beam.sdk.values.PDone;
+import org.joda.time.Duration;
+import org.joda.time.Instant;
+
+/**
+ * A IO to publish or consume messages with a RabbitMQ broker.
+ *
+ * Consuming messages from RabbitMQ server
+ *
+ * {@link RabbitMqIO} {@link Read} returns an unbounded {@link PCollection} 
containing RabbitMQ
+ * messages body (as {@code byte[]}).
+ *
+ * To configure a RabbitMQ source, you have to provide a RabbitMQ {@code 
URI} to connect
+ * to a RabbitMQ broker. The following example illustrates various options for 
configuring the
+ * source:
+ *
+ * {@code
+ *
+ *  pipeline.apply(
+ *
RabbitMqIO.read().withUri("amqp://user:password@localhost:5672").withQueue("QUEUE")
+ *
+ * }
+ *
+ * Publishing messages to RabbitMQ server
+ *
+ * {@link RabbitMqIO} {@link Write} can send {@code byte[]} to a RabbitMQ 
server queue.
+ *
+ * As for the {@link Read}, the {@link Write} is configured with a RabbitMQ 
URI.
+ *
+ * For instance:
+ *
+ * {@code
+ *
+ *  pipeline
+ *.apply(...) // provide PCollection
+ *
.apply(RabbitMqIO.write().withUri("amqp://user:password@localhost:5672").withQueue("QUEUE"));
+ *
+ * }
+ */
+@Experimental(Experimental.Kind.SOURCE_SINK)
+public class RabbitMqIO {
+  public static Read read() {
+return new AutoValue_RabbitMqIO_Read.Builder().setQueueDeclare(false)
+
.setMaxReadTime(null).setMaxNumRecords(Long.MAX_VALUE).setUseCorrelationId(false).build();
+  }
+
+  public static Write write() {
+return new 
AutoValue_RabbitMqIO_Write.Builder().setExchangeDeclare(false).build();
+  }
+
+  private RabbitMqIO() {
+  }
+
+  private static ConnectionFactory createConnectionFactory(String uri) throws 
URISyntaxException,
+  NoSuchAlgorithmException, KeyManagementException {
+ConnectionFactory connectionFactory = new ConnectionFactory();
+connectionFactory.setUri(uri);
+connectionFactory.setAutomaticRecoveryEnabled(true);
+connectionFactory.setConnectionTimeout(6);
+connectionFactory.setNetworkRecoveryInterval(5000);
+connectionFactory.setRequestedHeartbeat(60);
+connectionFactory.setTopologyRecoveryEnabled(true);
+connectionFactory.setRequestedChannelMax(0);
+connectionFactory.setRequestedFrameMax(0);
+return connectionFactory;
+  }
+
+  /**

[jira] [Work logged] (BEAM-1240) Create RabbitMqIO

2018-05-30 Thread ASF GitHub Bot (JIRA)



 [ 
https://issues.apache.org/jira/browse/BEAM-1240?focusedWorklogId=107360&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-107360
 ]

ASF GitHub Bot logged work on BEAM-1240:


Author: ASF GitHub Bot
Created on: 30/May/18 20:42
Start Date: 30/May/18 20:42
Worklog Time Spent: 10m 
  Work Description: jkff commented on a change in pull request #1729: 
[BEAM-1240] Create RabbitMqIO
URL: https://github.com/apache/beam/pull/1729#discussion_r191913128
 
 

 ##
 File path: 
sdks/java/io/rabbitmq/src/main/java/org/apache/beam/sdk/io/rabbitmq/RabbitMqIO.java
 ##
 @@ -0,0 +1,530 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.io.rabbitmq;
+
+import static com.google.common.base.Preconditions.checkArgument;
+
+import com.google.auto.value.AutoValue;
+import com.rabbitmq.client.Channel;
+import com.rabbitmq.client.Connection;
+import com.rabbitmq.client.ConnectionFactory;
+import com.rabbitmq.client.QueueingConsumer;
+import java.io.IOException;
+import java.io.Serializable;
+import java.net.URISyntaxException;
+import java.nio.charset.StandardCharsets;
+import java.security.KeyManagementException;
+import java.security.NoSuchAlgorithmException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.NoSuchElementException;
+import javax.annotation.Nullable;
+import org.apache.beam.sdk.annotations.Experimental;
+import org.apache.beam.sdk.coders.Coder;
+import org.apache.beam.sdk.coders.SerializableCoder;
+import org.apache.beam.sdk.io.UnboundedSource;
+import org.apache.beam.sdk.options.PipelineOptions;
+import org.apache.beam.sdk.transforms.DoFn;
+import org.apache.beam.sdk.transforms.PTransform;
+import org.apache.beam.sdk.transforms.ParDo;
+import org.apache.beam.sdk.values.PBegin;
+import org.apache.beam.sdk.values.PCollection;
+import org.apache.beam.sdk.values.PDone;
+import org.joda.time.Duration;
+import org.joda.time.Instant;
+
+/**
+ * A IO to publish or consume messages with a RabbitMQ broker.
+ *
+ * Consuming messages from RabbitMQ server
+ *
+ * {@link RabbitMqIO} {@link Read} returns an unbounded {@link PCollection} 
containing RabbitMQ
+ * messages body (as {@code byte[]}).
+ *
+ * To configure a RabbitMQ source, you have to provide a RabbitMQ {@code 
URI} to connect
+ * to a RabbitMQ broker. The following example illustrates various options for 
configuring the
+ * source:
+ *
+ * {@code
+ *
+ *  pipeline.apply(
+ *
RabbitMqIO.read().withUri("amqp://user:password@localhost:5672").withQueue("QUEUE")
+ *
+ * }
+ *
+ * Publishing messages to RabbitMQ server
+ *
+ * {@link RabbitMqIO} {@link Write} can send {@code byte[]} to a RabbitMQ 
server queue.
+ *
+ * As for the {@link Read}, the {@link Write} is configured with a RabbitMQ 
URI.
+ *
+ * For instance:
+ *
+ * {@code
+ *
+ *  pipeline
+ *.apply(...) // provide PCollection
+ *
.apply(RabbitMqIO.write().withUri("amqp://user:password@localhost:5672").withQueue("QUEUE"));
+ *
+ * }
+ */
+@Experimental(Experimental.Kind.SOURCE_SINK)
+public class RabbitMqIO {
+  public static Read read() {
+return new AutoValue_RabbitMqIO_Read.Builder().setQueueDeclare(false)
+
.setMaxReadTime(null).setMaxNumRecords(Long.MAX_VALUE).setUseCorrelationId(false).build();
+  }
+
+  public static Write write() {
+return new 
AutoValue_RabbitMqIO_Write.Builder().setExchangeDeclare(false).build();
+  }
+
+  private RabbitMqIO() {
+  }
+
+  private static ConnectionFactory createConnectionFactory(String uri) throws 
URISyntaxException,
+  NoSuchAlgorithmException, KeyManagementException {
+ConnectionFactory connectionFactory = new ConnectionFactory();
+connectionFactory.setUri(uri);
+connectionFactory.setAutomaticRecoveryEnabled(true);
+connectionFactory.setConnectionTimeout(6);
+connectionFactory.setNetworkRecoveryInterval(5000);
+connectionFactory.setRequestedHeartbeat(60);
+connectionFactory.setTopologyRecoveryEnabled(true);
+connectionFactory.setRequestedChannelMax(0);
+connectionFactory.setRequestedFrameMax(0);
+return connectionFactory;
+  }
+
+  /**

[jira] [Work logged] (BEAM-1240) Create RabbitMqIO

2018-05-30 Thread ASF GitHub Bot (JIRA)



 [ 
https://issues.apache.org/jira/browse/BEAM-1240?focusedWorklogId=107364&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-107364
 ]

ASF GitHub Bot logged work on BEAM-1240:


Author: ASF GitHub Bot
Created on: 30/May/18 20:42
Start Date: 30/May/18 20:42
Worklog Time Spent: 10m 
  Work Description: jkff commented on a change in pull request #1729: 
[BEAM-1240] Create RabbitMqIO
URL: https://github.com/apache/beam/pull/1729#discussion_r191910784
 
 

 ##
 File path: 
sdks/java/io/rabbitmq/src/main/java/org/apache/beam/sdk/io/rabbitmq/RabbitMqMessage.java
 ##
 @@ -0,0 +1,215 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.io.rabbitmq;
+
+import com.rabbitmq.client.AMQP;
+import java.io.Serializable;
+import java.util.Arrays;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Objects;
+
+/** It contains the message payload, and additional metadata like routing key 
or attributes. */
+public class RabbitMqMessage implements Serializable {
+  private final String routingKey;
 
 Review comment:
   Which of them are `@Nullable`?


This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
 
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org


Issue Time Tracking
---

Worklog Id: (was: 107364)
Time Spent: 5.5h  (was: 5h 20m)

> Create RabbitMqIO
> -
>
> Key: BEAM-1240
> URL: https://issues.apache.org/jira/browse/BEAM-1240
> Project: Beam
>  Issue Type: New Feature
>  Components: io-ideas
>Reporter: Jean-Baptiste Onofré
>Assignee: Jean-Baptiste Onofré
>Priority: Major
>  Time Spent: 5.5h
>  Remaining Estimate: 0h
>




--
This message was sent by Atlassian JIRA
(v7.6.3#76005)

[jira] [Work logged] (BEAM-1240) Create RabbitMqIO

2018-05-30 Thread ASF GitHub Bot (JIRA)



 [ 
https://issues.apache.org/jira/browse/BEAM-1240?focusedWorklogId=107352&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-107352
 ]

ASF GitHub Bot logged work on BEAM-1240:


Author: ASF GitHub Bot
Created on: 30/May/18 20:13
Start Date: 30/May/18 20:13
Worklog Time Spent: 10m 
  Work Description: jbonofre commented on issue #1729: [BEAM-1240] Create 
RabbitMqIO
URL: https://github.com/apache/beam/pull/1729#issuecomment-393302533
 
 
   @iemejia the PR is complete and cover the most used use cases. PTAL, thanks !


This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
 
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org


Issue Time Tracking
---

Worklog Id: (was: 107352)
Time Spent: 4h 50m  (was: 4h 40m)

> Create RabbitMqIO
> -
>
> Key: BEAM-1240
> URL: https://issues.apache.org/jira/browse/BEAM-1240
> Project: Beam
>  Issue Type: New Feature
>  Components: io-ideas
>Reporter: Jean-Baptiste Onofré
>Assignee: Jean-Baptiste Onofré
>Priority: Major
>  Time Spent: 4h 50m
>  Remaining Estimate: 0h
>




--
This message was sent by Atlassian JIRA
(v7.6.3#76005)

[jira] [Work logged] (BEAM-1240) Create RabbitMqIO

2018-05-24 Thread ASF GitHub Bot (JIRA)


 [ 
https://issues.apache.org/jira/browse/BEAM-1240?focusedWorklogId=105511&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-105511
 ]

ASF GitHub Bot logged work on BEAM-1240:


Author: ASF GitHub Bot
Created on: 24/May/18 08:46
Start Date: 24/May/18 08:46
Worklog Time Spent: 10m 
  Work Description: jbonofre commented on issue #1729: [BEAM-1240] Create 
RabbitMqIO
URL: https://github.com/apache/beam/pull/1729#issuecomment-391637840
 
 
   I gonna resolve the conflicts.


This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
 
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org


Issue Time Tracking
---

Worklog Id: (was: 105511)
Time Spent: 4h 40m  (was: 4.5h)

> Create RabbitMqIO
> -
>
> Key: BEAM-1240
> URL: https://issues.apache.org/jira/browse/BEAM-1240
> Project: Beam
>  Issue Type: New Feature
>  Components: io-ideas
>Reporter: Jean-Baptiste Onofré
>Assignee: Jean-Baptiste Onofré
>Priority: Major
>  Time Spent: 4h 40m
>  Remaining Estimate: 0h
>




--
This message was sent by Atlassian JIRA
(v7.6.3#76005)

[jira] [Work logged] (BEAM-1240) Create RabbitMqIO

2018-05-14 Thread ASF GitHub Bot (JIRA)


 [ 
https://issues.apache.org/jira/browse/BEAM-1240?focusedWorklogId=101746&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-101746
 ]

ASF GitHub Bot logged work on BEAM-1240:


Author: ASF GitHub Bot
Created on: 14/May/18 14:35
Start Date: 14/May/18 14:35
Worklog Time Spent: 10m 
  Work Description: jbonofre commented on issue #1729: [BEAM-1240] Create 
RabbitMqIO
URL: https://github.com/apache/beam/pull/1729#issuecomment-388839507
 
 
   retest this please


This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
 
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org


Issue Time Tracking
---

Worklog Id: (was: 101746)
Time Spent: 4.5h  (was: 4h 20m)

> Create RabbitMqIO
> -
>
> Key: BEAM-1240
> URL: https://issues.apache.org/jira/browse/BEAM-1240
> Project: Beam
>  Issue Type: New Feature
>  Components: io-ideas
>Reporter: Jean-Baptiste Onofré
>Assignee: Jean-Baptiste Onofré
>Priority: Major
>  Time Spent: 4.5h
>  Remaining Estimate: 0h
>




--
This message was sent by Atlassian JIRA
(v7.6.3#76005)

[jira] [Work logged] (BEAM-1240) Create RabbitMqIO

2018-05-07 Thread ASF GitHub Bot (JIRA)


 [ 
https://issues.apache.org/jira/browse/BEAM-1240?focusedWorklogId=99400&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-99400
 ]

ASF GitHub Bot logged work on BEAM-1240:


Author: ASF GitHub Bot
Created on: 08/May/18 06:30
Start Date: 08/May/18 06:30
Worklog Time Spent: 10m 
  Work Description: jbonofre commented on issue #1729: [BEAM-1240] Create 
RabbitMqIO
URL: https://github.com/apache/beam/pull/1729#issuecomment-387298935
 
 
   I'm fixing the build issue with gradle.


This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
 
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org


Issue Time Tracking
---

Worklog Id: (was: 99400)
Time Spent: 4h 20m  (was: 4h 10m)

> Create RabbitMqIO
> -
>
> Key: BEAM-1240
> URL: https://issues.apache.org/jira/browse/BEAM-1240
> Project: Beam
>  Issue Type: New Feature
>  Components: io-ideas
>Reporter: Jean-Baptiste Onofré
>Assignee: Jean-Baptiste Onofré
>Priority: Major
>  Time Spent: 4h 20m
>  Remaining Estimate: 0h
>




--
This message was sent by Atlassian JIRA
(v7.6.3#76005)

[jira] [Work logged] (BEAM-1240) Create RabbitMqIO

2018-05-03 Thread ASF GitHub Bot (JIRA)


 [ 
https://issues.apache.org/jira/browse/BEAM-1240?focusedWorklogId=97831&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-97831
 ]

ASF GitHub Bot logged work on BEAM-1240:


Author: ASF GitHub Bot
Created on: 03/May/18 09:33
Start Date: 03/May/18 09:33
Worklog Time Spent: 10m 
  Work Description: iemejia commented on issue #1729: [BEAM-1240] Create 
RabbitMqIO
URL: https://github.com/apache/beam/pull/1729#issuecomment-386240069
 
 
   @jbonofre can you please rebase this. It is passing but I would like to have 
a fresh build given that it has been quite some time since the last changes.


This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
 
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org


Issue Time Tracking
---

Worklog Id: (was: 97831)
Time Spent: 4h 10m  (was: 4h)

> Create RabbitMqIO
> -
>
> Key: BEAM-1240
> URL: https://issues.apache.org/jira/browse/BEAM-1240
> Project: Beam
>  Issue Type: New Feature
>  Components: io-ideas
>Reporter: Jean-Baptiste Onofré
>Assignee: Jean-Baptiste Onofré
>Priority: Major
>  Time Spent: 4h 10m
>  Remaining Estimate: 0h
>




--
This message was sent by Atlassian JIRA
(v7.6.3#76005)

[jira] [Work logged] (BEAM-1240) Create RabbitMqIO

2018-04-11 Thread ASF GitHub Bot (JIRA)


 [ 
https://issues.apache.org/jira/browse/BEAM-1240?focusedWorklogId=90213&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-90213
 ]

ASF GitHub Bot logged work on BEAM-1240:


Author: ASF GitHub Bot
Created on: 11/Apr/18 22:53
Start Date: 11/Apr/18 22:53
Worklog Time Spent: 10m 
  Work Description: jkff commented on issue #1729: [BEAM-1240] Create 
RabbitMqIO
URL: https://github.com/apache/beam/pull/1729#issuecomment-380620740
 
 
   (just got back from leave; happy to do the next review round after @iemejia)


This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
 
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org


Issue Time Tracking
---

Worklog Id: (was: 90213)
Time Spent: 4h  (was: 3h 50m)

> Create RabbitMqIO
> -
>
> Key: BEAM-1240
> URL: https://issues.apache.org/jira/browse/BEAM-1240
> Project: Beam
>  Issue Type: New Feature
>  Components: io-ideas
>Reporter: Jean-Baptiste Onofré
>Assignee: Jean-Baptiste Onofré
>Priority: Major
>  Time Spent: 4h
>  Remaining Estimate: 0h
>




--
This message was sent by Atlassian JIRA
(v7.6.3#76005)

[jira] [Work logged] (BEAM-1240) Create RabbitMqIO

2018-04-10 Thread ASF GitHub Bot (JIRA)


 [ 
https://issues.apache.org/jira/browse/BEAM-1240?focusedWorklogId=89439&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-89439
 ]

ASF GitHub Bot logged work on BEAM-1240:


Author: ASF GitHub Bot
Created on: 10/Apr/18 14:54
Start Date: 10/Apr/18 14:54
Worklog Time Spent: 10m 
  Work Description: jbonofre commented on issue #1729: [BEAM-1240] Create 
RabbitMqIO
URL: https://github.com/apache/beam/pull/1729#issuecomment-380130957
 
 
   @iemejia as @jkff is little bit busy right now, can you take a look on this 
one please ? Thanks !


This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
 
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org


Issue Time Tracking
---

Worklog Id: (was: 89439)
Time Spent: 3h 50m  (was: 3h 40m)

> Create RabbitMqIO
> -
>
> Key: BEAM-1240
> URL: https://issues.apache.org/jira/browse/BEAM-1240
> Project: Beam
>  Issue Type: New Feature
>  Components: io-ideas
>Reporter: Jean-Baptiste Onofré
>Assignee: Jean-Baptiste Onofré
>Priority: Major
>  Time Spent: 3h 50m
>  Remaining Estimate: 0h
>




--
This message was sent by Atlassian JIRA
(v7.6.3#76005)

[jira] [Work logged] (BEAM-1240) Create RabbitMqIO

2018-04-03 Thread ASF GitHub Bot (JIRA)


 [ 
https://issues.apache.org/jira/browse/BEAM-1240?focusedWorklogId=87115&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-87115
 ]

ASF GitHub Bot logged work on BEAM-1240:


Author: ASF GitHub Bot
Created on: 03/Apr/18 15:07
Start Date: 03/Apr/18 15:07
Worklog Time Spent: 10m 
  Work Description: jbonofre commented on issue #1729: [BEAM-1240] Create 
RabbitMqIO
URL: https://github.com/apache/beam/pull/1729#issuecomment-378283759
 
 
   I updated and rebased the PR. I also added `gradle` build support. @jkff I 
think it's ready for a first merge and then, improve in following steps.


This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
 
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org


Issue Time Tracking
---

Worklog Id: (was: 87115)
Time Spent: 3h 40m  (was: 3.5h)

> Create RabbitMqIO
> -
>
> Key: BEAM-1240
> URL: https://issues.apache.org/jira/browse/BEAM-1240
> Project: Beam
>  Issue Type: New Feature
>  Components: io-ideas
>Reporter: Jean-Baptiste Onofré
>Assignee: Jean-Baptiste Onofré
>Priority: Major
>  Time Spent: 3h 40m
>  Remaining Estimate: 0h
>




--
This message was sent by Atlassian JIRA
(v7.6.3#76005)

[jira] [Work logged] (BEAM-1240) Create RabbitMqIO

2018-04-03 Thread ASF GitHub Bot (JIRA)


 [ 
https://issues.apache.org/jira/browse/BEAM-1240?focusedWorklogId=87107&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-87107
 ]

ASF GitHub Bot logged work on BEAM-1240:


Author: ASF GitHub Bot
Created on: 03/Apr/18 14:36
Start Date: 03/Apr/18 14:36
Worklog Time Spent: 10m 
  Work Description: jbonofre commented on a change in pull request #1729: 
[BEAM-1240] Create RabbitMqIO
URL: https://github.com/apache/beam/pull/1729#discussion_r178846169
 
 

 ##
 File path: 
sdks/java/io/rabbitmq/src/main/java/org/apache/beam/sdk/io/rabbitmq/RabbitMqIO.java
 ##
 @@ -0,0 +1,711 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.io.rabbitmq;
+
+import static com.google.common.base.Preconditions.checkArgument;
+
+import com.google.auto.value.AutoValue;
+import com.rabbitmq.client.AMQP;
+import com.rabbitmq.client.Channel;
+import com.rabbitmq.client.Connection;
+import com.rabbitmq.client.ConnectionFactory;
+import com.rabbitmq.client.QueueingConsumer;
+import java.io.IOException;
+import java.io.Serializable;
+import java.net.URISyntaxException;
+import java.security.KeyManagementException;
+import java.security.NoSuchAlgorithmException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.NoSuchElementException;
+import javax.annotation.Nullable;
+import org.apache.beam.sdk.annotations.Experimental;
+import org.apache.beam.sdk.coders.Coder;
+import org.apache.beam.sdk.coders.SerializableCoder;
+import org.apache.beam.sdk.io.UnboundedSource;
+import org.apache.beam.sdk.options.PipelineOptions;
+import org.apache.beam.sdk.transforms.DoFn;
+import org.apache.beam.sdk.transforms.PTransform;
+import org.apache.beam.sdk.transforms.ParDo;
+import org.apache.beam.sdk.values.PBegin;
+import org.apache.beam.sdk.values.PCollection;
+import org.apache.beam.sdk.values.PDone;
+import org.joda.time.Duration;
+import org.joda.time.Instant;
+
+/**
+ * A IO to publish or consume messages with a RabbitMQ broker.
+ *
+ * Consuming messages from RabbitMQ server
+ *
+ * {@link RabbitMqIO} {@link Read} returns an unbounded {@link PCollection} 
containing RabbitMQ
+ * messages body (as {@code byte[]}).
+ *
+ * To configure a RabbitMQ source, you have to provide a RabbitMQ {@code 
URI} to connect
+ * to a RabbitMQ broker. The following example illustrates various options for 
configuring the
+ * source:
+ *
+ * {@code
+ *
+ *  pipeline.apply(
+ *
RabbitMqIO.read().withUri("amqp://user:password@localhost:5672").withQueue("QUEUE")
+ *
+ * }
+ *
+ * Publishing messages to RabbitMQ server
+ *
+ * {@link RabbitMqIO} {@link Write} can send {@code byte[]} to a RabbitMQ 
server queue.
+ *
+ * As for the {@link Read}, the {@link Write} is configured with a RabbitMQ
+ * {@link ConnectionConfig}.
+ *
+ * For instance:
+ *
+ * {@code
+ *
+ *  pipeline
+ *.apply(...) // provide PCollection
+ *
.apply(RabbitMqIO.write().withUri("amqp://user:password@localhost:5672").withQueue("QUEUE"));
+ *
+ * }
+ */
+@Experimental(Experimental.Kind.SOURCE_SINK)
+public class RabbitMqIO {
+
+  public static Read read() {
+return new AutoValue_RabbitMqIO_Read.Builder()
+.setConnectionConfig(ConnectionConfig.create()).setQueueDeclare(false)
+
.setMaxReadTime(null).setMaxNumRecords(Long.MAX_VALUE).setUseCorrelationId(false).build();
+  }
+
+  public static Write write() {
+return new AutoValue_RabbitMqIO_Write.Builder()
+.setConnectionConfig(ConnectionConfig.create())
+.setExchangeDeclare(false).build();
+  }
+
+  private RabbitMqIO() {
+  }
+
+  /**
+   * Describe a connection configuration to a RabbitMQ server.
+   */
+  @AutoValue
+  public abstract static class ConnectionConfig implements Serializable {
+
+@Nullable abstract String uri();
+
+abstract int networkRecoveryInterval();
+abstract boolean automaticRecovery();
+abstract boolean topologyRecovery();
+
+abstract int connectionTimeout();
+abstract int requestedChannelMax();
+abstract int requestedFrameMax();
+abstract int requestedHeartbeat();
+
+abstract Builder builder();
+
+

[jira] [Work logged] (BEAM-1240) Create RabbitMqIO

2018-04-03 Thread ASF GitHub Bot (JIRA)


 [ 
https://issues.apache.org/jira/browse/BEAM-1240?focusedWorklogId=87106&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-87106
 ]

ASF GitHub Bot logged work on BEAM-1240:


Author: ASF GitHub Bot
Created on: 03/Apr/18 14:35
Start Date: 03/Apr/18 14:35
Worklog Time Spent: 10m 
  Work Description: jbonofre commented on a change in pull request #1729: 
[BEAM-1240] Create RabbitMqIO
URL: https://github.com/apache/beam/pull/1729#discussion_r178845915
 
 

 ##
 File path: 
sdks/java/io/rabbitmq/src/main/java/org/apache/beam/sdk/io/rabbitmq/RabbitMqIO.java
 ##
 @@ -0,0 +1,711 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.io.rabbitmq;
+
+import static com.google.common.base.Preconditions.checkArgument;
+
+import com.google.auto.value.AutoValue;
+import com.rabbitmq.client.AMQP;
+import com.rabbitmq.client.Channel;
+import com.rabbitmq.client.Connection;
+import com.rabbitmq.client.ConnectionFactory;
+import com.rabbitmq.client.QueueingConsumer;
+import java.io.IOException;
+import java.io.Serializable;
+import java.net.URISyntaxException;
+import java.security.KeyManagementException;
+import java.security.NoSuchAlgorithmException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.NoSuchElementException;
+import javax.annotation.Nullable;
+import org.apache.beam.sdk.annotations.Experimental;
+import org.apache.beam.sdk.coders.Coder;
+import org.apache.beam.sdk.coders.SerializableCoder;
+import org.apache.beam.sdk.io.UnboundedSource;
+import org.apache.beam.sdk.options.PipelineOptions;
+import org.apache.beam.sdk.transforms.DoFn;
+import org.apache.beam.sdk.transforms.PTransform;
+import org.apache.beam.sdk.transforms.ParDo;
+import org.apache.beam.sdk.values.PBegin;
+import org.apache.beam.sdk.values.PCollection;
+import org.apache.beam.sdk.values.PDone;
+import org.joda.time.Duration;
+import org.joda.time.Instant;
+
+/**
+ * A IO to publish or consume messages with a RabbitMQ broker.
+ *
+ * Consuming messages from RabbitMQ server
+ *
+ * {@link RabbitMqIO} {@link Read} returns an unbounded {@link PCollection} 
containing RabbitMQ
+ * messages body (as {@code byte[]}).
+ *
+ * To configure a RabbitMQ source, you have to provide a RabbitMQ {@code 
URI} to connect
+ * to a RabbitMQ broker. The following example illustrates various options for 
configuring the
+ * source:
+ *
+ * {@code
+ *
+ *  pipeline.apply(
+ *
RabbitMqIO.read().withUri("amqp://user:password@localhost:5672").withQueue("QUEUE")
+ *
+ * }
+ *
+ * Publishing messages to RabbitMQ server
+ *
+ * {@link RabbitMqIO} {@link Write} can send {@code byte[]} to a RabbitMQ 
server queue.
+ *
+ * As for the {@link Read}, the {@link Write} is configured with a RabbitMQ
+ * {@link ConnectionConfig}.
+ *
+ * For instance:
+ *
+ * {@code
+ *
+ *  pipeline
+ *.apply(...) // provide PCollection
+ *
.apply(RabbitMqIO.write().withUri("amqp://user:password@localhost:5672").withQueue("QUEUE"));
+ *
+ * }
+ */
+@Experimental(Experimental.Kind.SOURCE_SINK)
+public class RabbitMqIO {
+
+  public static Read read() {
+return new AutoValue_RabbitMqIO_Read.Builder()
+.setConnectionConfig(ConnectionConfig.create()).setQueueDeclare(false)
+
.setMaxReadTime(null).setMaxNumRecords(Long.MAX_VALUE).setUseCorrelationId(false).build();
+  }
+
+  public static Write write() {
+return new AutoValue_RabbitMqIO_Write.Builder()
+.setConnectionConfig(ConnectionConfig.create())
+.setExchangeDeclare(false).build();
+  }
+
+  private RabbitMqIO() {
+  }
+
+  /**
+   * Describe a connection configuration to a RabbitMQ server.
+   */
+  @AutoValue
+  public abstract static class ConnectionConfig implements Serializable {
+
+@Nullable abstract String uri();
+
+abstract int networkRecoveryInterval();
+abstract boolean automaticRecovery();
+abstract boolean topologyRecovery();
+
+abstract int connectionTimeout();
+abstract int requestedChannelMax();
+abstract int requestedFrameMax();
+abstract int requestedHeartbeat();
+
+abstract Builder builder();
+
+

[jira] [Work logged] (BEAM-1240) Create RabbitMqIO

2018-04-03 Thread ASF GitHub Bot (JIRA)


 [ 
https://issues.apache.org/jira/browse/BEAM-1240?focusedWorklogId=87105&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-87105
 ]

ASF GitHub Bot logged work on BEAM-1240:


Author: ASF GitHub Bot
Created on: 03/Apr/18 14:34
Start Date: 03/Apr/18 14:34
Worklog Time Spent: 10m 
  Work Description: jbonofre commented on a change in pull request #1729: 
[BEAM-1240] Create RabbitMqIO
URL: https://github.com/apache/beam/pull/1729#discussion_r178845747
 
 

 ##
 File path: 
sdks/java/io/rabbitmq/src/main/java/org/apache/beam/sdk/io/rabbitmq/RabbitMqIO.java
 ##
 @@ -0,0 +1,711 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.io.rabbitmq;
+
+import static com.google.common.base.Preconditions.checkArgument;
+
+import com.google.auto.value.AutoValue;
+import com.rabbitmq.client.AMQP;
+import com.rabbitmq.client.Channel;
+import com.rabbitmq.client.Connection;
+import com.rabbitmq.client.ConnectionFactory;
+import com.rabbitmq.client.QueueingConsumer;
+import java.io.IOException;
+import java.io.Serializable;
+import java.net.URISyntaxException;
+import java.security.KeyManagementException;
+import java.security.NoSuchAlgorithmException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.NoSuchElementException;
+import javax.annotation.Nullable;
+import org.apache.beam.sdk.annotations.Experimental;
+import org.apache.beam.sdk.coders.Coder;
+import org.apache.beam.sdk.coders.SerializableCoder;
+import org.apache.beam.sdk.io.UnboundedSource;
+import org.apache.beam.sdk.options.PipelineOptions;
+import org.apache.beam.sdk.transforms.DoFn;
+import org.apache.beam.sdk.transforms.PTransform;
+import org.apache.beam.sdk.transforms.ParDo;
+import org.apache.beam.sdk.values.PBegin;
+import org.apache.beam.sdk.values.PCollection;
+import org.apache.beam.sdk.values.PDone;
+import org.joda.time.Duration;
+import org.joda.time.Instant;
+
+/**
+ * A IO to publish or consume messages with a RabbitMQ broker.
+ *
+ * Consuming messages from RabbitMQ server
+ *
+ * {@link RabbitMqIO} {@link Read} returns an unbounded {@link PCollection} 
containing RabbitMQ
+ * messages body (as {@code byte[]}).
+ *
+ * To configure a RabbitMQ source, you have to provide a RabbitMQ {@code 
URI} to connect
+ * to a RabbitMQ broker. The following example illustrates various options for 
configuring the
+ * source:
+ *
+ * {@code
+ *
+ *  pipeline.apply(
+ *
RabbitMqIO.read().withUri("amqp://user:password@localhost:5672").withQueue("QUEUE")
+ *
+ * }
+ *
+ * Publishing messages to RabbitMQ server
+ *
+ * {@link RabbitMqIO} {@link Write} can send {@code byte[]} to a RabbitMQ 
server queue.
+ *
+ * As for the {@link Read}, the {@link Write} is configured with a RabbitMQ
+ * {@link ConnectionConfig}.
+ *
+ * For instance:
+ *
+ * {@code
+ *
+ *  pipeline
+ *.apply(...) // provide PCollection
+ *
.apply(RabbitMqIO.write().withUri("amqp://user:password@localhost:5672").withQueue("QUEUE"));
+ *
+ * }
+ */
+@Experimental(Experimental.Kind.SOURCE_SINK)
+public class RabbitMqIO {
+
+  public static Read read() {
+return new AutoValue_RabbitMqIO_Read.Builder()
+.setConnectionConfig(ConnectionConfig.create()).setQueueDeclare(false)
+
.setMaxReadTime(null).setMaxNumRecords(Long.MAX_VALUE).setUseCorrelationId(false).build();
+  }
+
+  public static Write write() {
+return new AutoValue_RabbitMqIO_Write.Builder()
+.setConnectionConfig(ConnectionConfig.create())
+.setExchangeDeclare(false).build();
+  }
+
+  private RabbitMqIO() {
+  }
+
+  /**
+   * Describe a connection configuration to a RabbitMQ server.
+   */
+  @AutoValue
+  public abstract static class ConnectionConfig implements Serializable {
+
+@Nullable abstract String uri();
+
+abstract int networkRecoveryInterval();
+abstract boolean automaticRecovery();
+abstract boolean topologyRecovery();
+
+abstract int connectionTimeout();
+abstract int requestedChannelMax();
+abstract int requestedFrameMax();
+abstract int requestedHeartbeat();
+
+abstract Builder builder();
+
+

[jira] [Work logged] (BEAM-1240) Create RabbitMqIO

2018-04-03 Thread ASF GitHub Bot (JIRA)


 [ 
https://issues.apache.org/jira/browse/BEAM-1240?focusedWorklogId=87104&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-87104
 ]

ASF GitHub Bot logged work on BEAM-1240:


Author: ASF GitHub Bot
Created on: 03/Apr/18 14:34
Start Date: 03/Apr/18 14:34
Worklog Time Spent: 10m 
  Work Description: jbonofre commented on a change in pull request #1729: 
[BEAM-1240] Create RabbitMqIO
URL: https://github.com/apache/beam/pull/1729#discussion_r178845438
 
 

 ##
 File path: 
sdks/java/io/rabbitmq/src/main/java/org/apache/beam/sdk/io/rabbitmq/RabbitMqIO.java
 ##
 @@ -0,0 +1,711 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.io.rabbitmq;
+
+import static com.google.common.base.Preconditions.checkArgument;
+
+import com.google.auto.value.AutoValue;
+import com.rabbitmq.client.AMQP;
+import com.rabbitmq.client.Channel;
+import com.rabbitmq.client.Connection;
+import com.rabbitmq.client.ConnectionFactory;
+import com.rabbitmq.client.QueueingConsumer;
+import java.io.IOException;
+import java.io.Serializable;
+import java.net.URISyntaxException;
+import java.security.KeyManagementException;
+import java.security.NoSuchAlgorithmException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.NoSuchElementException;
+import javax.annotation.Nullable;
+import org.apache.beam.sdk.annotations.Experimental;
+import org.apache.beam.sdk.coders.Coder;
+import org.apache.beam.sdk.coders.SerializableCoder;
+import org.apache.beam.sdk.io.UnboundedSource;
+import org.apache.beam.sdk.options.PipelineOptions;
+import org.apache.beam.sdk.transforms.DoFn;
+import org.apache.beam.sdk.transforms.PTransform;
+import org.apache.beam.sdk.transforms.ParDo;
+import org.apache.beam.sdk.values.PBegin;
+import org.apache.beam.sdk.values.PCollection;
+import org.apache.beam.sdk.values.PDone;
+import org.joda.time.Duration;
+import org.joda.time.Instant;
+
+/**
+ * A IO to publish or consume messages with a RabbitMQ broker.
+ *
+ * Consuming messages from RabbitMQ server
+ *
+ * {@link RabbitMqIO} {@link Read} returns an unbounded {@link PCollection} 
containing RabbitMQ
+ * messages body (as {@code byte[]}).
+ *
+ * To configure a RabbitMQ source, you have to provide a RabbitMQ {@code 
URI} to connect
+ * to a RabbitMQ broker. The following example illustrates various options for 
configuring the
+ * source:
+ *
+ * {@code
+ *
+ *  pipeline.apply(
+ *
RabbitMqIO.read().withUri("amqp://user:password@localhost:5672").withQueue("QUEUE")
+ *
+ * }
+ *
+ * Publishing messages to RabbitMQ server
+ *
+ * {@link RabbitMqIO} {@link Write} can send {@code byte[]} to a RabbitMQ 
server queue.
+ *
+ * As for the {@link Read}, the {@link Write} is configured with a RabbitMQ
+ * {@link ConnectionConfig}.
+ *
+ * For instance:
+ *
+ * {@code
+ *
+ *  pipeline
+ *.apply(...) // provide PCollection
+ *
.apply(RabbitMqIO.write().withUri("amqp://user:password@localhost:5672").withQueue("QUEUE"));
+ *
+ * }
+ */
+@Experimental(Experimental.Kind.SOURCE_SINK)
+public class RabbitMqIO {
+
+  public static Read read() {
+return new AutoValue_RabbitMqIO_Read.Builder()
+.setConnectionConfig(ConnectionConfig.create()).setQueueDeclare(false)
+
.setMaxReadTime(null).setMaxNumRecords(Long.MAX_VALUE).setUseCorrelationId(false).build();
+  }
+
+  public static Write write() {
+return new AutoValue_RabbitMqIO_Write.Builder()
+.setConnectionConfig(ConnectionConfig.create())
+.setExchangeDeclare(false).build();
+  }
+
+  private RabbitMqIO() {
+  }
+
+  /**
+   * Describe a connection configuration to a RabbitMQ server.
+   */
+  @AutoValue
+  public abstract static class ConnectionConfig implements Serializable {
+
+@Nullable abstract String uri();
+
+abstract int networkRecoveryInterval();
+abstract boolean automaticRecovery();
+abstract boolean topologyRecovery();
+
+abstract int connectionTimeout();
+abstract int requestedChannelMax();
+abstract int requestedFrameMax();
+abstract int requestedHeartbeat();
+
+abstract Builder builder();
+
+

[jira] [Work logged] (BEAM-1240) Create RabbitMqIO

2018-04-03 Thread ASF GitHub Bot (JIRA)


 [ 
https://issues.apache.org/jira/browse/BEAM-1240?focusedWorklogId=87103&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-87103
 ]

ASF GitHub Bot logged work on BEAM-1240:


Author: ASF GitHub Bot
Created on: 03/Apr/18 14:33
Start Date: 03/Apr/18 14:33
Worklog Time Spent: 10m 
  Work Description: jbonofre commented on a change in pull request #1729: 
[BEAM-1240] Create RabbitMqIO
URL: https://github.com/apache/beam/pull/1729#discussion_r178845148
 
 

 ##
 File path: 
sdks/java/io/rabbitmq/src/main/java/org/apache/beam/sdk/io/rabbitmq/RabbitMqIO.java
 ##
 @@ -0,0 +1,711 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.io.rabbitmq;
+
+import static com.google.common.base.Preconditions.checkArgument;
+
+import com.google.auto.value.AutoValue;
+import com.rabbitmq.client.AMQP;
+import com.rabbitmq.client.Channel;
+import com.rabbitmq.client.Connection;
+import com.rabbitmq.client.ConnectionFactory;
+import com.rabbitmq.client.QueueingConsumer;
+import java.io.IOException;
+import java.io.Serializable;
+import java.net.URISyntaxException;
+import java.security.KeyManagementException;
+import java.security.NoSuchAlgorithmException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.NoSuchElementException;
+import javax.annotation.Nullable;
+import org.apache.beam.sdk.annotations.Experimental;
+import org.apache.beam.sdk.coders.Coder;
+import org.apache.beam.sdk.coders.SerializableCoder;
+import org.apache.beam.sdk.io.UnboundedSource;
+import org.apache.beam.sdk.options.PipelineOptions;
+import org.apache.beam.sdk.transforms.DoFn;
+import org.apache.beam.sdk.transforms.PTransform;
+import org.apache.beam.sdk.transforms.ParDo;
+import org.apache.beam.sdk.values.PBegin;
+import org.apache.beam.sdk.values.PCollection;
+import org.apache.beam.sdk.values.PDone;
+import org.joda.time.Duration;
+import org.joda.time.Instant;
+
+/**
+ * A IO to publish or consume messages with a RabbitMQ broker.
+ *
+ * Consuming messages from RabbitMQ server
+ *
+ * {@link RabbitMqIO} {@link Read} returns an unbounded {@link PCollection} 
containing RabbitMQ
+ * messages body (as {@code byte[]}).
+ *
+ * To configure a RabbitMQ source, you have to provide a RabbitMQ {@code 
URI} to connect
+ * to a RabbitMQ broker. The following example illustrates various options for 
configuring the
+ * source:
+ *
+ * {@code
+ *
+ *  pipeline.apply(
+ *
RabbitMqIO.read().withUri("amqp://user:password@localhost:5672").withQueue("QUEUE")
+ *
+ * }
+ *
+ * Publishing messages to RabbitMQ server
+ *
+ * {@link RabbitMqIO} {@link Write} can send {@code byte[]} to a RabbitMQ 
server queue.
+ *
+ * As for the {@link Read}, the {@link Write} is configured with a RabbitMQ
+ * {@link ConnectionConfig}.
+ *
+ * For instance:
+ *
+ * {@code
+ *
+ *  pipeline
+ *.apply(...) // provide PCollection
+ *
.apply(RabbitMqIO.write().withUri("amqp://user:password@localhost:5672").withQueue("QUEUE"));
+ *
+ * }
+ */
+@Experimental(Experimental.Kind.SOURCE_SINK)
+public class RabbitMqIO {
+
+  public static Read read() {
+return new AutoValue_RabbitMqIO_Read.Builder()
+.setConnectionConfig(ConnectionConfig.create()).setQueueDeclare(false)
+
.setMaxReadTime(null).setMaxNumRecords(Long.MAX_VALUE).setUseCorrelationId(false).build();
+  }
+
+  public static Write write() {
+return new AutoValue_RabbitMqIO_Write.Builder()
+.setConnectionConfig(ConnectionConfig.create())
+.setExchangeDeclare(false).build();
+  }
+
+  private RabbitMqIO() {
+  }
+
+  /**
+   * Describe a connection configuration to a RabbitMQ server.
+   */
+  @AutoValue
+  public abstract static class ConnectionConfig implements Serializable {
+
+@Nullable abstract String uri();
+
+abstract int networkRecoveryInterval();
+abstract boolean automaticRecovery();
+abstract boolean topologyRecovery();
+
+abstract int connectionTimeout();
+abstract int requestedChannelMax();
+abstract int requestedFrameMax();
+abstract int requestedHeartbeat();
+
+abstract Builder builder();
+
+

[jira] [Work logged] (BEAM-1240) Create RabbitMqIO

2018-04-03 Thread ASF GitHub Bot (JIRA)


 [ 
https://issues.apache.org/jira/browse/BEAM-1240?focusedWorklogId=87070&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-87070
 ]

ASF GitHub Bot logged work on BEAM-1240:


Author: ASF GitHub Bot
Created on: 03/Apr/18 13:03
Start Date: 03/Apr/18 13:03
Worklog Time Spent: 10m 
  Work Description: jbonofre commented on a change in pull request #1729: 
[BEAM-1240] Create RabbitMqIO
URL: https://github.com/apache/beam/pull/1729#discussion_r178815361
 
 

 ##
 File path: 
sdks/java/io/rabbitmq/src/main/java/org/apache/beam/sdk/io/rabbitmq/RabbitMqIO.java
 ##
 @@ -0,0 +1,711 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.io.rabbitmq;
+
+import static com.google.common.base.Preconditions.checkArgument;
+
+import com.google.auto.value.AutoValue;
+import com.rabbitmq.client.AMQP;
+import com.rabbitmq.client.Channel;
+import com.rabbitmq.client.Connection;
+import com.rabbitmq.client.ConnectionFactory;
+import com.rabbitmq.client.QueueingConsumer;
+import java.io.IOException;
+import java.io.Serializable;
+import java.net.URISyntaxException;
+import java.security.KeyManagementException;
+import java.security.NoSuchAlgorithmException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.NoSuchElementException;
+import javax.annotation.Nullable;
+import org.apache.beam.sdk.annotations.Experimental;
+import org.apache.beam.sdk.coders.Coder;
+import org.apache.beam.sdk.coders.SerializableCoder;
+import org.apache.beam.sdk.io.UnboundedSource;
+import org.apache.beam.sdk.options.PipelineOptions;
+import org.apache.beam.sdk.transforms.DoFn;
+import org.apache.beam.sdk.transforms.PTransform;
+import org.apache.beam.sdk.transforms.ParDo;
+import org.apache.beam.sdk.values.PBegin;
+import org.apache.beam.sdk.values.PCollection;
+import org.apache.beam.sdk.values.PDone;
+import org.joda.time.Duration;
+import org.joda.time.Instant;
+
+/**
+ * A IO to publish or consume messages with a RabbitMQ broker.
+ *
+ * Consuming messages from RabbitMQ server
+ *
+ * {@link RabbitMqIO} {@link Read} returns an unbounded {@link PCollection} 
containing RabbitMQ
+ * messages body (as {@code byte[]}).
+ *
+ * To configure a RabbitMQ source, you have to provide a RabbitMQ {@code 
URI} to connect
+ * to a RabbitMQ broker. The following example illustrates various options for 
configuring the
+ * source:
+ *
+ * {@code
+ *
+ *  pipeline.apply(
+ *
RabbitMqIO.read().withUri("amqp://user:password@localhost:5672").withQueue("QUEUE")
+ *
+ * }
+ *
+ * Publishing messages to RabbitMQ server
+ *
+ * {@link RabbitMqIO} {@link Write} can send {@code byte[]} to a RabbitMQ 
server queue.
+ *
+ * As for the {@link Read}, the {@link Write} is configured with a RabbitMQ
+ * {@link ConnectionConfig}.
+ *
+ * For instance:
+ *
+ * {@code
+ *
+ *  pipeline
+ *.apply(...) // provide PCollection
+ *
.apply(RabbitMqIO.write().withUri("amqp://user:password@localhost:5672").withQueue("QUEUE"));
+ *
+ * }
+ */
+@Experimental(Experimental.Kind.SOURCE_SINK)
+public class RabbitMqIO {
+
+  public static Read read() {
+return new AutoValue_RabbitMqIO_Read.Builder()
+.setConnectionConfig(ConnectionConfig.create()).setQueueDeclare(false)
+
.setMaxReadTime(null).setMaxNumRecords(Long.MAX_VALUE).setUseCorrelationId(false).build();
+  }
+
+  public static Write write() {
+return new AutoValue_RabbitMqIO_Write.Builder()
+.setConnectionConfig(ConnectionConfig.create())
+.setExchangeDeclare(false).build();
+  }
+
+  private RabbitMqIO() {
+  }
+
+  /**
+   * Describe a connection configuration to a RabbitMQ server.
+   */
+  @AutoValue
+  public abstract static class ConnectionConfig implements Serializable {
+
+@Nullable abstract String uri();
+
+abstract int networkRecoveryInterval();
+abstract boolean automaticRecovery();
+abstract boolean topologyRecovery();
+
+abstract int connectionTimeout();
+abstract int requestedChannelMax();
+abstract int requestedFrameMax();
+abstract int requestedHeartbeat();
+
+abstract Builder builder();
+
+

[jira] [Work logged] (BEAM-1240) Create RabbitMqIO

2018-03-09 Thread ASF GitHub Bot (JIRA)


 [ 
https://issues.apache.org/jira/browse/BEAM-1240?focusedWorklogId=79106&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-79106
 ]

ASF GitHub Bot logged work on BEAM-1240:


Author: ASF GitHub Bot
Created on: 09/Mar/18 22:02
Start Date: 09/Mar/18 22:02
Worklog Time Spent: 10m 
  Work Description: jkff commented on a change in pull request #1729: 
[BEAM-1240] Create RabbitMqIO
URL: https://github.com/apache/beam/pull/1729#discussion_r173577413
 
 

 ##
 File path: 
sdks/java/io/rabbitmq/src/main/java/org/apache/beam/sdk/io/rabbitmq/RabbitMqIO.java
 ##
 @@ -0,0 +1,711 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.io.rabbitmq;
+
+import static com.google.common.base.Preconditions.checkArgument;
+
+import com.google.auto.value.AutoValue;
+import com.rabbitmq.client.AMQP;
+import com.rabbitmq.client.Channel;
+import com.rabbitmq.client.Connection;
+import com.rabbitmq.client.ConnectionFactory;
+import com.rabbitmq.client.QueueingConsumer;
+import java.io.IOException;
+import java.io.Serializable;
+import java.net.URISyntaxException;
+import java.security.KeyManagementException;
+import java.security.NoSuchAlgorithmException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.NoSuchElementException;
+import javax.annotation.Nullable;
+import org.apache.beam.sdk.annotations.Experimental;
+import org.apache.beam.sdk.coders.Coder;
+import org.apache.beam.sdk.coders.SerializableCoder;
+import org.apache.beam.sdk.io.UnboundedSource;
+import org.apache.beam.sdk.options.PipelineOptions;
+import org.apache.beam.sdk.transforms.DoFn;
+import org.apache.beam.sdk.transforms.PTransform;
+import org.apache.beam.sdk.transforms.ParDo;
+import org.apache.beam.sdk.values.PBegin;
+import org.apache.beam.sdk.values.PCollection;
+import org.apache.beam.sdk.values.PDone;
+import org.joda.time.Duration;
+import org.joda.time.Instant;
+
+/**
+ * A IO to publish or consume messages with a RabbitMQ broker.
+ *
+ * Consuming messages from RabbitMQ server
+ *
+ * {@link RabbitMqIO} {@link Read} returns an unbounded {@link PCollection} 
containing RabbitMQ
+ * messages body (as {@code byte[]}).
+ *
+ * To configure a RabbitMQ source, you have to provide a RabbitMQ {@code 
URI} to connect
+ * to a RabbitMQ broker. The following example illustrates various options for 
configuring the
+ * source:
+ *
+ * {@code
+ *
+ *  pipeline.apply(
+ *
RabbitMqIO.read().withUri("amqp://user:password@localhost:5672").withQueue("QUEUE")
+ *
+ * }
+ *
+ * Publishing messages to RabbitMQ server
+ *
+ * {@link RabbitMqIO} {@link Write} can send {@code byte[]} to a RabbitMQ 
server queue.
+ *
+ * As for the {@link Read}, the {@link Write} is configured with a RabbitMQ
+ * {@link ConnectionConfig}.
+ *
+ * For instance:
+ *
+ * {@code
+ *
+ *  pipeline
+ *.apply(...) // provide PCollection
+ *
.apply(RabbitMqIO.write().withUri("amqp://user:password@localhost:5672").withQueue("QUEUE"));
+ *
+ * }
+ */
+@Experimental(Experimental.Kind.SOURCE_SINK)
+public class RabbitMqIO {
+
+  public static Read read() {
+return new AutoValue_RabbitMqIO_Read.Builder()
+.setConnectionConfig(ConnectionConfig.create()).setQueueDeclare(false)
+
.setMaxReadTime(null).setMaxNumRecords(Long.MAX_VALUE).setUseCorrelationId(false).build();
+  }
+
+  public static Write write() {
+return new AutoValue_RabbitMqIO_Write.Builder()
+.setConnectionConfig(ConnectionConfig.create())
+.setExchangeDeclare(false).build();
+  }
+
+  private RabbitMqIO() {
+  }
+
+  /**
+   * Describe a connection configuration to a RabbitMQ server.
+   */
+  @AutoValue
+  public abstract static class ConnectionConfig implements Serializable {
+
+@Nullable abstract String uri();
+
+abstract int networkRecoveryInterval();
+abstract boolean automaticRecovery();
+abstract boolean topologyRecovery();
+
+abstract int connectionTimeout();
+abstract int requestedChannelMax();
+abstract int requestedFrameMax();
+abstract int requestedHeartbeat();
+
+abstract Builder builder();
+
+@Aut

[jira] [Work logged] (BEAM-1240) Create RabbitMqIO

2018-03-09 Thread ASF GitHub Bot (JIRA)


 [ 
https://issues.apache.org/jira/browse/BEAM-1240?focusedWorklogId=79103&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-79103
 ]

ASF GitHub Bot logged work on BEAM-1240:


Author: ASF GitHub Bot
Created on: 09/Mar/18 22:02
Start Date: 09/Mar/18 22:02
Worklog Time Spent: 10m 
  Work Description: jkff commented on a change in pull request #1729: 
[BEAM-1240] Create RabbitMqIO
URL: https://github.com/apache/beam/pull/1729#discussion_r173575019
 
 

 ##
 File path: 
sdks/java/io/rabbitmq/src/main/java/org/apache/beam/sdk/io/rabbitmq/RabbitMqIO.java
 ##
 @@ -0,0 +1,711 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.io.rabbitmq;
+
+import static com.google.common.base.Preconditions.checkArgument;
+
+import com.google.auto.value.AutoValue;
+import com.rabbitmq.client.AMQP;
+import com.rabbitmq.client.Channel;
+import com.rabbitmq.client.Connection;
+import com.rabbitmq.client.ConnectionFactory;
+import com.rabbitmq.client.QueueingConsumer;
+import java.io.IOException;
+import java.io.Serializable;
+import java.net.URISyntaxException;
+import java.security.KeyManagementException;
+import java.security.NoSuchAlgorithmException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.NoSuchElementException;
+import javax.annotation.Nullable;
+import org.apache.beam.sdk.annotations.Experimental;
+import org.apache.beam.sdk.coders.Coder;
+import org.apache.beam.sdk.coders.SerializableCoder;
+import org.apache.beam.sdk.io.UnboundedSource;
+import org.apache.beam.sdk.options.PipelineOptions;
+import org.apache.beam.sdk.transforms.DoFn;
+import org.apache.beam.sdk.transforms.PTransform;
+import org.apache.beam.sdk.transforms.ParDo;
+import org.apache.beam.sdk.values.PBegin;
+import org.apache.beam.sdk.values.PCollection;
+import org.apache.beam.sdk.values.PDone;
+import org.joda.time.Duration;
+import org.joda.time.Instant;
+
+/**
+ * A IO to publish or consume messages with a RabbitMQ broker.
+ *
+ * Consuming messages from RabbitMQ server
+ *
+ * {@link RabbitMqIO} {@link Read} returns an unbounded {@link PCollection} 
containing RabbitMQ
+ * messages body (as {@code byte[]}).
+ *
+ * To configure a RabbitMQ source, you have to provide a RabbitMQ {@code 
URI} to connect
+ * to a RabbitMQ broker. The following example illustrates various options for 
configuring the
+ * source:
+ *
+ * {@code
+ *
+ *  pipeline.apply(
+ *
RabbitMqIO.read().withUri("amqp://user:password@localhost:5672").withQueue("QUEUE")
+ *
+ * }
+ *
+ * Publishing messages to RabbitMQ server
+ *
+ * {@link RabbitMqIO} {@link Write} can send {@code byte[]} to a RabbitMQ 
server queue.
+ *
+ * As for the {@link Read}, the {@link Write} is configured with a RabbitMQ
+ * {@link ConnectionConfig}.
+ *
+ * For instance:
+ *
+ * {@code
+ *
+ *  pipeline
+ *.apply(...) // provide PCollection
+ *
.apply(RabbitMqIO.write().withUri("amqp://user:password@localhost:5672").withQueue("QUEUE"));
+ *
+ * }
+ */
+@Experimental(Experimental.Kind.SOURCE_SINK)
+public class RabbitMqIO {
+
+  public static Read read() {
+return new AutoValue_RabbitMqIO_Read.Builder()
+.setConnectionConfig(ConnectionConfig.create()).setQueueDeclare(false)
+
.setMaxReadTime(null).setMaxNumRecords(Long.MAX_VALUE).setUseCorrelationId(false).build();
+  }
+
+  public static Write write() {
+return new AutoValue_RabbitMqIO_Write.Builder()
+.setConnectionConfig(ConnectionConfig.create())
+.setExchangeDeclare(false).build();
+  }
+
+  private RabbitMqIO() {
+  }
+
+  /**
+   * Describe a connection configuration to a RabbitMQ server.
+   */
+  @AutoValue
+  public abstract static class ConnectionConfig implements Serializable {
+
+@Nullable abstract String uri();
+
+abstract int networkRecoveryInterval();
+abstract boolean automaticRecovery();
+abstract boolean topologyRecovery();
+
+abstract int connectionTimeout();
+abstract int requestedChannelMax();
+abstract int requestedFrameMax();
+abstract int requestedHeartbeat();
+
+abstract Builder builder();
+
+@Aut

[jira] [Work logged] (BEAM-1240) Create RabbitMqIO

2018-03-09 Thread ASF GitHub Bot (JIRA)


 [ 
https://issues.apache.org/jira/browse/BEAM-1240?focusedWorklogId=79098&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-79098
 ]

ASF GitHub Bot logged work on BEAM-1240:


Author: ASF GitHub Bot
Created on: 09/Mar/18 22:02
Start Date: 09/Mar/18 22:02
Worklog Time Spent: 10m 
  Work Description: jkff commented on a change in pull request #1729: 
[BEAM-1240] Create RabbitMqIO
URL: https://github.com/apache/beam/pull/1729#discussion_r173574620
 
 

 ##
 File path: 
sdks/java/io/rabbitmq/src/main/java/org/apache/beam/sdk/io/rabbitmq/RabbitMqIO.java
 ##
 @@ -0,0 +1,711 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.io.rabbitmq;
+
+import static com.google.common.base.Preconditions.checkArgument;
+
+import com.google.auto.value.AutoValue;
+import com.rabbitmq.client.AMQP;
+import com.rabbitmq.client.Channel;
+import com.rabbitmq.client.Connection;
+import com.rabbitmq.client.ConnectionFactory;
+import com.rabbitmq.client.QueueingConsumer;
+import java.io.IOException;
+import java.io.Serializable;
+import java.net.URISyntaxException;
+import java.security.KeyManagementException;
+import java.security.NoSuchAlgorithmException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.NoSuchElementException;
+import javax.annotation.Nullable;
+import org.apache.beam.sdk.annotations.Experimental;
+import org.apache.beam.sdk.coders.Coder;
+import org.apache.beam.sdk.coders.SerializableCoder;
+import org.apache.beam.sdk.io.UnboundedSource;
+import org.apache.beam.sdk.options.PipelineOptions;
+import org.apache.beam.sdk.transforms.DoFn;
+import org.apache.beam.sdk.transforms.PTransform;
+import org.apache.beam.sdk.transforms.ParDo;
+import org.apache.beam.sdk.values.PBegin;
+import org.apache.beam.sdk.values.PCollection;
+import org.apache.beam.sdk.values.PDone;
+import org.joda.time.Duration;
+import org.joda.time.Instant;
+
+/**
+ * A IO to publish or consume messages with a RabbitMQ broker.
+ *
+ * Consuming messages from RabbitMQ server
+ *
+ * {@link RabbitMqIO} {@link Read} returns an unbounded {@link PCollection} 
containing RabbitMQ
+ * messages body (as {@code byte[]}).
+ *
+ * To configure a RabbitMQ source, you have to provide a RabbitMQ {@code 
URI} to connect
+ * to a RabbitMQ broker. The following example illustrates various options for 
configuring the
+ * source:
+ *
+ * {@code
+ *
+ *  pipeline.apply(
+ *
RabbitMqIO.read().withUri("amqp://user:password@localhost:5672").withQueue("QUEUE")
+ *
+ * }
+ *
+ * Publishing messages to RabbitMQ server
+ *
+ * {@link RabbitMqIO} {@link Write} can send {@code byte[]} to a RabbitMQ 
server queue.
+ *
+ * As for the {@link Read}, the {@link Write} is configured with a RabbitMQ
+ * {@link ConnectionConfig}.
+ *
+ * For instance:
+ *
+ * {@code
+ *
+ *  pipeline
+ *.apply(...) // provide PCollection
+ *
.apply(RabbitMqIO.write().withUri("amqp://user:password@localhost:5672").withQueue("QUEUE"));
+ *
+ * }
+ */
+@Experimental(Experimental.Kind.SOURCE_SINK)
+public class RabbitMqIO {
+
+  public static Read read() {
+return new AutoValue_RabbitMqIO_Read.Builder()
+.setConnectionConfig(ConnectionConfig.create()).setQueueDeclare(false)
+
.setMaxReadTime(null).setMaxNumRecords(Long.MAX_VALUE).setUseCorrelationId(false).build();
+  }
+
+  public static Write write() {
+return new AutoValue_RabbitMqIO_Write.Builder()
+.setConnectionConfig(ConnectionConfig.create())
+.setExchangeDeclare(false).build();
+  }
+
+  private RabbitMqIO() {
+  }
+
+  /**
+   * Describe a connection configuration to a RabbitMQ server.
+   */
+  @AutoValue
+  public abstract static class ConnectionConfig implements Serializable {
+
+@Nullable abstract String uri();
+
+abstract int networkRecoveryInterval();
+abstract boolean automaticRecovery();
+abstract boolean topologyRecovery();
+
+abstract int connectionTimeout();
+abstract int requestedChannelMax();
+abstract int requestedFrameMax();
+abstract int requestedHeartbeat();
+
+abstract Builder builder();
+
+@Aut

[jira] [Work logged] (BEAM-1240) Create RabbitMqIO

2018-03-09 Thread ASF GitHub Bot (JIRA)


 [ 
https://issues.apache.org/jira/browse/BEAM-1240?focusedWorklogId=79104&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-79104
 ]

ASF GitHub Bot logged work on BEAM-1240:


Author: ASF GitHub Bot
Created on: 09/Mar/18 22:02
Start Date: 09/Mar/18 22:02
Worklog Time Spent: 10m 
  Work Description: jkff commented on a change in pull request #1729: 
[BEAM-1240] Create RabbitMqIO
URL: https://github.com/apache/beam/pull/1729#discussion_r173575323
 
 

 ##
 File path: 
sdks/java/io/rabbitmq/src/main/java/org/apache/beam/sdk/io/rabbitmq/RabbitMqIO.java
 ##
 @@ -0,0 +1,711 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.io.rabbitmq;
+
+import static com.google.common.base.Preconditions.checkArgument;
+
+import com.google.auto.value.AutoValue;
+import com.rabbitmq.client.AMQP;
+import com.rabbitmq.client.Channel;
+import com.rabbitmq.client.Connection;
+import com.rabbitmq.client.ConnectionFactory;
+import com.rabbitmq.client.QueueingConsumer;
+import java.io.IOException;
+import java.io.Serializable;
+import java.net.URISyntaxException;
+import java.security.KeyManagementException;
+import java.security.NoSuchAlgorithmException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.NoSuchElementException;
+import javax.annotation.Nullable;
+import org.apache.beam.sdk.annotations.Experimental;
+import org.apache.beam.sdk.coders.Coder;
+import org.apache.beam.sdk.coders.SerializableCoder;
+import org.apache.beam.sdk.io.UnboundedSource;
+import org.apache.beam.sdk.options.PipelineOptions;
+import org.apache.beam.sdk.transforms.DoFn;
+import org.apache.beam.sdk.transforms.PTransform;
+import org.apache.beam.sdk.transforms.ParDo;
+import org.apache.beam.sdk.values.PBegin;
+import org.apache.beam.sdk.values.PCollection;
+import org.apache.beam.sdk.values.PDone;
+import org.joda.time.Duration;
+import org.joda.time.Instant;
+
+/**
+ * A IO to publish or consume messages with a RabbitMQ broker.
+ *
+ * Consuming messages from RabbitMQ server
+ *
+ * {@link RabbitMqIO} {@link Read} returns an unbounded {@link PCollection} 
containing RabbitMQ
+ * messages body (as {@code byte[]}).
+ *
+ * To configure a RabbitMQ source, you have to provide a RabbitMQ {@code 
URI} to connect
+ * to a RabbitMQ broker. The following example illustrates various options for 
configuring the
+ * source:
+ *
+ * {@code
+ *
+ *  pipeline.apply(
+ *
RabbitMqIO.read().withUri("amqp://user:password@localhost:5672").withQueue("QUEUE")
+ *
+ * }
+ *
+ * Publishing messages to RabbitMQ server
+ *
+ * {@link RabbitMqIO} {@link Write} can send {@code byte[]} to a RabbitMQ 
server queue.
+ *
+ * As for the {@link Read}, the {@link Write} is configured with a RabbitMQ
+ * {@link ConnectionConfig}.
+ *
+ * For instance:
+ *
+ * {@code
+ *
+ *  pipeline
+ *.apply(...) // provide PCollection
+ *
.apply(RabbitMqIO.write().withUri("amqp://user:password@localhost:5672").withQueue("QUEUE"));
+ *
+ * }
+ */
+@Experimental(Experimental.Kind.SOURCE_SINK)
+public class RabbitMqIO {
+
+  public static Read read() {
+return new AutoValue_RabbitMqIO_Read.Builder()
+.setConnectionConfig(ConnectionConfig.create()).setQueueDeclare(false)
+
.setMaxReadTime(null).setMaxNumRecords(Long.MAX_VALUE).setUseCorrelationId(false).build();
+  }
+
+  public static Write write() {
+return new AutoValue_RabbitMqIO_Write.Builder()
+.setConnectionConfig(ConnectionConfig.create())
+.setExchangeDeclare(false).build();
+  }
+
+  private RabbitMqIO() {
+  }
+
+  /**
+   * Describe a connection configuration to a RabbitMQ server.
+   */
+  @AutoValue
+  public abstract static class ConnectionConfig implements Serializable {
+
+@Nullable abstract String uri();
+
+abstract int networkRecoveryInterval();
+abstract boolean automaticRecovery();
+abstract boolean topologyRecovery();
+
+abstract int connectionTimeout();
+abstract int requestedChannelMax();
+abstract int requestedFrameMax();
+abstract int requestedHeartbeat();
+
+abstract Builder builder();
+
+@Aut

[jira] [Work logged] (BEAM-1240) Create RabbitMqIO

2018-03-09 Thread ASF GitHub Bot (JIRA)


 [ 
https://issues.apache.org/jira/browse/BEAM-1240?focusedWorklogId=79101&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-79101
 ]

ASF GitHub Bot logged work on BEAM-1240:


Author: ASF GitHub Bot
Created on: 09/Mar/18 22:02
Start Date: 09/Mar/18 22:02
Worklog Time Spent: 10m 
  Work Description: jkff commented on a change in pull request #1729: 
[BEAM-1240] Create RabbitMqIO
URL: https://github.com/apache/beam/pull/1729#discussion_r173575420
 
 

 ##
 File path: 
sdks/java/io/rabbitmq/src/main/java/org/apache/beam/sdk/io/rabbitmq/RabbitMqIO.java
 ##
 @@ -0,0 +1,711 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.io.rabbitmq;
+
+import static com.google.common.base.Preconditions.checkArgument;
+
+import com.google.auto.value.AutoValue;
+import com.rabbitmq.client.AMQP;
+import com.rabbitmq.client.Channel;
+import com.rabbitmq.client.Connection;
+import com.rabbitmq.client.ConnectionFactory;
+import com.rabbitmq.client.QueueingConsumer;
+import java.io.IOException;
+import java.io.Serializable;
+import java.net.URISyntaxException;
+import java.security.KeyManagementException;
+import java.security.NoSuchAlgorithmException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.NoSuchElementException;
+import javax.annotation.Nullable;
+import org.apache.beam.sdk.annotations.Experimental;
+import org.apache.beam.sdk.coders.Coder;
+import org.apache.beam.sdk.coders.SerializableCoder;
+import org.apache.beam.sdk.io.UnboundedSource;
+import org.apache.beam.sdk.options.PipelineOptions;
+import org.apache.beam.sdk.transforms.DoFn;
+import org.apache.beam.sdk.transforms.PTransform;
+import org.apache.beam.sdk.transforms.ParDo;
+import org.apache.beam.sdk.values.PBegin;
+import org.apache.beam.sdk.values.PCollection;
+import org.apache.beam.sdk.values.PDone;
+import org.joda.time.Duration;
+import org.joda.time.Instant;
+
+/**
+ * A IO to publish or consume messages with a RabbitMQ broker.
+ *
+ * Consuming messages from RabbitMQ server
+ *
+ * {@link RabbitMqIO} {@link Read} returns an unbounded {@link PCollection} 
containing RabbitMQ
+ * messages body (as {@code byte[]}).
+ *
+ * To configure a RabbitMQ source, you have to provide a RabbitMQ {@code 
URI} to connect
+ * to a RabbitMQ broker. The following example illustrates various options for 
configuring the
+ * source:
+ *
+ * {@code
+ *
+ *  pipeline.apply(
+ *
RabbitMqIO.read().withUri("amqp://user:password@localhost:5672").withQueue("QUEUE")
+ *
+ * }
+ *
+ * Publishing messages to RabbitMQ server
+ *
+ * {@link RabbitMqIO} {@link Write} can send {@code byte[]} to a RabbitMQ 
server queue.
+ *
+ * As for the {@link Read}, the {@link Write} is configured with a RabbitMQ
+ * {@link ConnectionConfig}.
+ *
+ * For instance:
+ *
+ * {@code
+ *
+ *  pipeline
+ *.apply(...) // provide PCollection
+ *
.apply(RabbitMqIO.write().withUri("amqp://user:password@localhost:5672").withQueue("QUEUE"));
+ *
+ * }
+ */
+@Experimental(Experimental.Kind.SOURCE_SINK)
+public class RabbitMqIO {
+
+  public static Read read() {
+return new AutoValue_RabbitMqIO_Read.Builder()
+.setConnectionConfig(ConnectionConfig.create()).setQueueDeclare(false)
+
.setMaxReadTime(null).setMaxNumRecords(Long.MAX_VALUE).setUseCorrelationId(false).build();
+  }
+
+  public static Write write() {
+return new AutoValue_RabbitMqIO_Write.Builder()
+.setConnectionConfig(ConnectionConfig.create())
+.setExchangeDeclare(false).build();
+  }
+
+  private RabbitMqIO() {
+  }
+
+  /**
+   * Describe a connection configuration to a RabbitMQ server.
+   */
+  @AutoValue
+  public abstract static class ConnectionConfig implements Serializable {
+
+@Nullable abstract String uri();
+
+abstract int networkRecoveryInterval();
+abstract boolean automaticRecovery();
+abstract boolean topologyRecovery();
+
+abstract int connectionTimeout();
+abstract int requestedChannelMax();
+abstract int requestedFrameMax();
+abstract int requestedHeartbeat();
+
+abstract Builder builder();
+
+@Aut

[jira] [Work logged] (BEAM-1240) Create RabbitMqIO

2018-03-09 Thread ASF GitHub Bot (JIRA)


 [ 
https://issues.apache.org/jira/browse/BEAM-1240?focusedWorklogId=79108&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-79108
 ]

ASF GitHub Bot logged work on BEAM-1240:


Author: ASF GitHub Bot
Created on: 09/Mar/18 22:02
Start Date: 09/Mar/18 22:02
Worklog Time Spent: 10m 
  Work Description: jkff commented on a change in pull request #1729: 
[BEAM-1240] Create RabbitMqIO
URL: https://github.com/apache/beam/pull/1729#discussion_r173576028
 
 

 ##
 File path: 
sdks/java/io/rabbitmq/src/main/java/org/apache/beam/sdk/io/rabbitmq/RabbitMqIO.java
 ##
 @@ -0,0 +1,711 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.io.rabbitmq;
+
+import static com.google.common.base.Preconditions.checkArgument;
+
+import com.google.auto.value.AutoValue;
+import com.rabbitmq.client.AMQP;
+import com.rabbitmq.client.Channel;
+import com.rabbitmq.client.Connection;
+import com.rabbitmq.client.ConnectionFactory;
+import com.rabbitmq.client.QueueingConsumer;
+import java.io.IOException;
+import java.io.Serializable;
+import java.net.URISyntaxException;
+import java.security.KeyManagementException;
+import java.security.NoSuchAlgorithmException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.NoSuchElementException;
+import javax.annotation.Nullable;
+import org.apache.beam.sdk.annotations.Experimental;
+import org.apache.beam.sdk.coders.Coder;
+import org.apache.beam.sdk.coders.SerializableCoder;
+import org.apache.beam.sdk.io.UnboundedSource;
+import org.apache.beam.sdk.options.PipelineOptions;
+import org.apache.beam.sdk.transforms.DoFn;
+import org.apache.beam.sdk.transforms.PTransform;
+import org.apache.beam.sdk.transforms.ParDo;
+import org.apache.beam.sdk.values.PBegin;
+import org.apache.beam.sdk.values.PCollection;
+import org.apache.beam.sdk.values.PDone;
+import org.joda.time.Duration;
+import org.joda.time.Instant;
+
+/**
+ * A IO to publish or consume messages with a RabbitMQ broker.
+ *
+ * Consuming messages from RabbitMQ server
+ *
+ * {@link RabbitMqIO} {@link Read} returns an unbounded {@link PCollection} 
containing RabbitMQ
+ * messages body (as {@code byte[]}).
+ *
+ * To configure a RabbitMQ source, you have to provide a RabbitMQ {@code 
URI} to connect
+ * to a RabbitMQ broker. The following example illustrates various options for 
configuring the
+ * source:
+ *
+ * {@code
+ *
+ *  pipeline.apply(
+ *
RabbitMqIO.read().withUri("amqp://user:password@localhost:5672").withQueue("QUEUE")
+ *
+ * }
+ *
+ * Publishing messages to RabbitMQ server
+ *
+ * {@link RabbitMqIO} {@link Write} can send {@code byte[]} to a RabbitMQ 
server queue.
+ *
+ * As for the {@link Read}, the {@link Write} is configured with a RabbitMQ
+ * {@link ConnectionConfig}.
+ *
+ * For instance:
+ *
+ * {@code
+ *
+ *  pipeline
+ *.apply(...) // provide PCollection
+ *
.apply(RabbitMqIO.write().withUri("amqp://user:password@localhost:5672").withQueue("QUEUE"));
+ *
+ * }
+ */
+@Experimental(Experimental.Kind.SOURCE_SINK)
+public class RabbitMqIO {
+
+  public static Read read() {
+return new AutoValue_RabbitMqIO_Read.Builder()
+.setConnectionConfig(ConnectionConfig.create()).setQueueDeclare(false)
+
.setMaxReadTime(null).setMaxNumRecords(Long.MAX_VALUE).setUseCorrelationId(false).build();
+  }
+
+  public static Write write() {
+return new AutoValue_RabbitMqIO_Write.Builder()
+.setConnectionConfig(ConnectionConfig.create())
+.setExchangeDeclare(false).build();
+  }
+
+  private RabbitMqIO() {
+  }
+
+  /**
+   * Describe a connection configuration to a RabbitMQ server.
+   */
+  @AutoValue
+  public abstract static class ConnectionConfig implements Serializable {
+
+@Nullable abstract String uri();
+
+abstract int networkRecoveryInterval();
+abstract boolean automaticRecovery();
+abstract boolean topologyRecovery();
+
+abstract int connectionTimeout();
+abstract int requestedChannelMax();
+abstract int requestedFrameMax();
+abstract int requestedHeartbeat();
+
+abstract Builder builder();
+
+@Aut

[jira] [Work logged] (BEAM-1240) Create RabbitMqIO

2018-03-09 Thread ASF GitHub Bot (JIRA)


 [ 
https://issues.apache.org/jira/browse/BEAM-1240?focusedWorklogId=79099&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-79099
 ]

ASF GitHub Bot logged work on BEAM-1240:


Author: ASF GitHub Bot
Created on: 09/Mar/18 22:02
Start Date: 09/Mar/18 22:02
Worklog Time Spent: 10m 
  Work Description: jkff commented on a change in pull request #1729: 
[BEAM-1240] Create RabbitMqIO
URL: https://github.com/apache/beam/pull/1729#discussion_r173574737
 
 

 ##
 File path: 
sdks/java/io/rabbitmq/src/main/java/org/apache/beam/sdk/io/rabbitmq/RabbitMqIO.java
 ##
 @@ -0,0 +1,711 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.io.rabbitmq;
+
+import static com.google.common.base.Preconditions.checkArgument;
+
+import com.google.auto.value.AutoValue;
+import com.rabbitmq.client.AMQP;
+import com.rabbitmq.client.Channel;
+import com.rabbitmq.client.Connection;
+import com.rabbitmq.client.ConnectionFactory;
+import com.rabbitmq.client.QueueingConsumer;
+import java.io.IOException;
+import java.io.Serializable;
+import java.net.URISyntaxException;
+import java.security.KeyManagementException;
+import java.security.NoSuchAlgorithmException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.NoSuchElementException;
+import javax.annotation.Nullable;
+import org.apache.beam.sdk.annotations.Experimental;
+import org.apache.beam.sdk.coders.Coder;
+import org.apache.beam.sdk.coders.SerializableCoder;
+import org.apache.beam.sdk.io.UnboundedSource;
+import org.apache.beam.sdk.options.PipelineOptions;
+import org.apache.beam.sdk.transforms.DoFn;
+import org.apache.beam.sdk.transforms.PTransform;
+import org.apache.beam.sdk.transforms.ParDo;
+import org.apache.beam.sdk.values.PBegin;
+import org.apache.beam.sdk.values.PCollection;
+import org.apache.beam.sdk.values.PDone;
+import org.joda.time.Duration;
+import org.joda.time.Instant;
+
+/**
+ * A IO to publish or consume messages with a RabbitMQ broker.
+ *
+ * Consuming messages from RabbitMQ server
+ *
+ * {@link RabbitMqIO} {@link Read} returns an unbounded {@link PCollection} 
containing RabbitMQ
+ * messages body (as {@code byte[]}).
+ *
+ * To configure a RabbitMQ source, you have to provide a RabbitMQ {@code 
URI} to connect
+ * to a RabbitMQ broker. The following example illustrates various options for 
configuring the
+ * source:
+ *
+ * {@code
+ *
+ *  pipeline.apply(
+ *
RabbitMqIO.read().withUri("amqp://user:password@localhost:5672").withQueue("QUEUE")
+ *
+ * }
+ *
+ * Publishing messages to RabbitMQ server
+ *
+ * {@link RabbitMqIO} {@link Write} can send {@code byte[]} to a RabbitMQ 
server queue.
+ *
+ * As for the {@link Read}, the {@link Write} is configured with a RabbitMQ
+ * {@link ConnectionConfig}.
+ *
+ * For instance:
+ *
+ * {@code
+ *
+ *  pipeline
+ *.apply(...) // provide PCollection
+ *
.apply(RabbitMqIO.write().withUri("amqp://user:password@localhost:5672").withQueue("QUEUE"));
+ *
+ * }
+ */
+@Experimental(Experimental.Kind.SOURCE_SINK)
+public class RabbitMqIO {
+
+  public static Read read() {
+return new AutoValue_RabbitMqIO_Read.Builder()
+.setConnectionConfig(ConnectionConfig.create()).setQueueDeclare(false)
+
.setMaxReadTime(null).setMaxNumRecords(Long.MAX_VALUE).setUseCorrelationId(false).build();
+  }
+
+  public static Write write() {
+return new AutoValue_RabbitMqIO_Write.Builder()
+.setConnectionConfig(ConnectionConfig.create())
+.setExchangeDeclare(false).build();
+  }
+
+  private RabbitMqIO() {
+  }
+
+  /**
+   * Describe a connection configuration to a RabbitMQ server.
+   */
+  @AutoValue
+  public abstract static class ConnectionConfig implements Serializable {
+
+@Nullable abstract String uri();
+
+abstract int networkRecoveryInterval();
+abstract boolean automaticRecovery();
+abstract boolean topologyRecovery();
+
+abstract int connectionTimeout();
+abstract int requestedChannelMax();
+abstract int requestedFrameMax();
+abstract int requestedHeartbeat();
+
+abstract Builder builder();
+
+@Aut

[jira] [Work logged] (BEAM-1240) Create RabbitMqIO

2018-03-09 Thread ASF GitHub Bot (JIRA)


 [ 
https://issues.apache.org/jira/browse/BEAM-1240?focusedWorklogId=79105&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-79105
 ]

ASF GitHub Bot logged work on BEAM-1240:


Author: ASF GitHub Bot
Created on: 09/Mar/18 22:02
Start Date: 09/Mar/18 22:02
Worklog Time Spent: 10m 
  Work Description: jkff commented on a change in pull request #1729: 
[BEAM-1240] Create RabbitMqIO
URL: https://github.com/apache/beam/pull/1729#discussion_r173576181
 
 

 ##
 File path: 
sdks/java/io/rabbitmq/src/main/java/org/apache/beam/sdk/io/rabbitmq/RabbitMqIO.java
 ##
 @@ -0,0 +1,711 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.io.rabbitmq;
+
+import static com.google.common.base.Preconditions.checkArgument;
+
+import com.google.auto.value.AutoValue;
+import com.rabbitmq.client.AMQP;
+import com.rabbitmq.client.Channel;
+import com.rabbitmq.client.Connection;
+import com.rabbitmq.client.ConnectionFactory;
+import com.rabbitmq.client.QueueingConsumer;
+import java.io.IOException;
+import java.io.Serializable;
+import java.net.URISyntaxException;
+import java.security.KeyManagementException;
+import java.security.NoSuchAlgorithmException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.NoSuchElementException;
+import javax.annotation.Nullable;
+import org.apache.beam.sdk.annotations.Experimental;
+import org.apache.beam.sdk.coders.Coder;
+import org.apache.beam.sdk.coders.SerializableCoder;
+import org.apache.beam.sdk.io.UnboundedSource;
+import org.apache.beam.sdk.options.PipelineOptions;
+import org.apache.beam.sdk.transforms.DoFn;
+import org.apache.beam.sdk.transforms.PTransform;
+import org.apache.beam.sdk.transforms.ParDo;
+import org.apache.beam.sdk.values.PBegin;
+import org.apache.beam.sdk.values.PCollection;
+import org.apache.beam.sdk.values.PDone;
+import org.joda.time.Duration;
+import org.joda.time.Instant;
+
+/**
+ * A IO to publish or consume messages with a RabbitMQ broker.
+ *
+ * Consuming messages from RabbitMQ server
+ *
+ * {@link RabbitMqIO} {@link Read} returns an unbounded {@link PCollection} 
containing RabbitMQ
+ * messages body (as {@code byte[]}).
+ *
+ * To configure a RabbitMQ source, you have to provide a RabbitMQ {@code 
URI} to connect
+ * to a RabbitMQ broker. The following example illustrates various options for 
configuring the
+ * source:
+ *
+ * {@code
+ *
+ *  pipeline.apply(
+ *
RabbitMqIO.read().withUri("amqp://user:password@localhost:5672").withQueue("QUEUE")
+ *
+ * }
+ *
+ * Publishing messages to RabbitMQ server
+ *
+ * {@link RabbitMqIO} {@link Write} can send {@code byte[]} to a RabbitMQ 
server queue.
+ *
+ * As for the {@link Read}, the {@link Write} is configured with a RabbitMQ
+ * {@link ConnectionConfig}.
+ *
+ * For instance:
+ *
+ * {@code
+ *
+ *  pipeline
+ *.apply(...) // provide PCollection
+ *
.apply(RabbitMqIO.write().withUri("amqp://user:password@localhost:5672").withQueue("QUEUE"));
+ *
+ * }
+ */
+@Experimental(Experimental.Kind.SOURCE_SINK)
+public class RabbitMqIO {
+
+  public static Read read() {
+return new AutoValue_RabbitMqIO_Read.Builder()
+.setConnectionConfig(ConnectionConfig.create()).setQueueDeclare(false)
+
.setMaxReadTime(null).setMaxNumRecords(Long.MAX_VALUE).setUseCorrelationId(false).build();
+  }
+
+  public static Write write() {
+return new AutoValue_RabbitMqIO_Write.Builder()
+.setConnectionConfig(ConnectionConfig.create())
+.setExchangeDeclare(false).build();
+  }
+
+  private RabbitMqIO() {
+  }
+
+  /**
+   * Describe a connection configuration to a RabbitMQ server.
+   */
+  @AutoValue
+  public abstract static class ConnectionConfig implements Serializable {
+
+@Nullable abstract String uri();
+
+abstract int networkRecoveryInterval();
+abstract boolean automaticRecovery();
+abstract boolean topologyRecovery();
+
+abstract int connectionTimeout();
+abstract int requestedChannelMax();
+abstract int requestedFrameMax();
+abstract int requestedHeartbeat();
+
+abstract Builder builder();
+
+@Aut

[jira] [Work logged] (BEAM-1240) Create RabbitMqIO

2018-03-09 Thread ASF GitHub Bot (JIRA)


 [ 
https://issues.apache.org/jira/browse/BEAM-1240?focusedWorklogId=79096&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-79096
 ]

ASF GitHub Bot logged work on BEAM-1240:


Author: ASF GitHub Bot
Created on: 09/Mar/18 22:02
Start Date: 09/Mar/18 22:02
Worklog Time Spent: 10m 
  Work Description: jkff commented on a change in pull request #1729: 
[BEAM-1240] Create RabbitMqIO
URL: https://github.com/apache/beam/pull/1729#discussion_r173574242
 
 

 ##
 File path: 
sdks/java/io/rabbitmq/src/main/java/org/apache/beam/sdk/io/rabbitmq/RabbitMqIO.java
 ##
 @@ -0,0 +1,711 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.io.rabbitmq;
+
+import static com.google.common.base.Preconditions.checkArgument;
+
+import com.google.auto.value.AutoValue;
+import com.rabbitmq.client.AMQP;
+import com.rabbitmq.client.Channel;
+import com.rabbitmq.client.Connection;
+import com.rabbitmq.client.ConnectionFactory;
+import com.rabbitmq.client.QueueingConsumer;
+import java.io.IOException;
+import java.io.Serializable;
+import java.net.URISyntaxException;
+import java.security.KeyManagementException;
+import java.security.NoSuchAlgorithmException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.NoSuchElementException;
+import javax.annotation.Nullable;
+import org.apache.beam.sdk.annotations.Experimental;
+import org.apache.beam.sdk.coders.Coder;
+import org.apache.beam.sdk.coders.SerializableCoder;
+import org.apache.beam.sdk.io.UnboundedSource;
+import org.apache.beam.sdk.options.PipelineOptions;
+import org.apache.beam.sdk.transforms.DoFn;
+import org.apache.beam.sdk.transforms.PTransform;
+import org.apache.beam.sdk.transforms.ParDo;
+import org.apache.beam.sdk.values.PBegin;
+import org.apache.beam.sdk.values.PCollection;
+import org.apache.beam.sdk.values.PDone;
+import org.joda.time.Duration;
+import org.joda.time.Instant;
+
+/**
+ * A IO to publish or consume messages with a RabbitMQ broker.
+ *
+ * Consuming messages from RabbitMQ server
+ *
+ * {@link RabbitMqIO} {@link Read} returns an unbounded {@link PCollection} 
containing RabbitMQ
+ * messages body (as {@code byte[]}).
+ *
+ * To configure a RabbitMQ source, you have to provide a RabbitMQ {@code 
URI} to connect
+ * to a RabbitMQ broker. The following example illustrates various options for 
configuring the
+ * source:
+ *
+ * {@code
+ *
+ *  pipeline.apply(
+ *
RabbitMqIO.read().withUri("amqp://user:password@localhost:5672").withQueue("QUEUE")
+ *
+ * }
+ *
+ * Publishing messages to RabbitMQ server
+ *
+ * {@link RabbitMqIO} {@link Write} can send {@code byte[]} to a RabbitMQ 
server queue.
+ *
+ * As for the {@link Read}, the {@link Write} is configured with a RabbitMQ
+ * {@link ConnectionConfig}.
+ *
+ * For instance:
+ *
+ * {@code
+ *
+ *  pipeline
+ *.apply(...) // provide PCollection
+ *
.apply(RabbitMqIO.write().withUri("amqp://user:password@localhost:5672").withQueue("QUEUE"));
+ *
+ * }
+ */
+@Experimental(Experimental.Kind.SOURCE_SINK)
+public class RabbitMqIO {
+
+  public static Read read() {
+return new AutoValue_RabbitMqIO_Read.Builder()
+.setConnectionConfig(ConnectionConfig.create()).setQueueDeclare(false)
+
.setMaxReadTime(null).setMaxNumRecords(Long.MAX_VALUE).setUseCorrelationId(false).build();
+  }
+
+  public static Write write() {
+return new AutoValue_RabbitMqIO_Write.Builder()
+.setConnectionConfig(ConnectionConfig.create())
+.setExchangeDeclare(false).build();
+  }
+
+  private RabbitMqIO() {
+  }
+
+  /**
+   * Describe a connection configuration to a RabbitMQ server.
+   */
+  @AutoValue
+  public abstract static class ConnectionConfig implements Serializable {
+
+@Nullable abstract String uri();
+
+abstract int networkRecoveryInterval();
+abstract boolean automaticRecovery();
+abstract boolean topologyRecovery();
+
+abstract int connectionTimeout();
+abstract int requestedChannelMax();
+abstract int requestedFrameMax();
+abstract int requestedHeartbeat();
+
+abstract Builder builder();
+
+@Aut

[jira] [Work logged] (BEAM-1240) Create RabbitMqIO

2018-03-09 Thread ASF GitHub Bot (JIRA)


 [ 
https://issues.apache.org/jira/browse/BEAM-1240?focusedWorklogId=79111&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-79111
 ]

ASF GitHub Bot logged work on BEAM-1240:


Author: ASF GitHub Bot
Created on: 09/Mar/18 22:02
Start Date: 09/Mar/18 22:02
Worklog Time Spent: 10m 
  Work Description: jkff commented on a change in pull request #1729: 
[BEAM-1240] Create RabbitMqIO
URL: https://github.com/apache/beam/pull/1729#discussion_r173575549
 
 

 ##
 File path: 
sdks/java/io/rabbitmq/src/main/java/org/apache/beam/sdk/io/rabbitmq/RabbitMqIO.java
 ##
 @@ -0,0 +1,711 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.io.rabbitmq;
+
+import static com.google.common.base.Preconditions.checkArgument;
+
+import com.google.auto.value.AutoValue;
+import com.rabbitmq.client.AMQP;
+import com.rabbitmq.client.Channel;
+import com.rabbitmq.client.Connection;
+import com.rabbitmq.client.ConnectionFactory;
+import com.rabbitmq.client.QueueingConsumer;
+import java.io.IOException;
+import java.io.Serializable;
+import java.net.URISyntaxException;
+import java.security.KeyManagementException;
+import java.security.NoSuchAlgorithmException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.NoSuchElementException;
+import javax.annotation.Nullable;
+import org.apache.beam.sdk.annotations.Experimental;
+import org.apache.beam.sdk.coders.Coder;
+import org.apache.beam.sdk.coders.SerializableCoder;
+import org.apache.beam.sdk.io.UnboundedSource;
+import org.apache.beam.sdk.options.PipelineOptions;
+import org.apache.beam.sdk.transforms.DoFn;
+import org.apache.beam.sdk.transforms.PTransform;
+import org.apache.beam.sdk.transforms.ParDo;
+import org.apache.beam.sdk.values.PBegin;
+import org.apache.beam.sdk.values.PCollection;
+import org.apache.beam.sdk.values.PDone;
+import org.joda.time.Duration;
+import org.joda.time.Instant;
+
+/**
+ * A IO to publish or consume messages with a RabbitMQ broker.
+ *
+ * Consuming messages from RabbitMQ server
+ *
+ * {@link RabbitMqIO} {@link Read} returns an unbounded {@link PCollection} 
containing RabbitMQ
+ * messages body (as {@code byte[]}).
+ *
+ * To configure a RabbitMQ source, you have to provide a RabbitMQ {@code 
URI} to connect
+ * to a RabbitMQ broker. The following example illustrates various options for 
configuring the
+ * source:
+ *
+ * {@code
+ *
+ *  pipeline.apply(
+ *
RabbitMqIO.read().withUri("amqp://user:password@localhost:5672").withQueue("QUEUE")
+ *
+ * }
+ *
+ * Publishing messages to RabbitMQ server
+ *
+ * {@link RabbitMqIO} {@link Write} can send {@code byte[]} to a RabbitMQ 
server queue.
+ *
+ * As for the {@link Read}, the {@link Write} is configured with a RabbitMQ
+ * {@link ConnectionConfig}.
+ *
+ * For instance:
+ *
+ * {@code
+ *
+ *  pipeline
+ *.apply(...) // provide PCollection
+ *
.apply(RabbitMqIO.write().withUri("amqp://user:password@localhost:5672").withQueue("QUEUE"));
+ *
+ * }
+ */
+@Experimental(Experimental.Kind.SOURCE_SINK)
+public class RabbitMqIO {
+
+  public static Read read() {
+return new AutoValue_RabbitMqIO_Read.Builder()
+.setConnectionConfig(ConnectionConfig.create()).setQueueDeclare(false)
+
.setMaxReadTime(null).setMaxNumRecords(Long.MAX_VALUE).setUseCorrelationId(false).build();
+  }
+
+  public static Write write() {
+return new AutoValue_RabbitMqIO_Write.Builder()
+.setConnectionConfig(ConnectionConfig.create())
+.setExchangeDeclare(false).build();
+  }
+
+  private RabbitMqIO() {
+  }
+
+  /**
+   * Describe a connection configuration to a RabbitMQ server.
+   */
+  @AutoValue
+  public abstract static class ConnectionConfig implements Serializable {
+
+@Nullable abstract String uri();
+
+abstract int networkRecoveryInterval();
+abstract boolean automaticRecovery();
+abstract boolean topologyRecovery();
+
+abstract int connectionTimeout();
+abstract int requestedChannelMax();
+abstract int requestedFrameMax();
+abstract int requestedHeartbeat();
+
+abstract Builder builder();
+
+@Aut

[jira] [Work logged] (BEAM-1240) Create RabbitMqIO

2018-03-09 Thread ASF GitHub Bot (JIRA)


 [ 
https://issues.apache.org/jira/browse/BEAM-1240?focusedWorklogId=79100&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-79100
 ]

ASF GitHub Bot logged work on BEAM-1240:


Author: ASF GitHub Bot
Created on: 09/Mar/18 22:02
Start Date: 09/Mar/18 22:02
Worklog Time Spent: 10m 
  Work Description: jkff commented on a change in pull request #1729: 
[BEAM-1240] Create RabbitMqIO
URL: https://github.com/apache/beam/pull/1729#discussion_r173574801
 
 

 ##
 File path: 
sdks/java/io/rabbitmq/src/main/java/org/apache/beam/sdk/io/rabbitmq/RabbitMqIO.java
 ##
 @@ -0,0 +1,711 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.io.rabbitmq;
+
+import static com.google.common.base.Preconditions.checkArgument;
+
+import com.google.auto.value.AutoValue;
+import com.rabbitmq.client.AMQP;
+import com.rabbitmq.client.Channel;
+import com.rabbitmq.client.Connection;
+import com.rabbitmq.client.ConnectionFactory;
+import com.rabbitmq.client.QueueingConsumer;
+import java.io.IOException;
+import java.io.Serializable;
+import java.net.URISyntaxException;
+import java.security.KeyManagementException;
+import java.security.NoSuchAlgorithmException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.NoSuchElementException;
+import javax.annotation.Nullable;
+import org.apache.beam.sdk.annotations.Experimental;
+import org.apache.beam.sdk.coders.Coder;
+import org.apache.beam.sdk.coders.SerializableCoder;
+import org.apache.beam.sdk.io.UnboundedSource;
+import org.apache.beam.sdk.options.PipelineOptions;
+import org.apache.beam.sdk.transforms.DoFn;
+import org.apache.beam.sdk.transforms.PTransform;
+import org.apache.beam.sdk.transforms.ParDo;
+import org.apache.beam.sdk.values.PBegin;
+import org.apache.beam.sdk.values.PCollection;
+import org.apache.beam.sdk.values.PDone;
+import org.joda.time.Duration;
+import org.joda.time.Instant;
+
+/**
+ * A IO to publish or consume messages with a RabbitMQ broker.
+ *
+ * Consuming messages from RabbitMQ server
+ *
+ * {@link RabbitMqIO} {@link Read} returns an unbounded {@link PCollection} 
containing RabbitMQ
+ * messages body (as {@code byte[]}).
+ *
+ * To configure a RabbitMQ source, you have to provide a RabbitMQ {@code 
URI} to connect
+ * to a RabbitMQ broker. The following example illustrates various options for 
configuring the
+ * source:
+ *
+ * {@code
+ *
+ *  pipeline.apply(
+ *
RabbitMqIO.read().withUri("amqp://user:password@localhost:5672").withQueue("QUEUE")
+ *
+ * }
+ *
+ * Publishing messages to RabbitMQ server
+ *
+ * {@link RabbitMqIO} {@link Write} can send {@code byte[]} to a RabbitMQ 
server queue.
+ *
+ * As for the {@link Read}, the {@link Write} is configured with a RabbitMQ
+ * {@link ConnectionConfig}.
+ *
+ * For instance:
+ *
+ * {@code
+ *
+ *  pipeline
+ *.apply(...) // provide PCollection
+ *
.apply(RabbitMqIO.write().withUri("amqp://user:password@localhost:5672").withQueue("QUEUE"));
+ *
+ * }
+ */
+@Experimental(Experimental.Kind.SOURCE_SINK)
+public class RabbitMqIO {
+
+  public static Read read() {
+return new AutoValue_RabbitMqIO_Read.Builder()
+.setConnectionConfig(ConnectionConfig.create()).setQueueDeclare(false)
+
.setMaxReadTime(null).setMaxNumRecords(Long.MAX_VALUE).setUseCorrelationId(false).build();
+  }
+
+  public static Write write() {
+return new AutoValue_RabbitMqIO_Write.Builder()
+.setConnectionConfig(ConnectionConfig.create())
+.setExchangeDeclare(false).build();
+  }
+
+  private RabbitMqIO() {
+  }
+
+  /**
+   * Describe a connection configuration to a RabbitMQ server.
+   */
+  @AutoValue
+  public abstract static class ConnectionConfig implements Serializable {
+
+@Nullable abstract String uri();
+
+abstract int networkRecoveryInterval();
+abstract boolean automaticRecovery();
+abstract boolean topologyRecovery();
+
+abstract int connectionTimeout();
+abstract int requestedChannelMax();
+abstract int requestedFrameMax();
+abstract int requestedHeartbeat();
+
+abstract Builder builder();
+
+@Aut

[jira] [Work logged] (BEAM-1240) Create RabbitMqIO

2018-03-09 Thread ASF GitHub Bot (JIRA)


 [ 
https://issues.apache.org/jira/browse/BEAM-1240?focusedWorklogId=79110&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-79110
 ]

ASF GitHub Bot logged work on BEAM-1240:


Author: ASF GitHub Bot
Created on: 09/Mar/18 22:02
Start Date: 09/Mar/18 22:02
Worklog Time Spent: 10m 
  Work Description: jkff commented on a change in pull request #1729: 
[BEAM-1240] Create RabbitMqIO
URL: https://github.com/apache/beam/pull/1729#discussion_r173576223
 
 

 ##
 File path: 
sdks/java/io/rabbitmq/src/main/java/org/apache/beam/sdk/io/rabbitmq/RabbitMqIO.java
 ##
 @@ -0,0 +1,711 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.io.rabbitmq;
+
+import static com.google.common.base.Preconditions.checkArgument;
+
+import com.google.auto.value.AutoValue;
+import com.rabbitmq.client.AMQP;
+import com.rabbitmq.client.Channel;
+import com.rabbitmq.client.Connection;
+import com.rabbitmq.client.ConnectionFactory;
+import com.rabbitmq.client.QueueingConsumer;
+import java.io.IOException;
+import java.io.Serializable;
+import java.net.URISyntaxException;
+import java.security.KeyManagementException;
+import java.security.NoSuchAlgorithmException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.NoSuchElementException;
+import javax.annotation.Nullable;
+import org.apache.beam.sdk.annotations.Experimental;
+import org.apache.beam.sdk.coders.Coder;
+import org.apache.beam.sdk.coders.SerializableCoder;
+import org.apache.beam.sdk.io.UnboundedSource;
+import org.apache.beam.sdk.options.PipelineOptions;
+import org.apache.beam.sdk.transforms.DoFn;
+import org.apache.beam.sdk.transforms.PTransform;
+import org.apache.beam.sdk.transforms.ParDo;
+import org.apache.beam.sdk.values.PBegin;
+import org.apache.beam.sdk.values.PCollection;
+import org.apache.beam.sdk.values.PDone;
+import org.joda.time.Duration;
+import org.joda.time.Instant;
+
+/**
+ * A IO to publish or consume messages with a RabbitMQ broker.
+ *
+ * Consuming messages from RabbitMQ server
+ *
+ * {@link RabbitMqIO} {@link Read} returns an unbounded {@link PCollection} 
containing RabbitMQ
+ * messages body (as {@code byte[]}).
+ *
+ * To configure a RabbitMQ source, you have to provide a RabbitMQ {@code 
URI} to connect
+ * to a RabbitMQ broker. The following example illustrates various options for 
configuring the
+ * source:
+ *
+ * {@code
+ *
+ *  pipeline.apply(
+ *
RabbitMqIO.read().withUri("amqp://user:password@localhost:5672").withQueue("QUEUE")
+ *
+ * }
+ *
+ * Publishing messages to RabbitMQ server
+ *
+ * {@link RabbitMqIO} {@link Write} can send {@code byte[]} to a RabbitMQ 
server queue.
+ *
+ * As for the {@link Read}, the {@link Write} is configured with a RabbitMQ
+ * {@link ConnectionConfig}.
+ *
+ * For instance:
+ *
+ * {@code
+ *
+ *  pipeline
+ *.apply(...) // provide PCollection
+ *
.apply(RabbitMqIO.write().withUri("amqp://user:password@localhost:5672").withQueue("QUEUE"));
+ *
+ * }
+ */
+@Experimental(Experimental.Kind.SOURCE_SINK)
+public class RabbitMqIO {
+
+  public static Read read() {
+return new AutoValue_RabbitMqIO_Read.Builder()
+.setConnectionConfig(ConnectionConfig.create()).setQueueDeclare(false)
+
.setMaxReadTime(null).setMaxNumRecords(Long.MAX_VALUE).setUseCorrelationId(false).build();
+  }
+
+  public static Write write() {
+return new AutoValue_RabbitMqIO_Write.Builder()
+.setConnectionConfig(ConnectionConfig.create())
+.setExchangeDeclare(false).build();
+  }
+
+  private RabbitMqIO() {
+  }
+
+  /**
+   * Describe a connection configuration to a RabbitMQ server.
+   */
+  @AutoValue
+  public abstract static class ConnectionConfig implements Serializable {
+
+@Nullable abstract String uri();
+
+abstract int networkRecoveryInterval();
+abstract boolean automaticRecovery();
+abstract boolean topologyRecovery();
+
+abstract int connectionTimeout();
+abstract int requestedChannelMax();
+abstract int requestedFrameMax();
+abstract int requestedHeartbeat();
+
+abstract Builder builder();
+
+@Aut

[jira] [Work logged] (BEAM-1240) Create RabbitMqIO

2018-03-09 Thread ASF GitHub Bot (JIRA)


 [ 
https://issues.apache.org/jira/browse/BEAM-1240?focusedWorklogId=79102&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-79102
 ]

ASF GitHub Bot logged work on BEAM-1240:


Author: ASF GitHub Bot
Created on: 09/Mar/18 22:02
Start Date: 09/Mar/18 22:02
Worklog Time Spent: 10m 
  Work Description: jkff commented on a change in pull request #1729: 
[BEAM-1240] Create RabbitMqIO
URL: https://github.com/apache/beam/pull/1729#discussion_r173574856
 
 

 ##
 File path: 
sdks/java/io/rabbitmq/src/main/java/org/apache/beam/sdk/io/rabbitmq/RabbitMqIO.java
 ##
 @@ -0,0 +1,748 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.io.rabbitmq;
+
+import static com.google.common.base.Preconditions.checkArgument;
+
+import com.google.auto.value.AutoValue;
+import com.rabbitmq.client.Channel;
+import com.rabbitmq.client.Connection;
+import com.rabbitmq.client.ConnectionFactory;
+import com.rabbitmq.client.QueueingConsumer;
+
+import java.io.IOException;
+import java.io.Serializable;
+import java.net.URISyntaxException;
+import java.security.KeyManagementException;
+import java.security.NoSuchAlgorithmException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.NoSuchElementException;
+
+import javax.annotation.Nullable;
+
+import org.apache.beam.sdk.annotations.Experimental;
+import org.apache.beam.sdk.coders.ByteArrayCoder;
+import org.apache.beam.sdk.coders.Coder;
+import org.apache.beam.sdk.coders.SerializableCoder;
+import org.apache.beam.sdk.io.UnboundedSource;
+import org.apache.beam.sdk.options.PipelineOptions;
+import org.apache.beam.sdk.transforms.DoFn;
+import org.apache.beam.sdk.transforms.PTransform;
+import org.apache.beam.sdk.transforms.ParDo;
+import org.apache.beam.sdk.values.PBegin;
+import org.apache.beam.sdk.values.PCollection;
+import org.apache.beam.sdk.values.PDone;
+import org.joda.time.Duration;
+import org.joda.time.Instant;
+
+/**
+ * A IO to publish or consume messages with a RabbitMQ broker.
+ *
+ * Consuming messages from RabbitMQ server
+ *
+ * {@link RabbitMqIO} {@link Read} returns an unbounded {@link PCollection} 
containing RabbitMQ
+ * messages body (as {@code byte[]}).
+ *
+ * To configure a RabbitMQ source, you have to provide a RabbitMQ {@code 
URI} to connect
+ * to a RabbitMQ broker. The following example illustrates various options for 
configuring the
+ * source:
+ *
+ * {@code
+ *
+ *  pipeline.apply(
+ *
RabbitMqIO.read().withUri("amqp://user:password@localhost:5672").withQueue("QUEUE")
+ *
+ * }
+ *
+ * Publishing messages to RabbitMQ server
+ *
+ * {@link RabbitMqIO} {@link Write} can send {@code byte[]} to a RabbitMQ 
server queue.
+ *
+ * As for the {@link Read}, the {@link Write} is configured with a RabbitMQ
+ * {@link ConnectionConfig}.
+ *
+ * For instance:
+ *
+ * {@code
+ *
+ *  pipeline
+ *.apply(...) // provide PCollection
+ *
.apply(RabbitMqIO.write().withUri("amqp://user:password@localhost:5672").withQueue("QUEUE"));
+ *
+ * }
+ */
+@Experimental(Experimental.Kind.SOURCE_SINK)
+public class RabbitMqIO {
+
+  public static Read read() {
+return new AutoValue_RabbitMqIO_Read.Builder()
+.setConnectionConfig(ConnectionConfig.create())
+
.setMaxReadTime(null).setMaxNumRecords(Long.MAX_VALUE).setUseCorrelationId(false).build();
+  }
+
+  public static Write write() {
+return new AutoValue_RabbitMqIO_Write.Builder()
+.setConnectionConfig(ConnectionConfig.create()).build();
+  }
+
+  private RabbitMqIO() {
+  }
+
+  /**
+   * Describe a connection configuration to a RabbitMQ server.
+   */
+  @AutoValue
+  public abstract static class ConnectionConfig implements Serializable {
+
+@Nullable abstract String uri();
+@Nullable abstract String queue();
+@Nullable abstract String exchange();
+@Nullable abstract String exchangeType();
+
+abstract int networkRecoveryInterval();
+abstract boolean automaticRecovery();
+abstract boolean topologyRecovery();
+
+abstract int connectionTimeout();
+abstract int requestedChannelMax();
+abstract int requestedFram

[jira] [Work logged] (BEAM-1240) Create RabbitMqIO

2018-03-09 Thread ASF GitHub Bot (JIRA)


 [ 
https://issues.apache.org/jira/browse/BEAM-1240?focusedWorklogId=79107&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-79107
 ]

ASF GitHub Bot logged work on BEAM-1240:


Author: ASF GitHub Bot
Created on: 09/Mar/18 22:02
Start Date: 09/Mar/18 22:02
Worklog Time Spent: 10m 
  Work Description: jkff commented on a change in pull request #1729: 
[BEAM-1240] Create RabbitMqIO
URL: https://github.com/apache/beam/pull/1729#discussion_r173574874
 
 

 ##
 File path: 
sdks/java/io/rabbitmq/src/main/java/org/apache/beam/sdk/io/rabbitmq/RabbitMqIO.java
 ##
 @@ -0,0 +1,748 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.io.rabbitmq;
+
+import static com.google.common.base.Preconditions.checkArgument;
+
+import com.google.auto.value.AutoValue;
+import com.rabbitmq.client.Channel;
+import com.rabbitmq.client.Connection;
+import com.rabbitmq.client.ConnectionFactory;
+import com.rabbitmq.client.QueueingConsumer;
+
+import java.io.IOException;
+import java.io.Serializable;
+import java.net.URISyntaxException;
+import java.security.KeyManagementException;
+import java.security.NoSuchAlgorithmException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.NoSuchElementException;
+
+import javax.annotation.Nullable;
+
+import org.apache.beam.sdk.annotations.Experimental;
+import org.apache.beam.sdk.coders.ByteArrayCoder;
+import org.apache.beam.sdk.coders.Coder;
+import org.apache.beam.sdk.coders.SerializableCoder;
+import org.apache.beam.sdk.io.UnboundedSource;
+import org.apache.beam.sdk.options.PipelineOptions;
+import org.apache.beam.sdk.transforms.DoFn;
+import org.apache.beam.sdk.transforms.PTransform;
+import org.apache.beam.sdk.transforms.ParDo;
+import org.apache.beam.sdk.values.PBegin;
+import org.apache.beam.sdk.values.PCollection;
+import org.apache.beam.sdk.values.PDone;
+import org.joda.time.Duration;
+import org.joda.time.Instant;
+
+/**
+ * A IO to publish or consume messages with a RabbitMQ broker.
+ *
+ * Consuming messages from RabbitMQ server
+ *
+ * {@link RabbitMqIO} {@link Read} returns an unbounded {@link PCollection} 
containing RabbitMQ
+ * messages body (as {@code byte[]}).
+ *
+ * To configure a RabbitMQ source, you have to provide a RabbitMQ {@code 
URI} to connect
+ * to a RabbitMQ broker. The following example illustrates various options for 
configuring the
+ * source:
+ *
+ * {@code
+ *
+ *  pipeline.apply(
+ *
RabbitMqIO.read().withUri("amqp://user:password@localhost:5672").withQueue("QUEUE")
+ *
+ * }
+ *
+ * Publishing messages to RabbitMQ server
+ *
+ * {@link RabbitMqIO} {@link Write} can send {@code byte[]} to a RabbitMQ 
server queue.
+ *
+ * As for the {@link Read}, the {@link Write} is configured with a RabbitMQ
+ * {@link ConnectionConfig}.
+ *
+ * For instance:
+ *
+ * {@code
+ *
+ *  pipeline
+ *.apply(...) // provide PCollection
+ *
.apply(RabbitMqIO.write().withUri("amqp://user:password@localhost:5672").withQueue("QUEUE"));
+ *
+ * }
+ */
+@Experimental(Experimental.Kind.SOURCE_SINK)
+public class RabbitMqIO {
+
+  public static Read read() {
+return new AutoValue_RabbitMqIO_Read.Builder()
+.setConnectionConfig(ConnectionConfig.create())
+
.setMaxReadTime(null).setMaxNumRecords(Long.MAX_VALUE).setUseCorrelationId(false).build();
+  }
+
+  public static Write write() {
+return new AutoValue_RabbitMqIO_Write.Builder()
+.setConnectionConfig(ConnectionConfig.create()).build();
+  }
+
+  private RabbitMqIO() {
+  }
+
+  /**
+   * Describe a connection configuration to a RabbitMQ server.
+   */
+  @AutoValue
+  public abstract static class ConnectionConfig implements Serializable {
+
+@Nullable abstract String uri();
+@Nullable abstract String queue();
+@Nullable abstract String exchange();
+@Nullable abstract String exchangeType();
+
+abstract int networkRecoveryInterval();
+abstract boolean automaticRecovery();
+abstract boolean topologyRecovery();
+
+abstract int connectionTimeout();
+abstract int requestedChannelMax();
+abstract int requestedFram

[jira] [Work logged] (BEAM-1240) Create RabbitMqIO

2018-03-09 Thread ASF GitHub Bot (JIRA)


 [ 
https://issues.apache.org/jira/browse/BEAM-1240?focusedWorklogId=79109&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-79109
 ]

ASF GitHub Bot logged work on BEAM-1240:


Author: ASF GitHub Bot
Created on: 09/Mar/18 22:02
Start Date: 09/Mar/18 22:02
Worklog Time Spent: 10m 
  Work Description: jkff commented on a change in pull request #1729: 
[BEAM-1240] Create RabbitMqIO
URL: https://github.com/apache/beam/pull/1729#discussion_r173577348
 
 

 ##
 File path: 
sdks/java/io/rabbitmq/src/main/java/org/apache/beam/sdk/io/rabbitmq/RabbitMqIO.java
 ##
 @@ -0,0 +1,711 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.io.rabbitmq;
+
+import static com.google.common.base.Preconditions.checkArgument;
+
+import com.google.auto.value.AutoValue;
+import com.rabbitmq.client.AMQP;
+import com.rabbitmq.client.Channel;
+import com.rabbitmq.client.Connection;
+import com.rabbitmq.client.ConnectionFactory;
+import com.rabbitmq.client.QueueingConsumer;
+import java.io.IOException;
+import java.io.Serializable;
+import java.net.URISyntaxException;
+import java.security.KeyManagementException;
+import java.security.NoSuchAlgorithmException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.NoSuchElementException;
+import javax.annotation.Nullable;
+import org.apache.beam.sdk.annotations.Experimental;
+import org.apache.beam.sdk.coders.Coder;
+import org.apache.beam.sdk.coders.SerializableCoder;
+import org.apache.beam.sdk.io.UnboundedSource;
+import org.apache.beam.sdk.options.PipelineOptions;
+import org.apache.beam.sdk.transforms.DoFn;
+import org.apache.beam.sdk.transforms.PTransform;
+import org.apache.beam.sdk.transforms.ParDo;
+import org.apache.beam.sdk.values.PBegin;
+import org.apache.beam.sdk.values.PCollection;
+import org.apache.beam.sdk.values.PDone;
+import org.joda.time.Duration;
+import org.joda.time.Instant;
+
+/**
+ * A IO to publish or consume messages with a RabbitMQ broker.
+ *
+ * Consuming messages from RabbitMQ server
+ *
+ * {@link RabbitMqIO} {@link Read} returns an unbounded {@link PCollection} 
containing RabbitMQ
+ * messages body (as {@code byte[]}).
+ *
+ * To configure a RabbitMQ source, you have to provide a RabbitMQ {@code 
URI} to connect
+ * to a RabbitMQ broker. The following example illustrates various options for 
configuring the
+ * source:
+ *
+ * {@code
+ *
+ *  pipeline.apply(
+ *
RabbitMqIO.read().withUri("amqp://user:password@localhost:5672").withQueue("QUEUE")
+ *
+ * }
+ *
+ * Publishing messages to RabbitMQ server
+ *
+ * {@link RabbitMqIO} {@link Write} can send {@code byte[]} to a RabbitMQ 
server queue.
+ *
+ * As for the {@link Read}, the {@link Write} is configured with a RabbitMQ
+ * {@link ConnectionConfig}.
+ *
+ * For instance:
+ *
+ * {@code
+ *
+ *  pipeline
+ *.apply(...) // provide PCollection
+ *
.apply(RabbitMqIO.write().withUri("amqp://user:password@localhost:5672").withQueue("QUEUE"));
+ *
+ * }
+ */
+@Experimental(Experimental.Kind.SOURCE_SINK)
+public class RabbitMqIO {
+
+  public static Read read() {
+return new AutoValue_RabbitMqIO_Read.Builder()
+.setConnectionConfig(ConnectionConfig.create()).setQueueDeclare(false)
+
.setMaxReadTime(null).setMaxNumRecords(Long.MAX_VALUE).setUseCorrelationId(false).build();
+  }
+
+  public static Write write() {
+return new AutoValue_RabbitMqIO_Write.Builder()
+.setConnectionConfig(ConnectionConfig.create())
+.setExchangeDeclare(false).build();
+  }
+
+  private RabbitMqIO() {
+  }
+
+  /**
+   * Describe a connection configuration to a RabbitMQ server.
+   */
+  @AutoValue
+  public abstract static class ConnectionConfig implements Serializable {
+
+@Nullable abstract String uri();
+
+abstract int networkRecoveryInterval();
+abstract boolean automaticRecovery();
+abstract boolean topologyRecovery();
+
+abstract int connectionTimeout();
+abstract int requestedChannelMax();
+abstract int requestedFrameMax();
+abstract int requestedHeartbeat();
+
+abstract Builder builder();
+
+@Aut

[jira] [Work logged] (BEAM-1240) Create RabbitMqIO

2018-03-09 Thread ASF GitHub Bot (JIRA)


 [ 
https://issues.apache.org/jira/browse/BEAM-1240?focusedWorklogId=79097&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-79097
 ]

ASF GitHub Bot logged work on BEAM-1240:


Author: ASF GitHub Bot
Created on: 09/Mar/18 22:02
Start Date: 09/Mar/18 22:02
Worklog Time Spent: 10m 
  Work Description: jkff commented on a change in pull request #1729: 
[BEAM-1240] Create RabbitMqIO
URL: https://github.com/apache/beam/pull/1729#discussion_r173574461
 
 

 ##
 File path: 
sdks/java/io/rabbitmq/src/main/java/org/apache/beam/sdk/io/rabbitmq/RabbitMqIO.java
 ##
 @@ -0,0 +1,711 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.sdk.io.rabbitmq;
+
+import static com.google.common.base.Preconditions.checkArgument;
+
+import com.google.auto.value.AutoValue;
+import com.rabbitmq.client.AMQP;
+import com.rabbitmq.client.Channel;
+import com.rabbitmq.client.Connection;
+import com.rabbitmq.client.ConnectionFactory;
+import com.rabbitmq.client.QueueingConsumer;
+import java.io.IOException;
+import java.io.Serializable;
+import java.net.URISyntaxException;
+import java.security.KeyManagementException;
+import java.security.NoSuchAlgorithmException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.NoSuchElementException;
+import javax.annotation.Nullable;
+import org.apache.beam.sdk.annotations.Experimental;
+import org.apache.beam.sdk.coders.Coder;
+import org.apache.beam.sdk.coders.SerializableCoder;
+import org.apache.beam.sdk.io.UnboundedSource;
+import org.apache.beam.sdk.options.PipelineOptions;
+import org.apache.beam.sdk.transforms.DoFn;
+import org.apache.beam.sdk.transforms.PTransform;
+import org.apache.beam.sdk.transforms.ParDo;
+import org.apache.beam.sdk.values.PBegin;
+import org.apache.beam.sdk.values.PCollection;
+import org.apache.beam.sdk.values.PDone;
+import org.joda.time.Duration;
+import org.joda.time.Instant;
+
+/**
+ * A IO to publish or consume messages with a RabbitMQ broker.
+ *
+ * Consuming messages from RabbitMQ server
+ *
+ * {@link RabbitMqIO} {@link Read} returns an unbounded {@link PCollection} 
containing RabbitMQ
+ * messages body (as {@code byte[]}).
+ *
+ * To configure a RabbitMQ source, you have to provide a RabbitMQ {@code 
URI} to connect
+ * to a RabbitMQ broker. The following example illustrates various options for 
configuring the
+ * source:
+ *
+ * {@code
+ *
+ *  pipeline.apply(
+ *
RabbitMqIO.read().withUri("amqp://user:password@localhost:5672").withQueue("QUEUE")
+ *
+ * }
+ *
+ * Publishing messages to RabbitMQ server
+ *
+ * {@link RabbitMqIO} {@link Write} can send {@code byte[]} to a RabbitMQ 
server queue.
+ *
+ * As for the {@link Read}, the {@link Write} is configured with a RabbitMQ
+ * {@link ConnectionConfig}.
+ *
+ * For instance:
+ *
+ * {@code
+ *
+ *  pipeline
+ *.apply(...) // provide PCollection
+ *
.apply(RabbitMqIO.write().withUri("amqp://user:password@localhost:5672").withQueue("QUEUE"));
+ *
+ * }
+ */
+@Experimental(Experimental.Kind.SOURCE_SINK)
+public class RabbitMqIO {
+
+  public static Read read() {
+return new AutoValue_RabbitMqIO_Read.Builder()
+.setConnectionConfig(ConnectionConfig.create()).setQueueDeclare(false)
+
.setMaxReadTime(null).setMaxNumRecords(Long.MAX_VALUE).setUseCorrelationId(false).build();
+  }
+
+  public static Write write() {
+return new AutoValue_RabbitMqIO_Write.Builder()
+.setConnectionConfig(ConnectionConfig.create())
+.setExchangeDeclare(false).build();
+  }
+
+  private RabbitMqIO() {
+  }
+
+  /**
+   * Describe a connection configuration to a RabbitMQ server.
+   */
+  @AutoValue
+  public abstract static class ConnectionConfig implements Serializable {
+
+@Nullable abstract String uri();
+
+abstract int networkRecoveryInterval();
+abstract boolean automaticRecovery();
+abstract boolean topologyRecovery();
+
+abstract int connectionTimeout();
+abstract int requestedChannelMax();
+abstract int requestedFrameMax();
+abstract int requestedHeartbeat();
+
+abstract Builder builder();
+
+@Aut

89 matches

Mail list logo