aromanenko-dev commented on a change in pull request #14244:
URL: https://github.com/apache/beam/pull/14244#discussion_r629317202
##########
File path:
sdks/java/io/amazon-web-services/src/main/java/org/apache/beam/sdk/io/aws/sqs/SqsUnboundedReader.java
##########
@@ -17,51 +17,385 @@
*/
package org.apache.beam.sdk.io.aws.sqs;
+import static java.nio.charset.StandardCharsets.UTF_8;
+import static java.util.stream.Collectors.groupingBy;
+import static java.util.stream.Collectors.toMap;
+import static
org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkState;
+
+import com.amazonaws.services.sqs.AmazonSQS;
+import com.amazonaws.services.sqs.model.BatchResultErrorEntry;
+import
com.amazonaws.services.sqs.model.ChangeMessageVisibilityBatchRequestEntry;
+import com.amazonaws.services.sqs.model.ChangeMessageVisibilityBatchResult;
+import com.amazonaws.services.sqs.model.DeleteMessageBatchRequestEntry;
+import com.amazonaws.services.sqs.model.DeleteMessageBatchResult;
+import com.amazonaws.services.sqs.model.GetQueueAttributesRequest;
import com.amazonaws.services.sqs.model.Message;
+import com.amazonaws.services.sqs.model.MessageAttributeValue;
import com.amazonaws.services.sqs.model.MessageSystemAttributeName;
+import com.amazonaws.services.sqs.model.QueueAttributeName;
import com.amazonaws.services.sqs.model.ReceiveMessageRequest;
import com.amazonaws.services.sqs.model.ReceiveMessageResult;
-import java.io.Serializable;
-import java.nio.charset.StandardCharsets;
+import java.io.IOException;
import java.util.ArrayDeque;
import java.util.ArrayList;
import java.util.Arrays;
-import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.LinkedHashMap;
import java.util.List;
+import java.util.Map;
import java.util.NoSuchElementException;
+import java.util.Objects;
import java.util.Queue;
+import java.util.Set;
+import java.util.concurrent.ConcurrentLinkedQueue;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.stream.Collectors;
+import java.util.stream.IntStream;
import org.apache.beam.sdk.io.UnboundedSource;
import org.apache.beam.sdk.io.UnboundedSource.CheckpointMark;
+import org.apache.beam.sdk.transforms.Combine;
+import org.apache.beam.sdk.transforms.Sum;
import org.apache.beam.sdk.transforms.windowing.BoundedWindow;
+import org.apache.beam.sdk.util.BucketingFunction;
+import org.apache.beam.sdk.util.MovingFunction;
+import
org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.EvictingQueue;
+import
org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables;
+import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists;
+import org.joda.time.Duration;
import org.joda.time.Instant;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
@SuppressWarnings({
"nullness" // TODO(https://issues.apache.org/jira/browse/BEAM-10402)
})
-class SqsUnboundedReader extends UnboundedSource.UnboundedReader<Message>
implements Serializable {
+class SqsUnboundedReader extends UnboundedSource.UnboundedReader<Message> {
+ private static final Logger LOG =
LoggerFactory.getLogger(SqsUnboundedReader.class);
+ /** Maximum number of messages to pull from SQS per request. */
public static final int MAX_NUMBER_OF_MESSAGES = 10;
+
+ /** Maximum times to retry batch SQS operations upon partial success. */
+ private static final int BATCH_OPERATION_MAX_RETIRES = 5;
Review comment:
How the values of this and other constants below were estimated?
##########
File path:
sdks/java/io/amazon-web-services/src/test/java/org/apache/beam/sdk/io/aws/sqs/SqsIOTest.java
##########
@@ -46,25 +52,151 @@
/** Tests on {@link SqsIO}. */
@RunWith(JUnit4.class)
public class SqsIOTest {
+ private static final String DATA = "testData";
@Rule public TestPipeline pipeline = TestPipeline.create();
@Rule public EmbeddedSqsServer embeddedSqsRestServer = new
EmbeddedSqsServer();
- @Test
- public void testRead() {
+ private SqsUnboundedSource source;
+
+ private void setupOneMessage() {
final AmazonSQS client = embeddedSqsRestServer.getClient();
final String queueUrl = embeddedSqsRestServer.getQueueUrl();
+ client.sendMessage(queueUrl, DATA);
+ source =
+ new SqsUnboundedSource(
+ SqsIO.read().withQueueUrl(queueUrl).withMaxNumRecords(1),
+ new SqsConfiguration(pipeline.getOptions().as(AwsOptions.class)));
+ }
+
+ private void setupMessages(List<String> messages) {
+ final AmazonSQS client = embeddedSqsRestServer.getClient();
+ final String queueUrl = embeddedSqsRestServer.getQueueUrl();
+ for (String message : messages) {
+ client.sendMessage(queueUrl, message);
+ }
+ source =
+ new SqsUnboundedSource(
+ SqsIO.read().withQueueUrl(queueUrl).withMaxNumRecords(1),
+ new SqsConfiguration(pipeline.getOptions().as(AwsOptions.class)));
+ }
+
+ @Test
+ public void checkpointCoderIsSane() {
Review comment:
Please, add "test" prefix for all test methods
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
