This is an automated email from the ASF dual-hosted git repository. aromanenko pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push: new f1e95970ca4 Make `withDesiredBundleSizeBytes` methods public (#25448) f1e95970ca4 is described below commit f1e95970ca4cb9f4ed9099761ba6a757099d22d8 Author: Israel Herraiz <i...@google.com> AuthorDate: Thu Apr 20 13:26:08 2023 +0200 Make `withDesiredBundleSizeBytes` methods public (#25448) * Make `withDesiredBundleSizeBytes` methods public With these methods being private, the bundle size is 64 MB, which is too large for streaming applications using these classes. This changes the method visibility to public so a smaller bundle size can be used for streaming. Fixes #25447 * Add Javadoc to new public methods * Make withDesiredBundleSizeBytes public, useful for streaming applications --- .../main/java/org/apache/beam/sdk/io/AvroIO.java | 28 +++++++++++++++------- .../apache/beam/sdk/extensions/avro/io/AvroIO.java | 28 +++++++++++++++------- 2 files changed, 40 insertions(+), 16 deletions(-) diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/AvroIO.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/AvroIO.java index c8ccf753229..42fab212576 100644 --- a/sdks/java/core/src/main/java/org/apache/beam/sdk/io/AvroIO.java +++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/io/AvroIO.java @@ -813,8 +813,11 @@ public class AvroIO { abstract ReadFiles<T> build(); } - @VisibleForTesting - ReadFiles<T> withDesiredBundleSizeBytes(long desiredBundleSizeBytes) { + /** + * Set a value for the bundle size for parallel reads. Default is 64 MB. You may want to use a + * lower value (e.g. 1 MB) for streaming applications. + */ + public ReadFiles<T> withDesiredBundleSizeBytes(long desiredBundleSizeBytes) { return toBuilder().setDesiredBundleSizeBytes(desiredBundleSizeBytes).build(); } @@ -927,8 +930,11 @@ public class AvroIO { getMatchConfiguration().continuously(pollInterval, terminationCondition)); } - @VisibleForTesting - ReadAll<T> withDesiredBundleSizeBytes(long desiredBundleSizeBytes) { + /** + * Set a value for the bundle size for parallel reads. Default is 64 MB. You may want to use a + * lower value (e.g. 1 MB) for streaming applications. + */ + public ReadAll<T> withDesiredBundleSizeBytes(long desiredBundleSizeBytes) { return toBuilder().setDesiredBundleSizeBytes(desiredBundleSizeBytes).build(); } @@ -1169,8 +1175,11 @@ public class AvroIO { return toBuilder().setFileExceptionHandler(exceptionHandler).build(); } - @VisibleForTesting - ParseFiles<T> withDesiredBundleSizeBytes(long desiredBundleSizeBytes) { + /** + * Set a value for the bundle size for parallel reads. Default is 64 MB. You may want to use a + * lower value (e.g. 1 MB) for streaming applications. + */ + public ParseFiles<T> withDesiredBundleSizeBytes(long desiredBundleSizeBytes) { return toBuilder().setDesiredBundleSizeBytes(desiredBundleSizeBytes).build(); } @@ -1278,8 +1287,11 @@ public class AvroIO { return toBuilder().setCoder(coder).build(); } - @VisibleForTesting - ParseAll<T> withDesiredBundleSizeBytes(long desiredBundleSizeBytes) { + /** + * Set a value for the bundle size for parallel reads. Default is 64 MB. You may want to use a + * lower value (e.g. 1 MB) for streaming applications. + */ + public ParseAll<T> withDesiredBundleSizeBytes(long desiredBundleSizeBytes) { return toBuilder().setDesiredBundleSizeBytes(desiredBundleSizeBytes).build(); } diff --git a/sdks/java/extensions/avro/src/main/java/org/apache/beam/sdk/extensions/avro/io/AvroIO.java b/sdks/java/extensions/avro/src/main/java/org/apache/beam/sdk/extensions/avro/io/AvroIO.java index b159d38c603..f35fc66e654 100644 --- a/sdks/java/extensions/avro/src/main/java/org/apache/beam/sdk/extensions/avro/io/AvroIO.java +++ b/sdks/java/extensions/avro/src/main/java/org/apache/beam/sdk/extensions/avro/io/AvroIO.java @@ -816,8 +816,11 @@ public class AvroIO { abstract ReadFiles<T> build(); } - @VisibleForTesting - ReadFiles<T> withDesiredBundleSizeBytes(long desiredBundleSizeBytes) { + /** + * Set a value for the bundle size for parallel reads. Default is 64 MB. You may want to use a + * lower value (e.g. 1 MB) for streaming applications. + */ + public ReadFiles<T> withDesiredBundleSizeBytes(long desiredBundleSizeBytes) { return toBuilder().setDesiredBundleSizeBytes(desiredBundleSizeBytes).build(); } @@ -930,8 +933,11 @@ public class AvroIO { getMatchConfiguration().continuously(pollInterval, terminationCondition)); } - @VisibleForTesting - ReadAll<T> withDesiredBundleSizeBytes(long desiredBundleSizeBytes) { + /** + * Set a value for the bundle size for parallel reads. Default is 64 MB. You may want to use a + * lower value (e.g. 1 MB) for streaming applications. + */ + public ReadAll<T> withDesiredBundleSizeBytes(long desiredBundleSizeBytes) { return toBuilder().setDesiredBundleSizeBytes(desiredBundleSizeBytes).build(); } @@ -1172,8 +1178,11 @@ public class AvroIO { return toBuilder().setFileExceptionHandler(exceptionHandler).build(); } - @VisibleForTesting - ParseFiles<T> withDesiredBundleSizeBytes(long desiredBundleSizeBytes) { + /** + * Set a value for the bundle size for parallel reads. Default is 64 MB. You may want to use a + * lower value (e.g. 1 MB) for streaming applications. + */ + public ParseFiles<T> withDesiredBundleSizeBytes(long desiredBundleSizeBytes) { return toBuilder().setDesiredBundleSizeBytes(desiredBundleSizeBytes).build(); } @@ -1281,8 +1290,11 @@ public class AvroIO { return toBuilder().setCoder(coder).build(); } - @VisibleForTesting - ParseAll<T> withDesiredBundleSizeBytes(long desiredBundleSizeBytes) { + /** + * Set a value for the bundle size for parallel reads. Default is 64 MB. You may want to use a + * lower value (e.g. 1 MB) for streaming applications. + */ + public ParseAll<T> withDesiredBundleSizeBytes(long desiredBundleSizeBytes) { return toBuilder().setDesiredBundleSizeBytes(desiredBundleSizeBytes).build(); }