This is an automated email from the ASF dual-hosted git repository. lcwik pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/beam.git
commit fa0428d1afa71a09582893a31a1ec70cd1f2b515 Merge: 834be30 e3e4e3c Author: 1028332163 <[email protected]> AuthorDate: Thu Feb 22 21:25:11 2018 +0800 Merge branch 'master' of https://github.com/apache/beam.git .gitignore | 1 + .test-infra/jenkins/common_job_properties.groovy | 7 + .test-infra/jenkins/job_beam_Inventory.groovy | 66 + .../jenkins/job_beam_PerformanceTests_JDBC.groovy | 76 +- .../job_beam_ReleaseCandidate_Python.groovy | 51 + assembly.xml | 1 + build.gradle | 5 +- build_rules.gradle | 60 +- examples/java/pom.xml | 2 +- .../beam/examples/complete/AutoComplete.java | 3 + .../beam/examples/cookbook/BigQueryTornadoes.java | 2 +- .../examples/cookbook/CombinePerKeyExamples.java | 2 +- .../beam/examples/cookbook/FilterExamples.java | 6 +- .../beam/examples/cookbook/TriggerExample.java | 7 +- .../apache/beam/examples/snippets/Snippets.java | 2 + examples/pom.xml | 2 +- model/fn-execution/pom.xml | 2 +- .../fn-execution/src/main/proto/beam_fn_api.proto | 7 + model/job-management/pom.xml | 2 +- model/pipeline/pom.xml | 2 +- model/pom.xml | 2 +- pom.xml | 13 +- .../run_release_candidate_python_validation.sh | 292 + runners/apex/build.gradle | 2 +- runners/apex/pom.xml | 2 +- .../operators/ApexProcessFnOperator.java | 2 +- runners/core-construction-java/build.gradle | 30 +- runners/core-construction-java/pom.xml | 2 +- .../runners/core/construction/Environments.java | 22 + .../core/construction/JavaReadViaImpulse.java | 95 + .../construction/WindowingStrategyTranslation.java | 1 + .../core/construction/graph/ExecutableStage.java | 84 +- .../core/construction/graph/FusedPipeline.java | 38 + .../graph/GreedilyFusedExecutableStage.java | 233 - .../graph/GreedyPCollectionFusers.java | 79 +- .../construction/graph/GreedyPipelineFuser.java | 6 +- .../core/construction/graph/GreedyStageFuser.java | 178 + .../graph/ImmutableExecutableStage.java | 52 + .../core/construction/graph/QueryablePipeline.java | 44 +- .../InMemoryArtifactStagerService.java | 19 +- .../core/construction/JavaReadViaImpulseTest.java | 131 + .../construction/graph/ExecutableStageTest.java | 156 + .../graph/GreedyPipelineFuserTest.java | 139 + ...bleStageTest.java => GreedyStageFuserTest.java} | 174 +- .../construction/graph/QueryablePipelineTest.java | 16 +- runners/core-java/build.gradle | 36 +- runners/core-java/pom.xml | 7 +- .../org/apache/beam/runners/core/DoFnRunners.java | 4 +- .../apache/beam/runners/core/ProcessFnRunner.java | 2 +- .../runners/core/metrics/MetricsTranslation.java | 138 + .../core/metrics/MetricsTranslationTest.java | 157 + runners/direct-java/build.gradle | 90 +- runners/direct-java/pom.xml | 3 +- .../beam/runners/direct/DirectRegistrar.java | 1 + .../runners/direct/ImpulseEvaluatorFactory.java | 104 + .../beam/runners/direct/RootProviderRegistry.java | 2 + .../runners/direct/TransformEvaluatorRegistry.java | 2 + .../direct/ImpulseEvaluatorFactoryTest.java | 120 + .../direct/UnboundedReadEvaluatorFactoryTest.java | 2 +- runners/flink/build.gradle | 26 +- runners/flink/pom.xml | 3 +- .../beam/runners/flink/FlinkPipelineOptions.java | 12 + .../flink/FlinkStreamingTransformTranslators.java | 35 +- .../apache/beam/runners/flink/TestFlinkRunner.java | 1 + .../wrappers/streaming/DoFnOperator.java | 127 +- .../wrappers/streaming/SplittableDoFnOperator.java | 16 + .../wrappers/streaming/WindowDoFnOperator.java | 18 + .../streaming/io/BoundedSourceWrapper.java | 28 + .../streaming/io/UnboundedSourceWrapper.java | 56 +- .../beam/runners/flink/ReadSourceITCase.java | 2 +- .../runners/flink/ReadSourceStreamingITCase.java | 2 +- .../flink/streaming/GroupByNullKeyTest.java | 2 +- .../streaming/TopWikipediaSessionsITCase.java | 2 +- runners/gcp/gcemd/pom.xml | 2 +- runners/gcp/gcsproxy/pom.xml | 2 +- runners/gcp/pom.xml | 2 +- runners/gearpump/build.gradle | 2 +- runners/gearpump/pom.xml | 2 +- runners/google-cloud-dataflow-java/build.gradle | 5 +- runners/google-cloud-dataflow-java/pom.xml | 4 +- .../beam/runners/dataflow/DataflowRunner.java | 118 +- .../beam/runners/dataflow/util/TimeUtil.java | 19 +- runners/java-fn-execution/pom.xml | 2 +- .../environment/EnvironmentManager.java | 28 +- .../fnexecution/environment/RemoteEnvironment.java | 26 +- .../fnexecution/environment/package-info.java | 19 +- runners/local-artifact-service-java/pom.xml | 2 +- runners/local-java/pom.xml | 2 +- runners/pom.xml | 2 +- runners/reference/java/pom.xml | 2 +- runners/reference/job-server/pom.xml | 2 +- runners/reference/pom.xml | 2 +- runners/spark/pom.xml | 2 +- .../beam/runners/spark/io/EmptyCheckpointMark.java | 2 +- .../spark/translation/MultiDoFnFunction.java | 2 +- .../spark/translation/SparkContextFactory.java | 2 +- .../runners/spark/ProvidedSparkContextTest.java | 2 +- sdks/go/README.md | 98 + sdks/go/cmd/specialize/main.go | 240 + sdks/go/cmd/symtab/main.go | 101 + sdks/go/container/Dockerfile | 30 + sdks/go/container/boot.go | 127 + sdks/{python => go}/container/pom.xml | 10 +- sdks/go/data/haiku/old_pond.txt | 3 + sdks/go/data/shakespeare/hamlet.txt | 6045 ++++++ sdks/go/data/shakespeare/kinglear-hashtag.txt | 5525 +++++ sdks/go/data/shakespeare/kinglear.txt | 5525 +++++ sdks/go/data/shakespeare/romeoandjuliet.txt | 4766 +++++ .../examples/complete/autocomplete/autocomplete.go | 71 + sdks/go/examples/contains/contains.go | 99 + sdks/go/examples/cookbook/combine/combine.go | 106 + sdks/go/examples/cookbook/filter/filter.go | 106 + sdks/go/examples/cookbook/join/join.go | 111 + sdks/go/examples/cookbook/max/max.go | 88 + sdks/go/examples/cookbook/tornadoes/tornadoes.go | 119 + .../debugging_wordcount/debugging_wordcount.go | 163 + sdks/go/examples/forest/forest.go | 81 + sdks/go/examples/grades/grades.go | 105 + .../minimal_wordcount/minimal_wordcount.go | 101 + sdks/go/examples/pingpong/pingpong.go | 131 + sdks/go/examples/wordcap/wordcap.go | 75 + sdks/go/examples/wordcount/wordcount.go | 168 + sdks/go/examples/yatzy/yatzy.go | 139 + sdks/go/pkg/beam/coder.go | 180 + .../pkg/beam/{util/grpcx/dial.go => coder_test.go} | 34 +- sdks/go/pkg/beam/combine.go | 78 + .../syscall_default.go => core/funcx/doc.go} | 20 +- sdks/go/pkg/beam/core/funcx/fn.go | 427 + sdks/go/pkg/beam/core/funcx/fn_test.go | 224 + sdks/go/pkg/beam/core/funcx/output.go | 76 + sdks/go/pkg/beam/core/funcx/output_test.go | 48 + sdks/go/pkg/beam/core/funcx/sideinput.go | 100 + sdks/go/pkg/beam/core/funcx/sideinput_test.go | 70 + sdks/go/pkg/beam/core/funcx/signature.go | 212 + sdks/go/pkg/beam/core/funcx/signature_test.go | 212 + sdks/go/pkg/beam/core/graph/bind.go | 325 + sdks/go/pkg/beam/core/graph/bind_test.go | 207 + sdks/go/pkg/beam/core/graph/coder/coder.go | 298 + sdks/go/pkg/beam/core/graph/coder/int.go | 71 + sdks/go/pkg/beam/core/graph/coder/int_test.go | 88 + .../grpcx/dial.go => core/graph/coder/time.go} | 31 +- sdks/go/pkg/beam/core/graph/coder/varint.go | 98 + sdks/go/pkg/beam/core/graph/coder/varint_test.go | 94 + .../syscall_default.go => core/graph/doc.go} | 20 +- sdks/go/pkg/beam/core/graph/edge.go | 397 + sdks/go/pkg/beam/core/graph/fn.go | 304 + sdks/go/pkg/beam/core/graph/graph.go | 126 + sdks/go/pkg/beam/core/graph/graph_test.go | 68 + sdks/go/pkg/beam/core/graph/node.go | 70 + .../syscall_default.go => core/graph/scope.go} | 27 +- sdks/go/pkg/beam/core/graph/window/window.go | 71 + sdks/go/pkg/beam/core/runtime/coderx/int.go | 98 + sdks/go/pkg/beam/core/runtime/coderx/varint.go | 139 + .../go/pkg/beam/core/runtime/coderx/varint_test.go | 84 + sdks/go/pkg/beam/core/runtime/exec/coder.go | 319 + sdks/go/pkg/beam/core/runtime/exec/cogbk.go | 171 + sdks/go/pkg/beam/core/runtime/exec/combine.go | 257 + sdks/go/pkg/beam/core/runtime/exec/combine_test.go | 67 + sdks/go/pkg/beam/core/runtime/exec/datasink.go | 90 + sdks/go/pkg/beam/core/runtime/exec/datasource.go | 203 + sdks/go/pkg/beam/core/runtime/exec/decode.go | 102 + .../runtime/exec/discard.go} | 41 +- sdks/go/pkg/beam/core/runtime/exec/emit.go | 126 + sdks/go/pkg/beam/core/runtime/exec/encode.go | 102 + sdks/go/pkg/beam/core/runtime/exec/flatten.go | 75 + sdks/go/pkg/beam/core/runtime/exec/flatten_test.go | 49 + sdks/go/pkg/beam/core/runtime/exec/fn.go | 201 + sdks/go/pkg/beam/core/runtime/exec/fn_test.go | 256 + sdks/go/pkg/beam/core/runtime/exec/fullvalue.go | 145 + .../pkg/beam/core/runtime/exec/fullvalue_test.go | 72 + sdks/go/pkg/beam/core/runtime/exec/input.go | 188 + sdks/go/pkg/beam/core/runtime/exec/multiplex.go | 62 + .../pkg/beam/core/runtime/exec/multiplex_test.go | 55 + .../beam/core/runtime/exec/optimized/callers.go | 632 + .../beam/core/runtime/exec/optimized/callers.tmpl | 69 + .../beam/core/runtime/exec/optimized/decoders.go | 2407 +++ .../beam/core/runtime/exec/optimized/decoders.tmpl | 146 + .../beam/core/runtime/exec/optimized/emitters.go | 14215 +++++++++++++ .../beam/core/runtime/exec/optimized/emitters.tmpl | 112 + .../beam/core/runtime/exec/optimized/encoders.go | 2299 ++ .../beam/core/runtime/exec/optimized/encoders.tmpl | 146 + .../runtime/exec/optimized/gen.go} | 19 +- .../pkg/beam/core/runtime/exec/optimized/inputs.go | 21272 +++++++++++++++++++ .../beam/core/runtime/exec/optimized/inputs.tmpl | 181 + sdks/go/pkg/beam/core/runtime/exec/pardo.go | 193 + sdks/go/pkg/beam/core/runtime/exec/pardo_test.go | 97 + sdks/go/pkg/beam/core/runtime/exec/plan.go | 153 + .../runtime/exec/status.go} | 21 +- sdks/go/pkg/beam/core/runtime/exec/translate.go | 433 + sdks/go/pkg/beam/core/runtime/exec/unit.go | 75 + sdks/go/pkg/beam/core/runtime/exec/unit_test.go | 111 + sdks/go/pkg/beam/core/runtime/exec/util.go | 72 + sdks/go/pkg/beam/core/runtime/graphx/coder.go | 415 + sdks/go/pkg/beam/core/runtime/graphx/coder_test.go | 107 + sdks/go/pkg/beam/core/runtime/graphx/cogbk.go | 108 + .../runtime/graphx/doc.go} | 27 +- sdks/go/pkg/beam/core/runtime/graphx/serialize.go | 962 + sdks/go/pkg/beam/core/runtime/graphx/translate.go | 402 + .../pkg/beam/core/runtime/graphx/translate_test.go | 90 + sdks/go/pkg/beam/core/runtime/graphx/tree.go | 121 + sdks/go/pkg/beam/core/runtime/graphx/user.go | 110 + .../runtime/graphx/v1/gen.go} | 14 +- sdks/go/pkg/beam/core/runtime/graphx/v1/v1.pb.go | 821 + sdks/go/pkg/beam/core/runtime/graphx/v1/v1.proto | 232 + sdks/go/pkg/beam/core/runtime/harness/datamgr.go | 308 + .../runtime/harness/gen.go} | 14 +- sdks/go/pkg/beam/core/runtime/harness/harness.go | 291 + sdks/go/pkg/beam/core/runtime/harness/init/init.go | 116 + sdks/go/pkg/beam/core/runtime/harness/logging.go | 151 + sdks/go/pkg/beam/core/runtime/harness/session.go | 215 + .../go/pkg/beam/core/runtime/harness/session.proto | 69 + .../core/runtime/harness/session/session.pb.go | 444 + sdks/go/pkg/beam/core/runtime/init.go | 44 + sdks/go/pkg/beam/core/runtime/options.go | 91 + .../grpcx/dial.go => core/runtime/options_test.go} | 44 +- sdks/go/pkg/beam/core/runtime/symbols.go | 93 + sdks/go/pkg/beam/core/runtime/types.go | 62 + sdks/go/pkg/beam/core/runtime/types_test.go | 74 + sdks/go/pkg/beam/core/typex/class.go | 174 + sdks/go/pkg/beam/core/typex/class_test.go | 104 + sdks/go/pkg/beam/core/typex/fulltype.go | 400 + sdks/go/pkg/beam/core/typex/fulltype_test.go | 125 + sdks/go/pkg/beam/core/typex/special.go | 74 + sdks/go/pkg/beam/core/util/dot/dot.go | 130 + .../syscall.go => core/util/ioutilx/read.go} | 29 +- sdks/go/pkg/beam/core/util/protox/any.go | 101 + sdks/go/pkg/beam/core/util/protox/any_test.go | 83 + .../grpcx/dial.go => core/util/protox/base64.go} | 37 +- .../util/protox/protox.go} | 20 +- .../grpcx/dial.go => core/util/protox/query.go} | 34 +- sdks/go/pkg/beam/core/util/reflectx/call.go | 120 + sdks/go/pkg/beam/core/util/reflectx/calls.go | 1341 ++ sdks/go/pkg/beam/core/util/reflectx/calls.tmpl | 72 + sdks/go/pkg/beam/core/util/reflectx/functions.go | 45 + .../syscall.go => core/util/reflectx/json.go} | 21 +- sdks/go/pkg/beam/core/util/reflectx/tags.go | 74 + sdks/go/pkg/beam/core/util/reflectx/types.go | 111 + sdks/go/pkg/beam/core/util/symtab/symtab.go | 127 + sdks/go/pkg/beam/create.go | 108 + .../beam/{util/grpcx/dial.go => create_test.go} | 38 +- sdks/go/pkg/beam/doc.go | 67 + sdks/go/pkg/beam/encoding.go | 148 + sdks/go/pkg/beam/external.go | 64 + sdks/go/pkg/beam/flatten.go | 72 + sdks/go/pkg/beam/forward.go | 94 + sdks/go/pkg/beam/gbk.go | 104 + sdks/go/pkg/beam/impulse.go | 46 + sdks/go/pkg/beam/io/bigqueryio/bigquery.go | 241 + .../dial.go => io/bigqueryio/bigquery_test.go} | 35 +- sdks/go/pkg/beam/io/textio/filesystem.go | 49 + sdks/go/pkg/beam/io/textio/gcs/gcs.go | 130 + .../grpcx/dial.go => io/textio/local/local.go} | 44 +- sdks/go/pkg/beam/io/textio/textio.go | 207 + sdks/go/pkg/beam/log/log.go | 183 + .../{util/syscallx/syscall.go => log/standard.go} | 26 +- .../beam/model/fnexecution_v1/beam_fn_api.pb.go | 424 +- sdks/go/pkg/beam/option.go | 72 + .../grpcx/dial.go => options/gcpopts/options.go} | 27 +- sdks/go/pkg/beam/pardo.go | 314 + sdks/go/pkg/beam/partition.go | 124 + sdks/go/pkg/beam/partition_test.go | 155 + sdks/go/pkg/beam/pcollection.go | 90 + sdks/go/pkg/beam/pipeline.go | 87 + sdks/go/pkg/beam/runner.go | 51 + sdks/go/pkg/beam/runners/dataflow/dataflow.go | 367 + sdks/go/pkg/beam/runners/dataflow/messages.go | 174 + sdks/go/pkg/beam/runners/dataflow/translate.go | 448 + sdks/go/pkg/beam/runners/direct/buffer.go | 165 + sdks/go/pkg/beam/runners/direct/direct.go | 274 + sdks/go/pkg/beam/runners/direct/gbk.go | 135 + sdks/go/pkg/beam/runners/direct/impulse.go | 61 + sdks/go/pkg/beam/runners/dot/dot.go | 55 + sdks/go/pkg/beam/runners/session/session.go | 337 + sdks/go/pkg/beam/testing/passert/passert.go | 214 + sdks/go/pkg/beam/testing/ptest/ptest.go | 59 + .../dial.go => transforms/filter/distinct.go} | 39 +- .../go/pkg/beam/transforms/filter/distinct_test.go | 70 + sdks/go/pkg/beam/transforms/filter/filter.go | 95 + sdks/go/pkg/beam/transforms/filter/filter_test.go | 90 + .../syscall.go => transforms/stats/count.go} | 23 +- sdks/go/pkg/beam/transforms/stats/count_test.go | 69 + sdks/go/pkg/beam/transforms/stats/max.go | 44 + sdks/go/pkg/beam/transforms/stats/max_switch.go | 138 + .../stats/max_switch.tmpl} | 30 +- sdks/go/pkg/beam/transforms/stats/max_test.go | 134 + sdks/go/pkg/beam/transforms/stats/mean.go | 91 + sdks/go/pkg/beam/transforms/stats/mean_test.go | 121 + sdks/go/pkg/beam/transforms/stats/min.go | 44 + sdks/go/pkg/beam/transforms/stats/min_switch.go | 138 + .../stats/min_switch.tmpl} | 30 +- sdks/go/pkg/beam/transforms/stats/min_test.go | 121 + sdks/go/pkg/beam/transforms/stats/sum.go | 44 + sdks/go/pkg/beam/transforms/stats/sum_switch.go | 102 + .../stats/sum_switch.tmpl} | 27 +- sdks/go/pkg/beam/transforms/stats/sum_test.go | 117 + sdks/go/pkg/beam/transforms/stats/util.go | 48 + sdks/go/pkg/beam/transforms/top/top.go | 180 + sdks/go/pkg/beam/transforms/top/top_test.go | 126 + sdks/go/pkg/beam/util.go | 117 + sdks/go/pkg/beam/util/gcsx/gcs.go | 40 + sdks/go/pkg/beam/util/grpcx/dial.go | 12 +- sdks/go/pkg/beam/util/syscallx/syscall.go | 6 +- sdks/go/pkg/beam/util/syscallx/syscall_default.go | 4 +- sdks/go/pkg/beam/validate.go | 79 + sdks/go/pkg/beam/x/beamx/run.go | 41 + sdks/go/pkg/beam/x/debug/head.go | 68 + sdks/go/pkg/beam/x/debug/print.go | 97 + sdks/go/pom.xml | 10 +- sdks/java/build-tools/pom.xml | 2 +- .../src/main/resources/beam/checkstyle.xml | 7 +- sdks/java/container/pom.xml | 2 +- sdks/java/core/pom.xml | 2 +- .../java/org/apache/beam/sdk/coders/Coder.java | 2 +- .../org/apache/beam/sdk/coders/CoderProviders.java | 2 +- .../org/apache/beam/sdk/coders/IterableCoder.java | 10 + .../java/org/apache/beam/sdk/coders/RowCoder.java | 4 +- .../apache/beam/sdk/coders/SerializableCoder.java | 41 + .../java/org/apache/beam/sdk/io/FileBasedSink.java | 26 +- .../main/java/org/apache/beam/sdk/io/FileIO.java | 112 +- .../apache/beam/sdk/options/PipelineOptions.java | 5 +- .../beam/sdk/options/PipelineOptionsFactory.java | 345 +- .../beam/sdk/options/ProxyInvocationHandler.java | 18 +- .../apache/beam/sdk/testing/CombineFnTester.java | 1 + .../java/org/apache/beam/sdk/testing/PAssert.java | 19 + .../apache/beam/sdk/testing/SuccessOrFailure.java | 19 + .../apache/beam/sdk/testing/WindowFnTestUtils.java | 6 +- .../beam/sdk/transforms/ApproximateUnique.java | 104 +- .../org/apache/beam/sdk/transforms/Combine.java | 2 + .../org/apache/beam/sdk/transforms/CombineFns.java | 18 + .../java/org/apache/beam/sdk/transforms/DoFn.java | 45 +- .../beam/sdk/transforms/GroupIntoBatches.java | 2 +- .../org/apache/beam/sdk/transforms/Impulse.java | 3 +- .../java/org/apache/beam/sdk/transforms/Wait.java | 2 +- .../beam/sdk/transforms/reflect/DoFnSignature.java | 4 + .../transforms/reflect/OnTimerMethodSpecifier.java | 2 +- .../windowing/MergeOverlappingIntervalWindows.java | 3 + .../beam/sdk/util/SerializableThrowable.java | 18 + .../beam/sdk/util/common/ReflectHelpers.java | 48 + .../java/org/apache/beam/sdk/values/RowType.java | 12 +- .../apache/beam/sdk/values/TimestampedValue.java | 6 + .../beam/sdk/coders/SerializableCoderTest.java | 49 + .../beam/sdk/coders/StructuredCoderTest.java | 3 +- .../java/org/apache/beam/sdk/io/FileIOTest.java | 75 + .../org/apache/beam/sdk/io/TextIOReadTest.java | 2 +- .../sdk/options/PipelineOptionsFactoryTest.java | 74 +- .../sdk/options/ProxyInvocationHandlerTest.java | 4 +- .../beam/sdk/testing/CombineFnTesterTest.java | 33 +- .../sdk/testing/InterceptingUrlClassLoader.java | 13 +- .../org/apache/beam/sdk/testing/PAssertTest.java | 1 + .../beam/sdk/transforms/ApproximateUniqueTest.java | 60 + .../apache/beam/sdk/transforms/CombineFnsTest.java | 16 + .../apache/beam/sdk/transforms/CombineTest.java | 132 +- .../apache/beam/sdk/transforms/DistinctTest.java | 61 + .../apache/beam/sdk/transforms/DoFnTesterTest.java | 1 + .../org/apache/beam/sdk/transforms/FilterTest.java | 75 + .../beam/sdk/transforms/FlatMapElementsTest.java | 43 + .../beam/sdk/transforms/GroupIntoBatchesTest.java | 2 +- .../beam/sdk/transforms/MapElementsTest.java | 63 + .../org/apache/beam/sdk/transforms/ParDoTest.java | 4 +- .../apache/beam/sdk/transforms/PartitionTest.java | 31 + .../beam/sdk/transforms/SimpleFunctionTest.java | 33 + .../org/apache/beam/sdk/transforms/WaitTest.java | 69 +- .../apache/beam/sdk/transforms/WithKeysTest.java | 34 + .../beam/sdk/transforms/WithTimestampsTest.java | 30 + .../display/DisplayDataMatchersTest.java | 2 +- .../sdk/transforms/windowing/WindowingTest.java | 1 + .../sdk/util/ExposedByteArrayOutputStreamTest.java | 2 +- .../beam/sdk/values/reflect/RowFactoryTest.java | 6 +- .../extensions/google-cloud-platform-core/pom.xml | 2 +- .../sdk/util/RetryHttpRequestInitializerTest.java | 2 +- sdks/java/extensions/jackson/pom.xml | 2 +- .../extensions/jackson/JacksonTransformsTest.java | 24 + sdks/java/extensions/join-library/pom.xml | 2 +- sdks/java/extensions/pom.xml | 2 +- sdks/java/extensions/protobuf/pom.xml | 2 +- sdks/java/extensions/sketching/pom.xml | 2 +- sdks/java/extensions/sorter/pom.xml | 2 +- sdks/java/extensions/sql/pom.xml | 2 +- .../apache/beam/sdk/extensions/sql/BeamSql.java | 1 + .../sdk/extensions/sql/BeamSqlSeekableTable.java | 10 + .../apache/beam/sdk/extensions/sql/BeamSqlUdf.java | 2 +- .../beam/sdk/extensions/sql/QueryTransform.java | 3 +- .../beam/sdk/extensions/sql/impl/BeamSqlEnv.java | 17 +- .../sql/impl/interpreter/BeamSqlFnExecutor.java | 4 + .../operator/BeamSqlDefaultExpression.java} | 28 +- .../interpreter/operator/BeamSqlPrimitive.java | 2 +- .../sql/impl/interpreter/operator/UdafImpl.java | 20 +- .../comparison/BeamSqlIsNotNullExpression.java | 2 +- .../comparison/BeamSqlIsNullExpression.java | 2 +- .../date/BeamSqlCurrentDateExpression.java | 1 + .../date/BeamSqlCurrentTimeExpression.java | 1 + .../date/BeamSqlCurrentTimestampExpression.java | 1 + .../operator/date/BeamSqlDateCeilExpression.java | 1 + .../operator/date/BeamSqlDateFloorExpression.java | 1 + .../operator/date/BeamSqlExtractExpression.java | 1 + .../operator/logical/BeamSqlLogicalExpression.java | 1 + .../sql/impl/planner/BeamQueryPlanner.java | 21 +- .../extensions/sql/impl/planner/BeamRuleSets.java | 54 +- .../sql/impl/rel/BeamAggregationRel.java | 19 +- .../sdk/extensions/sql/impl/rel/BeamFilterRel.java | 34 +- .../sdk/extensions/sql/impl/rel/BeamIOSinkRel.java | 85 +- .../extensions/sql/impl/rel/BeamIOSourceRel.java | 45 +- .../extensions/sql/impl/rel/BeamIntersectRel.java | 33 +- .../sdk/extensions/sql/impl/rel/BeamJoinRel.java | 328 +- .../sdk/extensions/sql/impl/rel/BeamMinusRel.java | 16 +- .../extensions/sql/impl/rel/BeamProjectRel.java | 37 +- .../sdk/extensions/sql/impl/rel/BeamRelNode.java | 15 +- .../sql/impl/rel/BeamSetOperatorRelBase.java | 14 +- .../sdk/extensions/sql/impl/rel/BeamSortRel.java | 71 +- .../sdk/extensions/sql/impl/rel/BeamUnionRel.java | 15 +- .../sdk/extensions/sql/impl/rel/BeamValuesRel.java | 40 +- .../extensions/sql/impl/rule/BeamIOSinkRule.java | 58 +- .../extensions/sql/impl/rule/BeamIOSourceRule.java | 30 +- .../sdk/extensions/sql/impl/rule/BeamJoinRule.java | 33 +- .../extensions/sql/impl/rule/BeamProjectRule.java | 37 +- .../sql/impl/schema/BeamPCollectionTable.java | 2 +- .../impl/transform/BeamAggregationTransforms.java | 2 +- .../sql/impl/transform/BeamJoinTransforms.java | 10 + .../sql/impl/transform/agg/CovarianceFn.java | 2 +- .../sql/impl/transform/agg/VarianceFn.java | 2 +- .../beam/sdk/extensions/sql/BeamSqlDslBase.java | 2 +- .../sdk/extensions/sql/BeamSqlDslJoinTest.java | 6 +- .../sdk/extensions/sql/BeamSqlDslUdfUdafTest.java | 120 +- .../interpreter/BeamSqlFnExecutorTestBase.java | 42 +- .../operator/BeamSqlCompareExpressionTest.java | 10 +- .../operator/BeamSqlInputRefExpressionTest.java | 4 +- .../interpreter/operator/BeamSqlPrimitiveTest.java | 10 +- .../sdk/extensions/sql/impl/rel/CheckSize.java | 1 + .../transform/agg/VarianceAccumulatorTest.java | 6 +- .../sql/impl/transform/agg/VarianceFnTest.java | 3 +- .../extensions/sql/mock/MockedBoundedTable.java | 2 +- .../extensions/sql/mock/MockedUnboundedTable.java | 2 +- sdks/java/fn-execution/pom.xml | 2 +- .../org/apache/beam/sdk/fn/stream/DataStreams.java | 3 +- sdks/java/harness/pom.xml | 2 +- .../fn/harness/state/StateFetchingIterators.java | 4 +- .../beam/fn/harness/BeamFnDataReadRunnerTest.java | 1 + .../beam/fn/harness/BeamFnDataWriteRunnerTest.java | 1 + sdks/java/io/amazon-web-services/pom.xml | 2 +- .../apache/beam/sdk/io/aws/s3/S3FileSystem.java | 132 +- .../beam/sdk/io/aws/s3/S3FileSystemTest.java | 228 +- sdks/java/io/amqp/pom.xml | 2 +- sdks/java/io/cassandra/pom.xml | 2 +- .../beam/sdk/io/cassandra/CassandraIOTest.java | 18 + sdks/java/io/common/pom.xml | 2 +- .../org/apache/beam/sdk/io/common/HashingFn.java | 19 + .../elasticsearch-tests-2/pom.xml | 2 +- .../elasticsearch-tests-5/pom.xml | 2 +- .../sdk/io/elasticsearch/ElasticsearchIOTest.java | 8 +- .../elasticsearch-tests-common/pom.xml | 2 +- .../io/elasticsearch/ElasticSearchIOTestUtils.java | 4 +- .../elasticsearch/ElasticsearchIOTestCommon.java | 2 +- sdks/java/io/elasticsearch-tests/pom.xml | 2 +- sdks/java/io/elasticsearch/pom.xml | 2 +- .../beam/sdk/io/elasticsearch/ElasticsearchIO.java | 13 +- sdks/java/io/file-based-io-tests/pom.xml | 3 +- .../java/org/apache/beam/sdk/io/avro/AvroIOIT.java | 4 +- sdks/java/io/google-cloud-platform/pom.xml | 2 +- .../beam/sdk/io/gcp/bigquery/BatchLoads.java | 50 +- .../beam/sdk/io/gcp/bigquery/BigQueryIO.java | 125 +- .../sdk/io/gcp/bigquery/BigQueryServicesImpl.java | 26 +- .../beam/sdk/io/gcp/bigquery/CreateTables.java | 3 +- .../gcp/bigquery/DynamicDestinationsHelpers.java | 1 + .../io/gcp/bigquery/PassThroughThenCleanup.java | 10 + .../beam/sdk/io/gcp/bigquery/TableDestination.java | 11 +- .../beam/sdk/io/gcp/bigquery/WriteTables.java | 27 +- .../beam/sdk/io/gcp/spanner/BatchSpannerRead.java | 161 + .../sdk/io/gcp/spanner/CreateTransactionFn.java | 29 +- .../beam/sdk/io/gcp/spanner/MutationGroup.java | 18 + .../sdk/io/gcp/spanner/NaiveSpannerReadFn.java | 85 - .../beam/sdk/io/gcp/spanner/OrderedCode.java | 2 +- .../beam/sdk/io/gcp/spanner/ReadOperation.java | 13 +- .../beam/sdk/io/gcp/spanner/SpannerAccessor.java | 9 +- .../beam/sdk/io/gcp/spanner/SpannerConfig.java | 5 +- .../apache/beam/sdk/io/gcp/spanner/SpannerIO.java | 51 +- .../beam/sdk/io/gcp/spanner/SpannerSchema.java | 19 + .../beam/sdk/io/gcp/spanner/Transaction.java | 10 +- .../cloud/spanner/FakeBatchTransactionId.java | 44 +- .../google/cloud/spanner/FakePartitionFactory.java | 43 + .../apache/beam/sdk/io/gcp/GcpApiSurfaceTest.java | 2 +- .../sdk/io/gcp/bigquery/TableRowJsonCoderTest.java | 1 + .../beam/sdk/io/gcp/bigtable/BigtableIOTest.java | 2 +- .../io/gcp/bigtable/BigtableServiceImplTest.java | 2 +- .../sdk/io/gcp/spanner/FakeServiceFactory.java | 13 + .../beam/sdk/io/gcp/spanner/SpannerIOReadTest.java | 329 +- .../beam/sdk/io/gcp/spanner/SpannerReadIT.java | 42 + sdks/java/io/hadoop-common/pom.xml | 2 +- sdks/java/io/hadoop-file-system/pom.xml | 2 +- sdks/java/io/hadoop-input-format/build.gradle | 7 + sdks/java/io/hadoop-input-format/pom.xml | 14 +- .../io/hadoop/inputformat/HadoopInputFormatIO.java | 2 +- .../inputformat/HadoopInputFormatIOTest.java | 2 +- sdks/java/io/hbase/pom.xml | 2 +- sdks/java/io/hcatalog/pom.xml | 2 +- sdks/java/io/jdbc/pom.xml | 4 +- .../java/org/apache/beam/sdk/io/jdbc/JdbcIO.java | 73 +- .../org/apache/beam/sdk/io/jdbc/JdbcIOTest.java | 18 +- sdks/java/io/jms/pom.xml | 2 +- .../java/org/apache/beam/sdk/io/jms/JmsIOTest.java | 2 +- sdks/java/io/kafka/README.md | 17 +- sdks/java/io/kafka/pom.xml | 2 +- .../org/apache/beam/sdk/io/kafka/ConsumerSpEL.java | 20 +- .../beam/sdk/io/kafka/KafkaCheckpointMark.java | 32 +- .../java/org/apache/beam/sdk/io/kafka/KafkaIO.java | 134 +- .../org/apache/beam/sdk/io/kafka/KafkaRecord.java | 14 +- .../apache/beam/sdk/io/kafka/KafkaRecordCoder.java | 3 + .../beam/sdk/io/kafka/KafkaTimestampType.java} | 43 +- .../beam/sdk/io/kafka/KafkaUnboundedReader.java | 200 +- .../apache/beam/sdk/io/kafka/TimestampPolicy.java | 68 + .../beam/sdk/io/kafka/TimestampPolicyFactory.java | 206 + .../org/apache/beam/sdk/io/kafka/KafkaIOTest.java | 151 +- sdks/java/io/kinesis/pom.xml | 2 +- .../beam/sdk/io/kinesis/KinesisReaderTest.java | 65 +- sdks/java/io/mongodb/pom.xml | 2 +- .../apache/beam/sdk/io/mongodb/MongoDbIOTest.java | 25 +- sdks/java/io/mqtt/pom.xml | 2 +- sdks/java/io/pom.xml | 2 +- sdks/java/io/redis/pom.xml | 2 +- sdks/java/io/solr/pom.xml | 2 +- sdks/java/io/tika/pom.xml | 2 +- sdks/java/io/xml/pom.xml | 2 +- sdks/java/java8tests/build.gradle | 31 - sdks/java/java8tests/pom.xml | 89 - .../options/PipelineOptionsFactoryJava8Test.java | 92 - .../beam/sdk/transforms/CombineJava8Test.java | 171 - .../beam/sdk/transforms/DistinctJava8Test.java | 97 - .../beam/sdk/transforms/FilterJava8Test.java | 114 - .../sdk/transforms/FlatMapElementsJava8Test.java | 84 - .../beam/sdk/transforms/MapElementsJava8Test.java | 97 - .../beam/sdk/transforms/PartitionJava8Test.java | 74 - .../sdk/transforms/SimpleFunctionJava8Test.java | 68 - .../beam/sdk/transforms/WithKeysJava8Test.java | 73 - .../sdk/transforms/WithTimestampsJava8Test.java | 71 - sdks/java/javadoc/pom.xml | 2 +- .../maven-archetypes/examples/generate-sources.sh | 26 +- sdks/java/maven-archetypes/examples/pom.xml | 2 +- sdks/java/maven-archetypes/pom.xml | 2 +- sdks/java/maven-archetypes/starter/pom.xml | 2 +- sdks/java/nexmark/pom.xml | 2 +- .../org/apache/beam/sdk/nexmark/model/Event.java | 4 +- sdks/java/pom.xml | 3 +- sdks/pom.xml | 2 +- sdks/python/apache_beam/coders/coder_impl.pxd | 2 + sdks/python/apache_beam/coders/coder_impl.py | 14 +- sdks/python/apache_beam/coders/coders.py | 9 +- sdks/python/apache_beam/coders/typecoders.py | 4 +- .../examples/complete/game/game_stats.py | 4 +- sdks/python/apache_beam/examples/complete/tfidf.py | 8 +- .../examples/cookbook/datastore_wordcount.py | 5 +- .../apache_beam/examples/snippets/snippets.py | 5 +- .../apache_beam/examples/streaming_wordcount.py | 4 +- .../apache_beam/examples/windowed_wordcount.py | 4 +- sdks/python/apache_beam/examples/wordcount.py | 4 +- .../apache_beam/examples/wordcount_debugging.py | 4 +- .../python/apache_beam/examples/wordcount_fnapi.py | 4 +- .../apache_beam/examples/wordcount_minimal.py | 4 +- sdks/python/apache_beam/io/filebasedsink.py | 4 +- sdks/python/apache_beam/io/filesystem_test.py | 5 + sdks/python/apache_beam/io/filesystemio_test.py | 5 + sdks/python/apache_beam/io/filesystems.py | 18 +- sdks/python/apache_beam/io/filesystems_test.py | 5 + .../apache_beam/io/gcp/datastore/v1/helper.py | 5 +- .../apache_beam/io/gcp/gcsfilesystem_test.py | 5 + sdks/python/apache_beam/io/gcp/pubsub.py | 180 +- sdks/python/apache_beam/io/gcp/pubsub_test.py | 179 +- .../python/apache_beam/io/hadoopfilesystem_test.py | 5 + .../io/hdfs_integration_test/Dockerfile | 38 + .../io/hdfs_integration_test/docker-compose.yml | 60 + .../hdfs_integration_test/hdfs_integration_test.sh | 40 + sdks/python/apache_beam/io/localfilesystem_test.py | 5 + sdks/python/apache_beam/metrics/cells.py | 15 +- sdks/python/apache_beam/metrics/execution.py | 7 +- sdks/python/apache_beam/options/value_provider.py | 3 + .../apache_beam/options/value_provider_test.py | 5 + sdks/python/apache_beam/pvalue.py | 28 + sdks/python/apache_beam/runners/common.pxd | 3 +- sdks/python/apache_beam/runners/common.py | 44 +- .../runners/dataflow/dataflow_metrics.py | 8 +- .../runners/dataflow/dataflow_runner.py | 4 + .../runners/dataflow/internal/apiclient.py | 3 +- .../runners/dataflow/internal/dependency.py | 4 +- .../apache_beam/runners/dataflow/internal/names.py | 1 + .../apache_beam/runners/direct/direct_runner.py | 14 +- sdks/python/apache_beam/runners/direct/executor.py | 7 +- .../runners/direct/sdf_direct_runner.py | 17 +- .../runners/direct/sdf_direct_runner_test.py | 29 +- .../runners/direct/transform_evaluator.py | 51 +- .../runners/portability/fn_api_runner.py | 45 +- .../runners/portability/fn_api_runner_test.py | 19 +- .../portability/universal_local_runner_test.py | 7 +- sdks/python/apache_beam/runners/sdf_common.py | 6 +- .../apache_beam/runners/worker/bundle_processor.py | 46 +- .../apache_beam/runners/worker/data_plane.py | 4 +- .../apache_beam/runners/worker/data_plane_test.py | 3 +- .../apache_beam/runners/worker/sdk_worker.py | 4 +- .../apache_beam/runners/worker/sdk_worker_main.py | 2 +- .../runners/worker/sdk_worker_main_test.py | 2 +- sdks/python/apache_beam/transforms/create_test.py | 5 + .../python/apache_beam/transforms/cy_combiners.pxd | 10 + sdks/python/apache_beam/transforms/cy_combiners.py | 97 + .../apache_beam/transforms/cy_combiners_test.py | 65 + sdks/python/apache_beam/transforms/display.py | 4 +- sdks/python/apache_beam/transforms/display_test.py | 6 +- .../apache_beam/transforms/ptransform_test.py | 3 +- sdks/python/apache_beam/transforms/trigger_test.py | 3 +- sdks/python/apache_beam/transforms/util.py | 7 +- sdks/python/apache_beam/transforms/util_test.py | 5 + sdks/python/apache_beam/typehints/typecheck.py | 11 +- .../python/apache_beam/typehints/typehints_test.py | 10 +- sdks/python/apache_beam/utils/counters.py | 3 + sdks/python/apache_beam/utils/profiler.py | 9 +- sdks/python/apache_beam/utils/retry.py | 7 +- sdks/python/apache_beam/version.py | 2 +- sdks/python/container/pom.xml | 2 +- sdks/python/pom.xml | 2 +- sdks/python/run_mini_py3lint.sh | 40 + sdks/python/run_postcommit.sh | 3 + sdks/python/run_pylint.sh | 17 + sdks/python/setup.py | 3 +- sdks/python/tox.ini | 21 +- settings.gradle | 1 - 621 files changed, 97527 insertions(+), 4176 deletions(-) -- To stop receiving notification emails like this one, please contact [email protected].
