This is an automated email from the ASF dual-hosted git repository. kenn pushed a commit to branch go-sdk in repository https://gitbox.apache.org/repos/asf/beam.git
commit f68f05ed3487f676d243879f1d7720d91040053d Merge: e6f3af2 192b4c7 Author: Kenn Knowles <[email protected]> AuthorDate: Mon Jan 8 09:25:41 2018 -0800 Merge pull request #4350: Integrate master into go-sdk .gitignore | 1 + ...job_beam_PerformanceTests_FileBasedIO_IT.groovy | 77 + ...stCommit_Python_ValidatesRunner_Dataflow.groovy | 2 +- .../job_beam_PostCommit_Python_Verify.groovy | 2 +- .../job_beam_PreCommit_Website_Merge.groovy | 7 +- .../job_beam_PreCommit_Website_Stage.groovy | 4 +- .../jenkins/job_beam_PreCommit_Website_Test.groovy | 4 +- build.gradle | 14 +- build_rules.gradle | 6 + examples/java/build.gradle | 2 +- examples/java/pom.xml | 6 +- examples/java8/pom.xml | 6 +- .../fn-execution/src/main/proto/beam_fn_api.proto | 37 +- pom.xml | 16 +- .../core/construction/PipelineResources.java | 57 + .../core/construction/PipelineResourcesTest.java | 76 + .../core/metrics/MetricsContainerStepMap.java | 19 + runners/flink/build.gradle | 14 +- runners/flink/pom.xml | 16 +- .../flink/FlinkBatchTransformTranslators.java | 27 +- .../beam/runners/flink/FlinkPipelineOptions.java | 1 - .../org/apache/beam/runners/flink/FlinkRunner.java | 36 +- runners/google-cloud-dataflow-java/pom.xml | 19 +- .../dataflow/DataflowPipelineTranslator.java | 37 +- .../beam/runners/dataflow/DataflowRunner.java | 62 +- .../options/DataflowPipelineWorkerPoolOptions.java | 1 - .../beam/runners/dataflow/DataflowRunnerTest.java | 36 - runners/java-fn-execution/pom.xml | 5 - runners/spark/build.gradle | 5 +- runners/spark/pom.xml | 96 +- .../beam/runners/spark/SparkContextOptions.java | 2 +- .../beam/runners/spark/SparkPipelineOptions.java | 14 +- .../org/apache/beam/runners/spark/SparkRunner.java | 13 + .../runners/spark/io/SparkUnboundedSource.java | 6 +- .../SparkGroupAlsoByWindowViaWindowSet.java | 5 +- .../spark/stateful/SparkTimerInternals.java | 11 +- .../spark/translation/MultiDoFnFunction.java | 5 +- .../spark/translation/SparkContextFactory.java | 5 + .../SparkGroupAlsoByWindowViaOutputBufferFn.java | 5 +- .../spark/translation/SparkProcessContext.java | 68 +- .../spark/translation/TranslationUtils.java | 39 +- .../runners/spark/util/GlobalWatermarkHolder.java | 24 +- .../main/java/org/apache/beam/sdk/io/AvroIO.java | 137 +- .../apache/beam/sdk/io/DefaultFilenamePolicy.java | 2 +- .../java/org/apache/beam/sdk/io/FileBasedSink.java | 52 +- .../main/java/org/apache/beam/sdk/io/FileIO.java | 997 +++++++- .../java/org/apache/beam/sdk/io/TFRecordIO.java | 36 + .../main/java/org/apache/beam/sdk/io/TextIO.java | 140 +- .../java/org/apache/beam/sdk/io/WriteFiles.java | 11 +- .../beam/sdk/options/PipelineOptionsFactory.java | 6 +- .../org/apache/beam/sdk/options/ValueProvider.java | 3 +- .../apache/beam/sdk/options/ValueProviders.java | 4 +- .../apache/beam/sdk/transforms/Requirements.java | 10 + .../beam/sdk/transforms/SerializableFunctions.java | 20 +- .../java/org/apache/beam/sdk/util/ShardedFile.java | 3 +- .../java/org/apache/beam/sdk/io/AvroIOTest.java | 275 ++- .../org/apache/beam/sdk/io/TFRecordIOTest.java | 137 +- .../org/apache/beam/sdk/io/TextIOWriteTest.java | 87 +- .../extensions/google-cloud-platform-core/pom.xml | 18 + sdks/java/extensions/sql/build.gradle | 11 +- sdks/java/extensions/sql/pom.xml | 1 + .../sql/impl/planner/BeamQueryPlanner.java | 10 + .../beam/sdk/extensions/sql/BeamSqlDslBase.java | 2 +- .../sdk/extensions/sql/BeamSqlNonAsciiTest.java | 57 + .../data/BeamFnDataBufferingOutboundObserver.java | 9 + .../BeamFnDataBufferingOutboundObserverTest.java | 19 + sdks/java/harness/pom.xml | 17 + .../fn/harness/control/ProcessBundleHandler.java | 1 - .../java/org/apache/beam/sdk/io/avro/AvroIOIT.java | 4 +- .../apache/beam/sdk/io/tfrecord/TFRecordIOIT.java | 4 +- sdks/java/io/google-cloud-platform/pom.xml | 17 + .../beam/sdk/io/gcp/bigtable/BigtableConfig.java | 261 ++ .../beam/sdk/io/gcp/bigtable/BigtableIO.java | 478 ++-- .../sdk/io/gcp/bigquery/BigQueryHelpersTest.java | 139 ++ .../sdk/io/gcp/bigquery/BigQueryIOReadTest.java | 787 ++++++ .../beam/sdk/io/gcp/bigquery/BigQueryIOTest.java | 2546 +------------------- .../sdk/io/gcp/bigquery/BigQueryIOWriteTest.java | 1388 +++++++++++ .../sdk/io/gcp/bigquery/FakeDatasetService.java | 51 +- .../beam/sdk/io/gcp/bigquery/FakeJobService.java | 20 +- .../beam/sdk/io/gcp/bigtable/BigtableIOTest.java | 27 +- sdks/java/io/hadoop/jdk1.8-tests/build.gradle | 1 + sdks/java/io/hadoop/jdk1.8-tests/pom.xml | 24 +- .../org/apache/beam/sdk/io/hbase/HBaseIOTest.java | 2 +- sdks/java/io/jdbc/pom.xml | 4 +- .../io/kafka/serialization/InstantSerializer.java | 2 +- sdks/java/io/kinesis/build.gradle | 6 +- sdks/java/io/kinesis/pom.xml | 27 +- .../org/apache/beam/sdk/io/kinesis/KinesisIO.java | 67 +- .../apache/beam/sdk/io/kinesis/KinesisReader.java | 2 +- .../beam/sdk/io/kinesis/AmazonKinesisMock.java | 46 +- .../beam/sdk/io/kinesis/KinesisReaderIT.java | 86 +- .../beam/sdk/io/kinesis/KinesisUploader.java | 28 +- .../java/org/apache/beam/sdk/io/xml/XmlIO.java | 265 +- .../java/org/apache/beam/sdk/io/xml/XmlSink.java | 156 -- .../java/org/apache/beam/sdk/io/xml/XmlIOTest.java | 270 +++ .../org/apache/beam/sdk/io/xml/XmlSinkTest.java | 280 --- .../org/apache/beam/sdk/io/xml/XmlSourceTest.java | 87 +- sdks/java/javadoc/pom.xml | 6 +- .../src/main/resources/archetype-resources/pom.xml | 6 +- .../src/main/resources/archetype-resources/pom.xml | 6 +- sdks/java/nexmark/build.gradle | 6 + sdks/java/nexmark/pom.xml | 29 +- .../apache/beam/sdk/nexmark/NexmarkLauncher.java | 121 +- .../apache/beam/sdk/nexmark/NexmarkOptions.java | 7 + .../org/apache/beam/sdk/nexmark/model/Auction.java | 8 +- .../org/apache/beam/sdk/nexmark/model/Person.java | 6 +- .../beam/sdk/nexmark/model/sql/BeamRecordSize.java | 134 ++ .../beam/sdk/nexmark/model/sql/ToBeamRecord.java | 82 + .../model/sql/adapter/ModelAdaptersMapping.java | 124 + .../model/sql/adapter/ModelFieldsAdapter.java} | 34 +- .../nexmark/model/sql/adapter/package-info.java} | 24 +- .../beam/sdk/nexmark/model/sql/package-info.java} | 24 +- .../beam/sdk/nexmark/queries/NexmarkQuery.java | 5 +- .../sdk/nexmark/queries/sql/NexmarkSqlQuery.java | 55 + .../beam/sdk/nexmark/queries/sql/SqlQuery0.java | 97 + .../sdk/nexmark/queries/sql/package-info.java} | 24 +- .../sdk/nexmark/model/sql/BeamRecordSizeTest.java | 112 + .../sdk/nexmark/model/sql/ToBeamRecordTest.java | 109 + .../sql/adapter/ModelAdaptersMappingTest.java | 159 ++ .../sdk/nexmark/queries/sql/SqlQuery0Test.java | 70 + sdks/python/apache_beam/coders/coders.py | 40 + sdks/python/apache_beam/coders/typecoders.py | 11 +- .../apache_beam/examples/snippets/snippets_test.py | 4 +- .../python/apache_beam/examples/wordcount_fnapi.py | 11 +- sdks/python/apache_beam/internal/gcp/auth.py | 8 +- sdks/python/apache_beam/io/filebasedsource.py | 2 +- sdks/python/apache_beam/io/filesystem.py | 4 +- sdks/python/apache_beam/io/filesystems.py | 1 + sdks/python/apache_beam/io/gcp/gcsio.py | 99 +- sdks/python/apache_beam/io/gcp/gcsio_test.py | 38 - sdks/python/apache_beam/io/hadoopfilesystem.py | 285 +++ .../python/apache_beam/io/hadoopfilesystem_test.py | 472 ++++ sdks/python/apache_beam/io/iobase.py | 15 +- sdks/python/apache_beam/io/localfilesystem_test.py | 2 +- sdks/python/apache_beam/io/range_trackers.py | 44 +- sdks/python/apache_beam/io/range_trackers_test.py | 37 - sdks/python/apache_beam/io/restriction_trackers.py | 131 + .../apache_beam/io/restriction_trackers_test.py | 159 ++ sdks/python/apache_beam/metrics/cells.py | 13 +- sdks/python/apache_beam/metrics/execution.py | 24 + sdks/python/apache_beam/pipeline.py | 4 +- sdks/python/apache_beam/pipeline_test.py | 14 +- sdks/python/apache_beam/runners/common.pxd | 34 +- sdks/python/apache_beam/runners/common.py | 160 +- .../runners/dataflow/dataflow_runner.py | 4 +- .../runners/dataflow/internal/apiclient_test.py | 131 +- .../runners/dataflow/internal/dependency.py | 60 +- .../apache_beam/runners/direct/direct_runner.py | 25 +- sdks/python/apache_beam/runners/direct/executor.py | 3 + .../runners/direct/sdf_direct_runner.py | 354 +++ .../runners/direct/sdf_direct_runner_test.py | 235 ++ .../runners/direct/transform_evaluator.py | 102 +- .../runners/portability/fn_api_runner.py | 547 +++-- .../runners/portability/fn_api_runner_test.py | 73 +- .../runners/portability/maptask_executor_runner.py | 4 +- .../runners/portability/universal_local_runner.py | 2 +- sdks/python/apache_beam/runners/runner.py | 17 +- sdks/python/apache_beam/runners/runner_test.py | 12 +- sdks/python/apache_beam/runners/sdf_common.py | 168 ++ .../apache_beam/runners/worker/bundle_processor.py | 65 +- .../apache_beam/runners/worker/data_plane.py | 62 +- .../apache_beam/runners/worker/operations.py | 9 +- .../apache_beam/runners/worker/sdk_worker.py | 175 +- .../apache_beam/runners/worker/sdk_worker_main.py | 38 +- .../runners/worker/sdk_worker_main_test.py | 33 +- .../apache_beam/runners/worker/sdk_worker_test.py | 68 +- sdks/python/apache_beam/transforms/core.py | 24 +- sdks/python/apache_beam/transforms/trigger.py | 3 + sdks/python/apache_beam/typehints/opcodes.py | 51 +- .../apache_beam/typehints/trivial_inference.py | 53 +- .../typehints/trivial_inference_test.py | 38 +- sdks/python/apache_beam/utils/urns.py | 3 + sdks/python/container/boot.go | 57 +- sdks/python/container/piputil.go | 131 + sdks/python/gen_protos.py | 3 +- sdks/python/generate_pydoc.sh | 7 +- sdks/python/setup.py | 15 +- sdks/python/tox.ini | 2 +- settings.gradle | 4 +- 179 files changed, 10518 insertions(+), 4948 deletions(-) -- To stop receiving notification emails like this one, please contact "[email protected]" <[email protected]>.
