This closes #1872
Project: http://git-wip-us.apache.org/repos/asf/beam/repo Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/c3b97a28 Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/c3b97a28 Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/c3b97a28 Branch: refs/heads/master Commit: c3b97a2878a6ccb7b380cb7724ee0719a1d25d2e Parents: 847e4e9 2d7ce32 Author: Davor Bonaci <da...@google.com> Authored: Mon Jan 30 14:59:03 2017 -0800 Committer: Davor Bonaci <da...@google.com> Committed: Mon Jan 30 14:59:03 2017 -0800 ---------------------------------------------------------------------- .gitignore | 10 + .travis.yml | 21 +- pom.xml | 20 + sdks/pom.xml | 2 + sdks/python/.pylintrc | 164 + sdks/python/MANIFEST.in | 19 + sdks/python/README.md | 372 ++ sdks/python/apache_beam/__init__.py | 82 + sdks/python/apache_beam/coders/__init__.py | 19 + sdks/python/apache_beam/coders/coder_impl.pxd | 143 + sdks/python/apache_beam/coders/coder_impl.py | 597 +++ sdks/python/apache_beam/coders/coders.py | 707 +++ sdks/python/apache_beam/coders/coders_test.py | 115 + .../apache_beam/coders/coders_test_common.py | 355 ++ .../apache_beam/coders/fast_coders_test.py | 37 + sdks/python/apache_beam/coders/observable.py | 38 + .../apache_beam/coders/observable_test.py | 57 + .../coders/proto2_coder_test_messages_pb2.py | 318 ++ .../apache_beam/coders/slow_coders_test.py | 39 + sdks/python/apache_beam/coders/slow_stream.py | 154 + .../apache_beam/coders/standard_coders.yaml | 67 + .../apache_beam/coders/standard_coders_test.py | 136 + sdks/python/apache_beam/coders/stream.pxd | 63 + sdks/python/apache_beam/coders/stream.pyx | 215 + sdks/python/apache_beam/coders/stream_test.py | 169 + sdks/python/apache_beam/coders/typecoders.py | 186 + .../apache_beam/coders/typecoders_test.py | 117 + sdks/python/apache_beam/error.py | 42 + sdks/python/apache_beam/examples/__init__.py | 16 + .../apache_beam/examples/complete/__init__.py | 16 + .../examples/complete/autocomplete.py | 89 + .../examples/complete/autocomplete_test.py | 52 + .../examples/complete/estimate_pi.py | 125 + .../examples/complete/estimate_pi_test.py | 52 + .../examples/complete/juliaset/__init__.py | 16 + .../complete/juliaset/juliaset/__init__.py | 16 + .../complete/juliaset/juliaset/juliaset.py | 123 + .../complete/juliaset/juliaset/juliaset_test.py | 86 + .../examples/complete/juliaset/juliaset_main.py | 58 + .../examples/complete/juliaset/setup.py | 116 + .../apache_beam/examples/complete/tfidf.py | 208 + .../apache_beam/examples/complete/tfidf_test.py | 92 + .../examples/complete/top_wikipedia_sessions.py | 180 + .../complete/top_wikipedia_sessions_test.py | 62 + .../apache_beam/examples/cookbook/__init__.py | 16 + .../examples/cookbook/bigquery_schema.py | 130 + .../examples/cookbook/bigquery_side_input.py | 123 + .../cookbook/bigquery_side_input_test.py | 54 + .../examples/cookbook/bigquery_tornadoes.py | 99 + .../cookbook/bigquery_tornadoes_test.py | 44 + .../apache_beam/examples/cookbook/bigshuffle.py | 95 + .../examples/cookbook/bigshuffle_test.py | 63 + .../apache_beam/examples/cookbook/coders.py | 101 + .../examples/cookbook/coders_test.py | 49 + .../examples/cookbook/combiners_test.py | 74 + .../examples/cookbook/custom_ptransform.py | 134 + .../examples/cookbook/custom_ptransform_test.py | 53 + .../examples/cookbook/datastore_wordcount.py | 256 ++ .../apache_beam/examples/cookbook/filters.py | 107 + .../examples/cookbook/filters_test.py | 69 + .../examples/cookbook/group_with_coder.py | 122 + .../examples/cookbook/group_with_coder_test.py | 89 + .../examples/cookbook/mergecontacts.py | 133 + .../examples/cookbook/mergecontacts_test.py | 125 + .../examples/cookbook/multiple_output_pardo.py | 181 + .../cookbook/multiple_output_pardo_test.py | 72 + .../apache_beam/examples/snippets/__init__.py | 16 + .../apache_beam/examples/snippets/snippets.py | 1142 +++++ .../examples/snippets/snippets_test.py | 758 ++++ .../apache_beam/examples/streaming_wordcap.py | 64 + .../apache_beam/examples/streaming_wordcount.py | 74 + sdks/python/apache_beam/examples/wordcount.py | 109 + .../apache_beam/examples/wordcount_debugging.py | 166 + .../examples/wordcount_debugging_test.py | 59 + .../apache_beam/examples/wordcount_it_test.py | 59 + .../apache_beam/examples/wordcount_minimal.py | 121 + .../examples/wordcount_minimal_test.py | 59 + .../apache_beam/examples/wordcount_test.py | 58 + sdks/python/apache_beam/internal/__init__.py | 16 + sdks/python/apache_beam/internal/apiclient.py | 639 +++ .../apache_beam/internal/apiclient_test.py | 42 + sdks/python/apache_beam/internal/auth.py | 185 + sdks/python/apache_beam/internal/auth_test.py | 44 + .../apache_beam/internal/clients/__init__.py | 16 + .../internal/clients/bigquery/__init__.py | 27 + .../clients/bigquery/bigquery_v2_client.py | 659 +++ .../clients/bigquery/bigquery_v2_messages.py | 1910 ++++++++ .../internal/clients/dataflow/__init__.py | 27 + .../clients/dataflow/dataflow_v1b3_client.py | 683 +++ .../clients/dataflow/dataflow_v1b3_messages.py | 4173 ++++++++++++++++++ .../clients/dataflow/message_matchers.py | 124 + .../clients/dataflow/message_matchers_test.py | 69 + .../internal/clients/storage/__init__.py | 27 + .../clients/storage/storage_v1_client.py | 1038 +++++ .../clients/storage/storage_v1_messages.py | 1920 ++++++++ sdks/python/apache_beam/internal/json_value.py | 141 + .../apache_beam/internal/json_value_test.py | 84 + sdks/python/apache_beam/internal/module_test.py | 62 + sdks/python/apache_beam/internal/pickler.py | 230 + .../python/apache_beam/internal/pickler_test.py | 84 + sdks/python/apache_beam/internal/util.py | 127 + sdks/python/apache_beam/internal/util_test.py | 61 + .../python/apache_beam/internal/windmill_pb2.py | 2278 ++++++++++ .../internal/windmill_service_pb2.py | 164 + sdks/python/apache_beam/io/__init__.py | 31 + sdks/python/apache_beam/io/avroio.py | 345 ++ sdks/python/apache_beam/io/avroio_test.py | 349 ++ sdks/python/apache_beam/io/bigquery.py | 1081 +++++ sdks/python/apache_beam/io/bigquery_test.py | 806 ++++ sdks/python/apache_beam/io/concat_source.py | 263 ++ .../python/apache_beam/io/concat_source_test.py | 231 + .../python/apache_beam/io/datastore/__init__.py | 16 + .../apache_beam/io/datastore/v1/__init__.py | 16 + .../apache_beam/io/datastore/v1/datastoreio.py | 391 ++ .../io/datastore/v1/datastoreio_test.py | 247 ++ .../io/datastore/v1/fake_datastore.py | 92 + .../apache_beam/io/datastore/v1/helper.py | 267 ++ .../apache_beam/io/datastore/v1/helper_test.py | 256 ++ .../io/datastore/v1/query_splitter.py | 269 ++ .../io/datastore/v1/query_splitter_test.py | 201 + sdks/python/apache_beam/io/filebasedsource.py | 329 ++ .../apache_beam/io/filebasedsource_test.py | 708 +++ sdks/python/apache_beam/io/fileio.py | 797 ++++ sdks/python/apache_beam/io/fileio_test.py | 289 ++ sdks/python/apache_beam/io/gcsio.py | 820 ++++ sdks/python/apache_beam/io/gcsio_test.py | 748 ++++ sdks/python/apache_beam/io/iobase.py | 854 ++++ sdks/python/apache_beam/io/pubsub.py | 91 + sdks/python/apache_beam/io/pubsub_test.py | 62 + sdks/python/apache_beam/io/range_trackers.py | 493 +++ .../apache_beam/io/range_trackers_test.py | 538 +++ sdks/python/apache_beam/io/source_test_utils.py | 641 +++ .../apache_beam/io/source_test_utils_test.py | 123 + sdks/python/apache_beam/io/sources_test.py | 111 + sdks/python/apache_beam/io/textio.py | 318 ++ sdks/python/apache_beam/io/textio_test.py | 537 +++ sdks/python/apache_beam/io/tfrecordio.py | 271 ++ sdks/python/apache_beam/io/tfrecordio_test.py | 389 ++ sdks/python/apache_beam/metrics/__init__.py | 17 + sdks/python/apache_beam/metrics/cells.py | 315 ++ sdks/python/apache_beam/metrics/cells_test.py | 143 + sdks/python/apache_beam/metrics/execution.pxd | 31 + sdks/python/apache_beam/metrics/execution.py | 228 + .../apache_beam/metrics/execution_test.py | 131 + sdks/python/apache_beam/metrics/metric.py | 165 + sdks/python/apache_beam/metrics/metric_test.py | 85 + sdks/python/apache_beam/metrics/metricbase.py | 81 + sdks/python/apache_beam/pipeline.py | 442 ++ sdks/python/apache_beam/pipeline_test.py | 399 ++ sdks/python/apache_beam/pvalue.py | 495 +++ sdks/python/apache_beam/pvalue_test.py | 67 + sdks/python/apache_beam/runners/__init__.py | 29 + sdks/python/apache_beam/runners/common.pxd | 75 + sdks/python/apache_beam/runners/common.py | 448 ++ .../apache_beam/runners/dataflow/__init__.py | 16 + .../runners/dataflow/native_io/__init__.py | 16 + .../runners/dataflow/native_io/iobase.py | 311 ++ .../apache_beam/runners/dataflow_runner.py | 731 +++ .../apache_beam/runners/dataflow_runner_test.py | 63 + .../apache_beam/runners/direct/__init__.py | 19 + .../runners/direct/bundle_factory.py | 102 + sdks/python/apache_beam/runners/direct/clock.py | 50 + .../consumer_tracking_pipeline_visitor.py | 59 + .../consumer_tracking_pipeline_visitor_test.py | 126 + .../runners/direct/direct_metrics.py | 112 + .../runners/direct/direct_metrics_test.py | 211 + .../apache_beam/runners/direct/direct_runner.py | 164 + .../runners/direct/evaluation_context.py | 281 ++ .../apache_beam/runners/direct/executor.py | 568 +++ .../runners/direct/transform_evaluator.py | 568 +++ .../runners/direct/transform_result.py | 64 + .../runners/direct/watermark_manager.py | 224 + sdks/python/apache_beam/runners/runner.py | 351 ++ sdks/python/apache_beam/runners/runner_test.py | 217 + .../apache_beam/runners/template_runner_test.py | 88 + .../python/apache_beam/runners/test/__init__.py | 24 + .../runners/test/test_dataflow_runner.py | 40 + sdks/python/apache_beam/test_pipeline.py | 163 + sdks/python/apache_beam/test_pipeline_test.py | 112 + sdks/python/apache_beam/tests/__init__.py | 16 + sdks/python/apache_beam/tests/data/README.md | 20 + .../apache_beam/tests/data/privatekey.p12 | Bin 0 -> 2452 bytes .../apache_beam/tests/pipeline_verifiers.py | 119 + .../tests/pipeline_verifiers_test.py | 118 + sdks/python/apache_beam/tests/test_utils.py | 57 + sdks/python/apache_beam/transforms/__init__.py | 26 + .../python/apache_beam/transforms/aggregator.py | 120 + .../apache_beam/transforms/aggregator_test.py | 77 + sdks/python/apache_beam/transforms/combiners.py | 595 +++ .../apache_beam/transforms/combiners_test.py | 335 ++ sdks/python/apache_beam/transforms/core.py | 1473 +++++++ .../apache_beam/transforms/cy_combiners.pxd | 92 + .../apache_beam/transforms/cy_combiners.py | 306 ++ sdks/python/apache_beam/transforms/display.py | 329 ++ .../apache_beam/transforms/display_test.py | 205 + .../python/apache_beam/transforms/ptransform.py | 742 ++++ .../apache_beam/transforms/ptransform_test.py | 1953 ++++++++ .../python/apache_beam/transforms/sideinputs.py | 214 + .../apache_beam/transforms/sideinputs_test.py | 337 ++ sdks/python/apache_beam/transforms/timeutil.py | 133 + sdks/python/apache_beam/transforms/trigger.py | 974 ++++ .../apache_beam/transforms/trigger_test.py | 582 +++ .../transforms/trigger_transcripts.yaml | 224 + sdks/python/apache_beam/transforms/util.py | 235 + sdks/python/apache_beam/transforms/window.py | 355 ++ .../apache_beam/transforms/window_test.py | 214 + .../transforms/write_ptransform_test.py | 126 + sdks/python/apache_beam/typehints/__init__.py | 22 + sdks/python/apache_beam/typehints/decorators.py | 533 +++ sdks/python/apache_beam/typehints/opcodes.py | 334 ++ .../apache_beam/typehints/trivial_inference.py | 417 ++ .../typehints/trivial_inference_test.py | 151 + sdks/python/apache_beam/typehints/typecheck.py | 309 ++ .../typehints/typed_pipeline_test.py | 251 ++ sdks/python/apache_beam/typehints/typehints.py | 1061 +++++ .../apache_beam/typehints/typehints_test.py | 1062 +++++ sdks/python/apache_beam/utils/__init__.py | 22 + sdks/python/apache_beam/utils/annotations.py | 99 + .../apache_beam/utils/annotations_test.py | 126 + sdks/python/apache_beam/utils/counters.pxd | 30 + sdks/python/apache_beam/utils/counters.py | 183 + sdks/python/apache_beam/utils/dependency.py | 504 +++ .../python/apache_beam/utils/dependency_test.py | 425 ++ sdks/python/apache_beam/utils/names.py | 82 + sdks/python/apache_beam/utils/path.py | 47 + sdks/python/apache_beam/utils/path_test.py | 70 + .../apache_beam/utils/pipeline_options.py | 528 +++ .../apache_beam/utils/pipeline_options_test.py | 175 + .../utils/pipeline_options_validator.py | 200 + .../utils/pipeline_options_validator_test.py | 334 ++ sdks/python/apache_beam/utils/processes.py | 52 + sdks/python/apache_beam/utils/processes_test.py | 106 + sdks/python/apache_beam/utils/profiler.py | 69 + sdks/python/apache_beam/utils/retry.py | 200 + sdks/python/apache_beam/utils/retry_test.py | 210 + sdks/python/apache_beam/utils/timestamp.py | 213 + sdks/python/apache_beam/utils/timestamp_test.py | 168 + .../python/apache_beam/utils/windowed_value.pxd | 38 + sdks/python/apache_beam/utils/windowed_value.py | 122 + .../apache_beam/utils/windowed_value_test.py | 71 + sdks/python/apache_beam/version.py | 57 + sdks/python/pom.xml | 169 + sdks/python/run_postcommit.sh | 102 + sdks/python/run_pylint.sh | 53 + sdks/python/setup.cfg | 27 + sdks/python/setup.py | 146 + sdks/python/test_config.py | 44 + sdks/python/tox.ini | 38 + 248 files changed, 64912 insertions(+), 1 deletion(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/beam/blob/c3b97a28/pom.xml ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/beam/blob/c3b97a28/sdks/pom.xml ---------------------------------------------------------------------- diff --cc sdks/pom.xml index 150e1ae,45d8df0..3d0b893 --- a/sdks/pom.xml +++ b/sdks/pom.xml @@@ -33,8 -33,8 +33,9 @@@ <name>Apache Beam :: SDKs</name> <modules> + <module>common</module> <module>java</module> + <module>python</module> </modules> <profiles>