Repository: beam Updated Branches: refs/heads/master 72b361e9a -> 9fc8198cd
Remove old imports meant for backwards compatibility Project: http://git-wip-us.apache.org/repos/asf/beam/repo Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/b7b56022 Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/b7b56022 Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/b7b56022 Branch: refs/heads/master Commit: b7b56022dac11e5fb344c93b6d8cbf49c4babd44 Parents: 72b361e Author: Sourabh Bajaj <[email protected]> Authored: Tue May 9 11:41:56 2017 -0700 Committer: Ahmet Altay <[email protected]> Committed: Tue May 9 12:08:00 2017 -0700 ---------------------------------------------------------------------- sdks/python/apache_beam/io/gcp/bigquery.py | 8 ++------ sdks/python/apache_beam/io/iobase.py | 5 ----- sdks/python/apache_beam/pvalue.py | 7 ------- .../runners/dataflow/internal/dependency.py | 2 +- .../runners/direct/watermark_manager.py | 4 ++-- .../portability/maptask_executor_runner.py | 3 ++- sdks/python/apache_beam/transforms/timeutil.py | 10 ---------- sdks/python/apache_beam/transforms/trigger.py | 4 ++-- sdks/python/apache_beam/transforms/window.py | 12 +++++------ .../apache_beam/transforms/window_test.py | 4 ++-- .../apache_beam/utils/pipeline_options.py | 21 -------------------- sdks/python/apache_beam/utils/value_provider.py | 21 -------------------- .../apache_beam/utils/windowed_value_test.py | 2 +- 13 files changed, 18 insertions(+), 85 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/beam/blob/b7b56022/sdks/python/apache_beam/io/gcp/bigquery.py ---------------------------------------------------------------------- diff --git a/sdks/python/apache_beam/io/gcp/bigquery.py b/sdks/python/apache_beam/io/gcp/bigquery.py index 1ed28a8..201c798 100644 --- a/sdks/python/apache_beam/io/gcp/bigquery.py +++ b/sdks/python/apache_beam/io/gcp/bigquery.py @@ -1120,12 +1120,8 @@ class BigQueryWrapper(object): value = cell['v'] if 'v' in cell else None if field.mode == 'REPEATED': if value is None: - # We receive 'None' for repeated fields without any values when - # 'flatten_results' is 'False'. - # When 'flatten_results' is 'True', we receive individual values - # instead of a list of values hence we do not hit this condition. - # We return an empty list here instead of 'None' to be consistent with - # other runners and to be backwards compatible to users. + # Ideally this should never happen as repeated fields default to + # returning an empty list result[field.name] = [] else: result[field.name] = [self._convert_cell_value_to_dict(x['v'], field) http://git-wip-us.apache.org/repos/asf/beam/blob/b7b56022/sdks/python/apache_beam/io/iobase.py ---------------------------------------------------------------------- diff --git a/sdks/python/apache_beam/io/iobase.py b/sdks/python/apache_beam/io/iobase.py index d47ef5b..e8ffb72 100644 --- a/sdks/python/apache_beam/io/iobase.py +++ b/sdks/python/apache_beam/io/iobase.py @@ -979,8 +979,3 @@ class _RoundRobinKeyFn(core.DoFn): if self.counter >= self.count: self.counter -= self.count yield self.counter, element - - -# For backwards compatibility. -# pylint: disable=wrong-import-position -from apache_beam.runners.dataflow.native_io.iobase import * http://git-wip-us.apache.org/repos/asf/beam/blob/b7b56022/sdks/python/apache_beam/pvalue.py ---------------------------------------------------------------------- diff --git a/sdks/python/apache_beam/pvalue.py b/sdks/python/apache_beam/pvalue.py index 2242c5a..fa91fe3 100644 --- a/sdks/python/apache_beam/pvalue.py +++ b/sdks/python/apache_beam/pvalue.py @@ -393,13 +393,6 @@ class AsDict(AsSideInput): return dict(it) -# For backwards compatibility with worker code. -SingletonPCollectionView = AsSingleton -IterablePCollectionView = AsIter -ListPCollectionView = AsList -DictPCollectionView = AsDict - - class EmptySideInput(object): """Value indicating when a singleton side input was empty. http://git-wip-us.apache.org/repos/asf/beam/blob/b7b56022/sdks/python/apache_beam/runners/dataflow/internal/dependency.py ---------------------------------------------------------------------- diff --git a/sdks/python/apache_beam/runners/dataflow/internal/dependency.py b/sdks/python/apache_beam/runners/dataflow/internal/dependency.py index 186ad3f..63e593c 100644 --- a/sdks/python/apache_beam/runners/dataflow/internal/dependency.py +++ b/sdks/python/apache_beam/runners/dataflow/internal/dependency.py @@ -72,7 +72,7 @@ from apache_beam.options.pipeline_options import SetupOptions # Update this version to the next version whenever there is a change that will # require changes to the execution environment. -BEAM_CONTAINER_VERSION = 'beam-0.7.0-20170505' +BEAM_CONTAINER_VERSION = '2.0.0' # Standard file names used for staging files. WORKFLOW_TARBALL_FILE = 'workflow.tar.gz' http://git-wip-us.apache.org/repos/asf/beam/blob/b7b56022/sdks/python/apache_beam/runners/direct/watermark_manager.py ---------------------------------------------------------------------- diff --git a/sdks/python/apache_beam/runners/direct/watermark_manager.py b/sdks/python/apache_beam/runners/direct/watermark_manager.py index 19d9085..7793478 100644 --- a/sdks/python/apache_beam/runners/direct/watermark_manager.py +++ b/sdks/python/apache_beam/runners/direct/watermark_manager.py @@ -23,8 +23,8 @@ import threading from apache_beam import pipeline from apache_beam import pvalue -from apache_beam.transforms.timeutil import MAX_TIMESTAMP -from apache_beam.transforms.timeutil import MIN_TIMESTAMP +from apache_beam.utils.timestamp import MAX_TIMESTAMP +from apache_beam.utils.timestamp import MIN_TIMESTAMP class WatermarkManager(object): http://git-wip-us.apache.org/repos/asf/beam/blob/b7b56022/sdks/python/apache_beam/runners/portability/maptask_executor_runner.py ---------------------------------------------------------------------- diff --git a/sdks/python/apache_beam/runners/portability/maptask_executor_runner.py b/sdks/python/apache_beam/runners/portability/maptask_executor_runner.py index c07f27f..3e08780 100644 --- a/sdks/python/apache_beam/runners/portability/maptask_executor_runner.py +++ b/sdks/python/apache_beam/runners/portability/maptask_executor_runner.py @@ -30,6 +30,7 @@ from apache_beam.options import pipeline_options from apache_beam.runners import DataflowRunner from apache_beam.runners.dataflow.internal.dependency import _dependency_file_copy from apache_beam.runners.dataflow.internal.names import PropertyNames +from apache_beam.runners.dataflow.native_io.iobase import NativeSource from apache_beam.runners.runner import PipelineResult from apache_beam.runners.runner import PipelineRunner from apache_beam.runners.runner import PipelineState @@ -144,7 +145,7 @@ class MapTaskExecutorRunner(PipelineRunner): def _run_read_from(self, transform_node, source): """Used when this operation is the result of reading source.""" - if not isinstance(source, iobase.NativeSource): + if not isinstance(source, NativeSource): source = iobase.SourceBundle(1.0, source, None, None) output = transform_node.outputs[None] element_coder = self._get_coder(output) http://git-wip-us.apache.org/repos/asf/beam/blob/b7b56022/sdks/python/apache_beam/transforms/timeutil.py ---------------------------------------------------------------------- diff --git a/sdks/python/apache_beam/transforms/timeutil.py b/sdks/python/apache_beam/transforms/timeutil.py index 435d41b..ba4ef36 100644 --- a/sdks/python/apache_beam/transforms/timeutil.py +++ b/sdks/python/apache_beam/transforms/timeutil.py @@ -23,16 +23,6 @@ from abc import ABCMeta from abc import abstractmethod -# For backwards compatibility. -# TODO(robertwb): Remove. -# pylint: disable=unused-import -from apache_beam.utils.timestamp import Duration -from apache_beam.utils.timestamp import MAX_TIMESTAMP -from apache_beam.utils.timestamp import MIN_TIMESTAMP -from apache_beam.utils.timestamp import Timestamp -# pylint: enable=unused-import - - class TimeDomain(object): """Time domain for streaming timers.""" http://git-wip-us.apache.org/repos/asf/beam/blob/b7b56022/sdks/python/apache_beam/transforms/trigger.py ---------------------------------------------------------------------- diff --git a/sdks/python/apache_beam/transforms/trigger.py b/sdks/python/apache_beam/transforms/trigger.py index bcb9dd3..97240d3 100644 --- a/sdks/python/apache_beam/transforms/trigger.py +++ b/sdks/python/apache_beam/transforms/trigger.py @@ -28,14 +28,14 @@ import copy from apache_beam.coders import observable from apache_beam.transforms import combiners from apache_beam.transforms import core -from apache_beam.transforms.timeutil import MAX_TIMESTAMP -from apache_beam.transforms.timeutil import MIN_TIMESTAMP from apache_beam.transforms.timeutil import TimeDomain from apache_beam.transforms.window import GlobalWindow from apache_beam.transforms.window import TimestampCombiner from apache_beam.transforms.window import WindowedValue from apache_beam.transforms.window import WindowFn from apache_beam.runners.api import beam_runner_api_pb2 +from apache_beam.utils.timestamp import MAX_TIMESTAMP +from apache_beam.utils.timestamp import MIN_TIMESTAMP class AccumulationMode(object): http://git-wip-us.apache.org/repos/asf/beam/blob/b7b56022/sdks/python/apache_beam/transforms/window.py ---------------------------------------------------------------------- diff --git a/sdks/python/apache_beam/transforms/window.py b/sdks/python/apache_beam/transforms/window.py index 44a5a26..29994c0 100644 --- a/sdks/python/apache_beam/transforms/window.py +++ b/sdks/python/apache_beam/transforms/window.py @@ -56,13 +56,13 @@ from google.protobuf import struct_pb2 from apache_beam import coders from apache_beam.runners.api import beam_runner_api_pb2 from apache_beam.transforms import timeutil -from apache_beam.transforms.timeutil import Duration -from apache_beam.transforms.timeutil import MAX_TIMESTAMP -from apache_beam.transforms.timeutil import MIN_TIMESTAMP -from apache_beam.transforms.timeutil import Timestamp -from apache_beam.utils.windowed_value import WindowedValue from apache_beam.utils import proto_utils from apache_beam.utils import urns +from apache_beam.utils.timestamp import Duration +from apache_beam.utils.timestamp import MAX_TIMESTAMP +from apache_beam.utils.timestamp import MIN_TIMESTAMP +from apache_beam.utils.timestamp import Timestamp +from apache_beam.utils.windowed_value import WindowedValue # TODO(ccy): revisit naming and semantics once Java Apache Beam finalizes their @@ -414,7 +414,7 @@ class Sessions(WindowFn): def merge(self, merge_context): to_merge = [] - end = timeutil.MIN_TIMESTAMP + end = MIN_TIMESTAMP for w in sorted(merge_context.windows, key=lambda w: w.start): if to_merge: if end > w.start: http://git-wip-us.apache.org/repos/asf/beam/blob/b7b56022/sdks/python/apache_beam/transforms/window_test.py ---------------------------------------------------------------------- diff --git a/sdks/python/apache_beam/transforms/window_test.py b/sdks/python/apache_beam/transforms/window_test.py index 0f613d7..2d2b03d 100644 --- a/sdks/python/apache_beam/transforms/window_test.py +++ b/sdks/python/apache_beam/transforms/window_test.py @@ -29,8 +29,6 @@ from apache_beam.transforms import GroupByKey from apache_beam.transforms import Map from apache_beam.transforms import WindowInto from apache_beam.transforms.core import Windowing -from apache_beam.transforms.timeutil import MAX_TIMESTAMP -from apache_beam.transforms.timeutil import MIN_TIMESTAMP from apache_beam.transforms.trigger import AccumulationMode from apache_beam.transforms.trigger import AfterCount from apache_beam.transforms.util import assert_that, equal_to @@ -44,6 +42,8 @@ from apache_beam.transforms.window import SlidingWindows from apache_beam.transforms.window import TimestampedValue from apache_beam.transforms.window import WindowedValue from apache_beam.transforms.window import WindowFn +from apache_beam.utils.timestamp import MAX_TIMESTAMP +from apache_beam.utils.timestamp import MIN_TIMESTAMP def context(element, timestamp): http://git-wip-us.apache.org/repos/asf/beam/blob/b7b56022/sdks/python/apache_beam/utils/pipeline_options.py ---------------------------------------------------------------------- diff --git a/sdks/python/apache_beam/utils/pipeline_options.py b/sdks/python/apache_beam/utils/pipeline_options.py deleted file mode 100644 index 51a17ed..0000000 --- a/sdks/python/apache_beam/utils/pipeline_options.py +++ /dev/null @@ -1,21 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -"""For backwards compatibility""" - -# pylint: disable=unused-import -from apache_beam.options.pipeline_options import * http://git-wip-us.apache.org/repos/asf/beam/blob/b7b56022/sdks/python/apache_beam/utils/value_provider.py ---------------------------------------------------------------------- diff --git a/sdks/python/apache_beam/utils/value_provider.py b/sdks/python/apache_beam/utils/value_provider.py deleted file mode 100644 index 46f39ae..0000000 --- a/sdks/python/apache_beam/utils/value_provider.py +++ /dev/null @@ -1,21 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -"""For backwards compatibility""" - -# pylint: disable=unused-import -from apache_beam.options.value_provider import * http://git-wip-us.apache.org/repos/asf/beam/blob/b7b56022/sdks/python/apache_beam/utils/windowed_value_test.py ---------------------------------------------------------------------- diff --git a/sdks/python/apache_beam/utils/windowed_value_test.py b/sdks/python/apache_beam/utils/windowed_value_test.py index f257410..7883d1d 100644 --- a/sdks/python/apache_beam/utils/windowed_value_test.py +++ b/sdks/python/apache_beam/utils/windowed_value_test.py @@ -22,7 +22,7 @@ import pickle import unittest from apache_beam.utils import windowed_value -from apache_beam.transforms.timeutil import Timestamp +from apache_beam.utils.timestamp import Timestamp class WindowedValueTest(unittest.TestCase):
