Repository: beam Updated Branches: refs/heads/master 0a901d948 -> fed80cec2
In some instances where a PipelineOptions subclass was defined in the main session and save_main_session option is enabled, that subclass may appear multiple times in the PipelineOptions.__subclassess__() list. This is causing problems with the argparse because options are not unique any more. This changes filter the subclasses by name, and pick the last unique instance of each subclass. Project: http://git-wip-us.apache.org/repos/asf/beam/repo Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/2ce77847 Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/2ce77847 Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/2ce77847 Branch: refs/heads/master Commit: 2ce7784731a0a2e4daaaefbc1bb475c0f2a575d4 Parents: 0a901d9 Author: Ahmet Altay <[email protected]> Authored: Wed Feb 1 10:39:59 2017 -0800 Committer: Ahmet Altay <[email protected]> Committed: Thu Feb 2 10:30:20 2017 -0800 ---------------------------------------------------------------------- sdks/python/apache_beam/utils/pipeline_options.py | 7 +++++++ .../apache_beam/utils/pipeline_options_test.py | 17 +++++++++++++++++ 2 files changed, 24 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/beam/blob/2ce77847/sdks/python/apache_beam/utils/pipeline_options.py ---------------------------------------------------------------------- diff --git a/sdks/python/apache_beam/utils/pipeline_options.py b/sdks/python/apache_beam/utils/pipeline_options.py index f7d7597..7840eb7 100644 --- a/sdks/python/apache_beam/utils/pipeline_options.py +++ b/sdks/python/apache_beam/utils/pipeline_options.py @@ -116,9 +116,16 @@ class PipelineOptions(HasDisplayData): Returns: Dictionary of all args and values. """ + + # TODO(BEAM-1319): PipelineOption sub-classes in the main session might be + # repeated. Pick last unique instance of each subclass to avoid conflicts. parser = argparse.ArgumentParser() + subset = {} for cls in PipelineOptions.__subclasses__(): + subset[str(cls)] = cls + for cls in subset.values(): cls._add_argparse_args(parser) # pylint: disable=protected-access + known_args, _ = parser.parse_known_args(self._flags) result = vars(known_args) http://git-wip-us.apache.org/repos/asf/beam/blob/2ce77847/sdks/python/apache_beam/utils/pipeline_options_test.py ---------------------------------------------------------------------- diff --git a/sdks/python/apache_beam/utils/pipeline_options_test.py b/sdks/python/apache_beam/utils/pipeline_options_test.py index 054b6a5..507a827 100644 --- a/sdks/python/apache_beam/utils/pipeline_options_test.py +++ b/sdks/python/apache_beam/utils/pipeline_options_test.py @@ -170,6 +170,23 @@ class PipelineOptionsTest(unittest.TestCase): options = PipelineOptions(flags=['']) self.assertEqual(options.get_all_options()['template_location'], None) + def test_redefine_options(self): + + class TestRedefinedOptios(PipelineOptions): # pylint: disable=unused-variable + + @classmethod + def _add_argparse_args(cls, parser): + parser.add_argument('--redefined_flag', action='store_true') + + class TestRedefinedOptios(PipelineOptions): + + @classmethod + def _add_argparse_args(cls, parser): + parser.add_argument('--redefined_flag', action='store_true') + + options = PipelineOptions(['--redefined_flag']) + self.assertEqual(options.get_all_options()['redefined_flag'], True) + if __name__ == '__main__': logging.getLogger().setLevel(logging.INFO) unittest.main()
