This is an automated email from the ASF dual-hosted git repository. damccorm pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push: new 3e5894be029 [Playground] Make fields "categories" and "context_line" optional in examples tags (#26235) 3e5894be029 is described below commit 3e5894be02945058732a4b8f90a316fee7d7563b Author: Timur Sultanov <timur.sulta...@akvelon.com> AuthorDate: Tue May 2 17:52:31 2023 +0400 [Playground] Make fields "categories" and "context_line" optional in examples tags (#26235) * Make context_line and categories optional in the example tag. * Process multifile examples correctly in GetCatalog() * Fix example with context line pointing inside of tag --- .../Core Transforms/Combine/Combine PerKey/task.py | 2 +- .../db/mapper/precompiled_object_mapper.go | 9 +- .../db/mapper/precompiled_object_mapper_test.go | 36 ++-- playground/infrastructure/conftest.py | 8 +- playground/infrastructure/datastore_client.py | 2 +- playground/infrastructure/helper.py | 6 +- playground/infrastructure/models.py | 7 +- playground/infrastructure/test_helper.py | 193 +++++++++++++++++++-- 8 files changed, 224 insertions(+), 39 deletions(-) diff --git a/learning/katas/python/Core Transforms/Combine/Combine PerKey/task.py b/learning/katas/python/Core Transforms/Combine/Combine PerKey/task.py index 6ab01d20872..9a112a725a7 100644 --- a/learning/katas/python/Core Transforms/Combine/Combine PerKey/task.py +++ b/learning/katas/python/Core Transforms/Combine/Combine PerKey/task.py @@ -18,7 +18,7 @@ # name: CombinePerKey # description: Task from katas to implement the summation of scores per player. # multifile: false -# context_line: 29 +# context_line: 30 # categories: # - Combiners # complexity: BASIC diff --git a/playground/backend/internal/db/mapper/precompiled_object_mapper.go b/playground/backend/internal/db/mapper/precompiled_object_mapper.go index cf78a030f92..07481373c3f 100644 --- a/playground/backend/internal/db/mapper/precompiled_object_mapper.go +++ b/playground/backend/internal/db/mapper/precompiled_object_mapper.go @@ -61,10 +61,17 @@ func (pom *PrecompiledObjectMapper) ToArrayCategories(catalogDTO *dto.CatalogDTO numberOfExamples := len(catalogDTO.Examples) sdkToCategories := make(dto.SdkToCategories, 0) datasetBySnippetIDMap := catalogDTO.DatasetBySnippetIDMap + fileIdx := 0 for exampleIndx := 0; exampleIndx < numberOfExamples; exampleIndx++ { example := catalogDTO.Examples[exampleIndx] snippet := catalogDTO.Snippets[exampleIndx] - files := []*entity.FileEntity{catalogDTO.Files[exampleIndx]} + + var files []*entity.FileEntity + for idx := 0; idx < snippet.NumberOfFiles; idx++ { + files = append(files, catalogDTO.Files[fileIdx]) + fileIdx++ + } + var datasetsDTO []*dto.DatasetDTO if len(datasetBySnippetIDMap) != 0 { datasetsDTO = datasetBySnippetIDMap[snippet.Key.Name] diff --git a/playground/backend/internal/db/mapper/precompiled_object_mapper_test.go b/playground/backend/internal/db/mapper/precompiled_object_mapper_test.go index 7e90d6470c6..40a63714dc0 100644 --- a/playground/backend/internal/db/mapper/precompiled_object_mapper_test.go +++ b/playground/backend/internal/db/mapper/precompiled_object_mapper_test.go @@ -40,7 +40,7 @@ func TestPrecompiledObjectMapper_ToObjectInfo(t *testing.T) { Categories: []string{"MOCK_CAT_1", "MOCK_CAT_2", "MOCK_CAT_3"}, PipelineOptions: "MOCK_OPTIONS", Link: "MOCK_PATH", - Multifile: false, + Multifile: true, DefaultExample: false, ContextLine: 32, Sdk: pb.Sdk_SDK_JAVA, @@ -67,7 +67,7 @@ func TestPrecompiledObjectMapper_ToPrecompiledObj(t *testing.T) { Complexity: pb.Complexity_COMPLEXITY_MEDIUM, PipelineOptions: "MOCK_OPTIONS", Link: "MOCK_PATH", - Multifile: false, + Multifile: true, DefaultExample: false, ContextLine: 32, Sdk: pb.Sdk_SDK_JAVA, @@ -152,7 +152,7 @@ func TestPrecompiledObjectMapper_ToArrayCategories(t *testing.T) { javaCatalog.Categories[0].PrecompiledObjects[0].Name != "SDK_JAVA_MOCK_NAME" || javaCatalog.Categories[0].PrecompiledObjects[0].Description != "MOCK_DESCR" || javaCatalog.Categories[0].PrecompiledObjects[0].CloudPath != "SDK_JAVA_SDK_JAVA_MOCK_NAME" || - javaCatalog.Categories[0].PrecompiledObjects[0].Multifile != false || + javaCatalog.Categories[0].PrecompiledObjects[0].Multifile != true || javaCatalog.Categories[0].PrecompiledObjects[0].DefaultExample != false || javaCatalog.Categories[0].PrecompiledObjects[0].Link != "MOCK_PATH" || javaCatalog.Categories[0].PrecompiledObjects[0].PipelineOptions != "MOCK_OPTIONS" || @@ -167,7 +167,7 @@ func TestPrecompiledObjectMapper_ToArrayCategories(t *testing.T) { goCatalog.Categories[0].PrecompiledObjects[0].Name != "SDK_GO_MOCK_NAME" || goCatalog.Categories[0].PrecompiledObjects[0].Description != "MOCK_DESCR" || goCatalog.Categories[0].PrecompiledObjects[0].CloudPath != "SDK_GO_SDK_GO_MOCK_NAME" || - goCatalog.Categories[0].PrecompiledObjects[0].Multifile != false || + goCatalog.Categories[0].PrecompiledObjects[0].Multifile != true || goCatalog.Categories[0].PrecompiledObjects[0].DefaultExample != false || goCatalog.Categories[0].PrecompiledObjects[0].Link != "MOCK_PATH" || goCatalog.Categories[0].PrecompiledObjects[0].PipelineOptions != "MOCK_OPTIONS" || @@ -182,7 +182,7 @@ func TestPrecompiledObjectMapper_ToArrayCategories(t *testing.T) { pythonCatalog.Categories[0].PrecompiledObjects[0].Name != "SDK_PYTHON_MOCK_NAME" || pythonCatalog.Categories[0].PrecompiledObjects[0].Description != "MOCK_DESCR" || pythonCatalog.Categories[0].PrecompiledObjects[0].CloudPath != "SDK_PYTHON_SDK_PYTHON_MOCK_NAME" || - pythonCatalog.Categories[0].PrecompiledObjects[0].Multifile != false || + pythonCatalog.Categories[0].PrecompiledObjects[0].Multifile != true || pythonCatalog.Categories[0].PrecompiledObjects[0].DefaultExample != false || pythonCatalog.Categories[0].PrecompiledObjects[0].Link != "MOCK_PATH" || pythonCatalog.Categories[0].PrecompiledObjects[0].PipelineOptions != "MOCK_OPTIONS" || @@ -197,7 +197,7 @@ func TestPrecompiledObjectMapper_ToArrayCategories(t *testing.T) { scioCatalog.Categories[0].PrecompiledObjects[0].Name != "SDK_SCIO_MOCK_NAME" || scioCatalog.Categories[0].PrecompiledObjects[0].Description != "MOCK_DESCR" || scioCatalog.Categories[0].PrecompiledObjects[0].CloudPath != "SDK_SCIO_SDK_SCIO_MOCK_NAME" || - scioCatalog.Categories[0].PrecompiledObjects[0].Multifile != false || + scioCatalog.Categories[0].PrecompiledObjects[0].Multifile != true || scioCatalog.Categories[0].PrecompiledObjects[0].DefaultExample != false || scioCatalog.Categories[0].PrecompiledObjects[0].Link != "MOCK_PATH" || scioCatalog.Categories[0].PrecompiledObjects[0].PipelineOptions != "MOCK_OPTIONS" || @@ -234,15 +234,23 @@ func getExampleDTO(name, defaultName, sdk string) *dto.ExampleDTO { PipeOpts: "MOCK_OPTIONS", Origin: constants.ExampleOrigin, SchVer: utils.GetSchemaVerKey(pcObjMapperCtx, "MOCK_VERSION"), - NumberOfFiles: 1, + NumberOfFiles: 2, Complexity: pb.Complexity_COMPLEXITY_MEDIUM.String(), }, - Files: []*entity.FileEntity{{ - Name: "MOCK_NAME", - Content: "MOCK_CONTENT", - CntxLine: 32, - IsMain: true, - }}, + Files: []*entity.FileEntity{ + { + Name: "MOCK_NAME_0", + Content: "MOCK_CONTENT", + CntxLine: 32, + IsMain: true, + }, + { + Name: "MOCK_NAME_1", + Content: "MOCK_CONTENT", + CntxLine: 16, + IsMain: false, + }, + }, DefaultExampleName: defaultName, Datasets: []*dto.DatasetDTO{ { @@ -288,7 +296,7 @@ func getCatalogDTO() *dto.CatalogDTO { exampleDTO := getExampleDTO(utils.GetIDWithDelimiter(sdk.Name, "MOCK_NAME"), "MOCK_DEFAULT_EXAMPLE", sdk.Name) examples = append(examples, exampleDTO.Example) snippets = append(snippets, exampleDTO.Snippet) - files = append(files, exampleDTO.Files[0]) + files = append(files, exampleDTO.Files...) } return &dto.CatalogDTO{ Examples: examples, diff --git a/playground/infrastructure/conftest.py b/playground/infrastructure/conftest.py index face74f0e2b..73db43fe085 100644 --- a/playground/infrastructure/conftest.py +++ b/playground/infrastructure/conftest.py @@ -78,7 +78,7 @@ def create_test_example(create_test_tag): @pytest.fixture def create_test_tag(): - def _create_test_tag(with_kafka=False, is_multifile=False, **tag_meta) -> Tag: + def _create_test_tag(with_kafka=False, is_multifile=False, context_line=30, line_start=10, line_finish=20, **tag_meta) -> Tag: meta = { "name": "MOCK_NAME", "description": "MOCK_DESCRIPTION", @@ -112,9 +112,9 @@ def create_test_tag(): meta[k] = v return Tag( filepath="../../examples/MOCK_EXAMPLE/main.java", - line_start=10, - line_finish=20, - context_line=30, + line_start=line_start, + line_finish=line_finish, + context_line=context_line, **meta, ) diff --git a/playground/infrastructure/datastore_client.py b/playground/infrastructure/datastore_client.py index d5f59ce28bf..8cb03ba709a 100644 --- a/playground/infrastructure/datastore_client.py +++ b/playground/infrastructure/datastore_client.py @@ -352,7 +352,7 @@ class DatastoreClient: example.tag.name, example.sdk ), "content": example.code, - "cntxLine": example.tag.context_line, + "cntxLine": example.context_line, "isMain": True, } ) diff --git a/playground/infrastructure/helper.py b/playground/infrastructure/helper.py index 50ed4a20ea4..581d9d5c201 100644 --- a/playground/infrastructure/helper.py +++ b/playground/infrastructure/helper.py @@ -283,6 +283,10 @@ def _get_example(filepath: str, filename: str, tag: Tag, sdk: int) -> Example: Returns: Parsed Example object. """ + + # Calculate context line with tag removed. Note: context_line is 1-based, line_start and line_finish are 0-based. + context_line = tag.context_line if tag.context_line <= tag.line_start else tag.context_line - (tag.line_finish - tag.line_start) + return Example( sdk=SdkEnum(sdk), tag=tag, @@ -291,7 +295,7 @@ def _get_example(filepath: str, filename: str, tag: Tag, sdk: int) -> Example: type=_get_object_type(filename, filepath), code=_get_content(filepath, tag.line_start, tag.line_finish), url_vcs=_get_url_vcs(filepath), # type: ignore - context_line=tag.context_line - (tag.line_finish - tag.line_start), + context_line=context_line, ) diff --git a/playground/infrastructure/models.py b/playground/infrastructure/models.py index 904a3e14058..bf667eeac38 100644 --- a/playground/infrastructure/models.py +++ b/playground/infrastructure/models.py @@ -75,11 +75,11 @@ class Tag(BaseModel): filepath: str = Field(..., min_length=1) line_start: int line_finish: int - context_line: int + context_line: int = 1 name: str = Field(..., min_length=1) complexity: ComplexityEnum description: str - categories: List[str] + categories: List[str] = [] pipeline_options: str = "" datasets: Dict[str, Dataset] = {} emulators: List[Emulator] = [] @@ -98,7 +98,8 @@ class Tag(BaseModel): @root_validator(skip_on_failure=True) def lines_order(cls, values): assert ( - 0 < values["line_start"] < values["line_finish"] <= values["context_line"] + (0 <= values["line_start"] < values["line_finish"]) and + (values["context_line"] <= values["line_start"] or values["context_line"] > values["line_finish"]) ), f"line ordering error: {values}" return values diff --git a/playground/infrastructure/test_helper.py b/playground/infrastructure/test_helper.py index 164adf588e1..063f3285a61 100644 --- a/playground/infrastructure/test_helper.py +++ b/playground/infrastructure/test_helper.py @@ -129,8 +129,7 @@ async def test_get_statuses(mock_update_example_status, create_test_example): @mock.patch( "builtins.open", mock_open( - read_data=""" -// license line 1 + read_data="""// license line 1 // license line 2 // // beam-playground: @@ -171,8 +170,7 @@ def test_load_example(): sdk=SdkEnum.JAVA, type=PRECOMPILED_OBJECT_TYPE_EXAMPLE, filepath="../../examples/MOCK_EXAMPLE/main.java", - code=""" -// license line 1 + code="""// license line 1 // license line 2 // @@ -184,8 +182,8 @@ code line 2 context_line=5, tag=Tag( filepath="../../examples/MOCK_EXAMPLE/main.java", - line_start=4, - line_finish=27, + line_start=3, + line_finish=26, name="KafkaWordCount", description="Test example with Apache Kafka", multifile=False, @@ -208,6 +206,181 @@ code line 2 ) +@mock.patch( + "builtins.open", + mock_open( + read_data="""// license line 1 +// license line 2 +// +// beam-playground: +// name: KafkaWordCount +// description: Test example with Apache Kafka +// multifile: false +// context_line: 27 +// categories: +// - Filtering +// - Options +// - Quickstart +// complexity: MEDIUM +// tags: +// - filter +// - strings +// - emulator +// emulators: +// - type: kafka +// topic: +// id: topic_1 +// source_dataset: dataset_id_1 +// datasets: +// dataset_id_1: +// location: local +// format: json + +code line 1 +code line 2 + +""" + ), +) +def test_load_example_context_at_the_end_of_tag(): + example = _load_example( + "kafka.java", "../../examples/MOCK_EXAMPLE/main.java", SdkEnum.JAVA + ) + assert example == Example( + sdk=SdkEnum.JAVA, + type=PRECOMPILED_OBJECT_TYPE_EXAMPLE, + filepath="../../examples/MOCK_EXAMPLE/main.java", + code="""// license line 1 +// license line 2 +// + +code line 1 +code line 2 + +""", + url_vcs="https://github.com/apache/beam/blob/master/examples/MOCK_EXAMPLE/main.java", # type: ignore + context_line=4, + tag=Tag( + filepath="../../examples/MOCK_EXAMPLE/main.java", + line_start=3, + line_finish=26, + name="KafkaWordCount", + description="Test example with Apache Kafka", + multifile=False, + context_line=27, + categories=["Filtering", "Options", "Quickstart"], + complexity=ComplexityEnum.MEDIUM, + tags=["filter", "strings", "emulator"], + emulators=[ + Emulator( + type=EmulatorType.KAFKA, + topic=Topic(id="topic_1", source_dataset="dataset_id_1"), + ) + ], + datasets={ + "dataset_id_1": Dataset( + location=DatasetLocation.LOCAL, format=DatasetFormat.JSON + ) + }, + ), + ) + +@mock.patch( + "builtins.open", + mock_open( + read_data="""// license line 1 +// license line 2 +// +// beam-playground: +// name: KafkaWordCount +// description: Test example with Apache Kafka +// multifile: false +// context_line: 3 +// categories: +// - Filtering +// - Options +// - Quickstart +// complexity: MEDIUM +// tags: +// - filter +// - strings +// - emulator +// emulators: +// - type: kafka +// topic: +// id: topic_1 +// source_dataset: dataset_id_1 +// datasets: +// dataset_id_1: +// location: local +// format: json + +code line 1 +code line 2 + +""" + ), +) +def test_load_example_context_before_of_tag(): + example = _load_example( + "kafka.java", "../../examples/MOCK_EXAMPLE/main.java", SdkEnum.JAVA + ) + assert example == Example( + sdk=SdkEnum.JAVA, + type=PRECOMPILED_OBJECT_TYPE_EXAMPLE, + filepath="../../examples/MOCK_EXAMPLE/main.java", + code="""// license line 1 +// license line 2 +// + +code line 1 +code line 2 + +""", + url_vcs="https://github.com/apache/beam/blob/master/examples/MOCK_EXAMPLE/main.java", # type: ignore + context_line=3, + tag=Tag( + filepath="../../examples/MOCK_EXAMPLE/main.java", + line_start=3, + line_finish=26, + name="KafkaWordCount", + description="Test example with Apache Kafka", + multifile=False, + context_line=3, + categories=["Filtering", "Options", "Quickstart"], + complexity=ComplexityEnum.MEDIUM, + tags=["filter", "strings", "emulator"], + emulators=[ + Emulator( + type=EmulatorType.KAFKA, + topic=Topic(id="topic_1", source_dataset="dataset_id_1"), + ) + ], + datasets={ + "dataset_id_1": Dataset( + location=DatasetLocation.LOCAL, format=DatasetFormat.JSON + ) + }, + ), + ) + + +def test__validate_context_line_at_beggining_of_tag(create_test_tag): + with pytest.raises( + pydantic.ValidationError, + match="line ordering error", + ): + create_test_tag(context_line=4, line_start=3, line_finish=27) + + +def test__validate_context_line_at_end_of_tag(create_test_tag): + with pytest.raises( + pydantic.ValidationError, + match="line ordering error", + ): + create_test_tag(context_line=27, line_start=4, line_finish=27) + + def test__validate_without_name_field(create_test_tag): with pytest.raises( pydantic.ValidationError, @@ -232,14 +405,6 @@ def test__validate_with_incorrect_multifile_field(create_test_tag): create_test_tag(multifile="multifile") -def test__validate_without_categories_field(create_test_tag): - with pytest.raises( - pydantic.ValidationError, - match="field required", - ): - create_test_tag(categories=None) - - def test__validate_with_incorrect_categories_field(create_test_tag): with pytest.raises( pydantic.ValidationError,