Tyooughtul commented on code in PR #3381: URL: https://github.com/apache/fory/pull/3381#discussion_r2862085832
########## ci/extract_cpp_doc_code.py: ########## @@ -0,0 +1,377 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +""" +Extract C++ code examples from markdown documentation and generate test files. + +This script scans markdown files in docs/guide/cpp/, extracts ```cpp code blocks, +and generates compilable C++ test files that can be run to verify the documentation +examples are correct. +""" + +import argparse +import logging +import re +import sys +from pathlib import Path +from typing import List, Tuple + +logging.basicConfig( + level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s" +) + + +def extract_cpp_code_blocks(content: str) -> List[Tuple[str, int]]: + # Extract C++ code blocks from markdown content. + code_blocks = [] + pattern = r"```cpp\n(.*?)```" + + for match in re.finditer(pattern, content, re.DOTALL): + code = match.group(1).strip() + line_num = content[: match.start()].count("\n") + 1 + code_blocks.append((code, line_num)) + + return code_blocks + + +def is_complete_example(code: str) -> bool: + # Check if the code block is a complete, runnable example.A complete example should have a main function with all code inside it. + # Code blocks with statements outside of functions are not complete examples. + has_main = "int main()" in code or "int main (" in code + + # Check if there are statements outside of any function + # look for lines that look like function calls + # or object declarations at the top level + lines = code.split("\n") + brace_depth = 0 + in_main = False + + for line in lines: + stripped = line.strip() + if not stripped or stripped.startswith("//"): + continue + + # Track braces + for char in stripped: + if char == "{": + brace_depth += 1 + elif char == "}": + brace_depth -= 1 + + # Check if entering main + if "int main" in stripped: + in_main = True + continue + + # If code is not inside any braces and not in a struct/class declaration, + # and we see what looks like a function call or object usage, + # this is a uncompleted example + if brace_depth == 0 and not in_main: + # Skip struct/class/enum declarations + if any( + keyword in stripped + for keyword in [ + "struct ", + "class ", + "enum ", + "using ", + "namespace ", + "#include", + "FORY_STRUCT", + "FORY_ENUM", + ] + ): + continue + # Skip forward declarations + if stripped.endswith(";"): + continue + # If we see code that looks like it's executing (not declaring), + # this is not a complete example + if re.search(r"\w+\s*\([^)]*\)\s*;", stripped) and not re.search( + r"^(struct|class|enum|using|namespace|#include|FORY_)", stripped + ): + return False + + return has_main + + +def wrap_code_as_test(code: str, doc_file: str, block_index: int) -> str: + # Wrap a code snippet as a complete, compilable test file. + + includes = set() + + if "#include" not in code: + includes.add('#include "fory/serialization/fory.h"') + + if "std::string" in code and "#include <string>" not in code: + includes.add("#include <string>") + if "std::vector" in code and "#include <vector>" not in code: + includes.add("#include <vector>") + if "std::map" in code and "#include <map>" not in code: + includes.add("#include <map>") + if "std::set" in code and "#include <set>" not in code: + includes.add("#include <set>") + if "std::unordered_map" in code and "#include <unordered_map>" not in code: + includes.add("#include <unordered_map>") + if "std::unordered_set" in code and "#include <unordered_set>" not in code: + includes.add("#include <unordered_set>") + if "std::optional" in code and "#include <optional>" not in code: + includes.add("#include <optional>") + if "std::shared_ptr" in code and "#include <memory>" not in code: + includes.add("#include <memory>") + if "std::unique_ptr" in code and "#include <memory>" not in code: + includes.add("#include <memory>") + if "std::variant" in code and "#include <variant>" not in code: + includes.add("#include <variant>") + if "std::chrono" in code and "#include <chrono>" not in code: + includes.add("#include <chrono>") + if "std::make_shared" in code and "#include <memory>" not in code: + includes.add("#include <memory>") + if "std::make_unique" in code and "#include <memory>" not in code: + includes.add("#include <memory>") + if "assert(" in code and "#include <cassert>" not in code: + includes.add("#include <cassert>") + if "std::cout" in code and "#include <iostream>" not in code: + includes.add("#include <iostream>") + if "RowEncoder" in code and '#include "fory/encoder/row_encoder.h"' not in code: + includes.add('#include "fory/encoder/row_encoder.h"') + if "Row" in code and '#include "fory/row/row.h"' not in code: + includes.add('#include "fory/row/row.h"') + + include_section = "\n".join(sorted(includes)) + + if "int main()" in code or "int main (" in code: + # Only add namespace if not already present + if "using namespace" not in code: + code = f"using namespace fory::serialization;\n\n{code}" + return f"""// Auto-generated test from {doc_file} +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +{include_section} + +{code} +""" + else: + return f"""// Auto-generated test from {doc_file}, block {block_index} +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include <iostream> +{include_section} + +using namespace fory::serialization; + +{code} + +int main() {{ + std::cout << "Documentation example compiled successfully" << std::endl; + return 0; +}} +""" + + +def generate_test_file_name(doc_file: str, block_index: int) -> str: + # Generate a test file name from documentation file and block index. + + base_name = Path(doc_file).stem + return f"doc_test_{base_name}_{block_index}.cc" + + +def process_markdown_file(md_path: Path, output_dir: Path) -> List[Path]: + # logging.info(f"Processing {md_path}") + + with open(md_path, "r", encoding="utf-8") as f: + content = f.read() + + code_blocks = extract_cpp_code_blocks(content) + logging.info(f" Found {len(code_blocks)} C++ code blocks") + + generated_files = [] + + for i, (code, line_num) in enumerate(code_blocks): + if not is_complete_example(code): + logging.debug(f" Skipping incomplete example at line {line_num}") + continue + + test_content = wrap_code_as_test(code, md_path.name, i) + test_file_name = generate_test_file_name(md_path.name, i) + test_path = output_dir / test_file_name + + with open(test_path, "w", encoding="utf-8") as f: + f.write(test_content) + + generated_files.append(test_path) + logging.info(f" Generated {test_file_name}") + + return generated_files + + +def generate_bazel_build(test_files: List[Path], output_dir: Path) -> None: + build_path = output_dir / "BUILD" + + build_content = """# Licensed to the Apache Software Foundation (ASF) under one Review Comment: Thanks for comment! 😊 I‘ve fixed it. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected] --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
