bitflicker64 commented on code in PR #816:
URL:
https://github.com/apache/hugegraph-toolchain/pull/816#discussion_r4267202462
##########
.github/scripts/ci-policy.py:
##########
@@ -44,10 +44,82 @@
"toolchain": {"client-ci.yml": ["client"], "client-go-ci.yml": ["go"],
"loader-ci.yml": ["loader"], "tools-ci.yml": ["tools"],
"spark-connector-ci.yml": ["spark"], "hubble-ci.yml":
["hubble"],
- "codeql-analysis.yml": [], "dependency-license.yml": [],
"license-checker.yml": []},
+ "codeql-analysis.yml": [], "dependency-license.yml": [],
"license-checker.yml": [],
+ "ci.yml": MODULES["toolchain"], "spark-loader-ci.yml":
["loader"],
+ "flink-cdc-ci.yml": ["loader"], "hubble-launcher-ci.yml": [],
+ "upstream-compatibility.yml": [], "image-ci.yml": [],
"java17-images-ci.yml": [], "labeler.yml": []},
}
IMAGES = {"loader_image", "hubble_image"}
+# Optional compatibility checks follow direct inputs, rather than consumer
fan-out.
+LOADER = "hugegraph-loader/"
+SERVER_BOOTSTRAP = (".github/actions/get-hugegraph-commit/",
".github/actions/setup-candidate-sdk/",
+ ".github/actions/setup-hugegraph-server/",
".github/actions/setup-java-env/",
+ "hugegraph-client/assembly/travis/")
+SHARED_BUILD_INPUTS = {"pom.xml", "hugegraph-client/pom.xml",
"hugegraph-loader/pom.xml",
+ ".github/configs/settings.xml"}
+MAINTENANCE_INPUTS = {".asf.yaml", ".editorconfig", ".licenserc.yaml",
".github/PULL_REQUEST_TEMPLATE.md",
+ ".github/dependabot.yml", ".github/labeler.yml"}
+CONTRACT_TESTS = {".github/actions/setup-hugegraph-server/runtime_test.py",
+ ".github/actions/get-hugegraph-commit/resolve.test.cjs",
+ "hugegraph-client/assembly/travis/checkout-server.test.cjs",
+ "hugegraph-client/assembly/travis/install-server.test.cjs"}
+LAUNCHER_INPUTS =
("hugegraph-hubble/hubble-dist/assembly/static/bin/start-hubble.sh",
+
"hugegraph-hubble/hubble-dist/assembly/static/bin/stop-hubble.sh",
+
"hugegraph-hubble/hubble-dist/assembly/travis/test_hubble_launchers.py",
+ ".github/workflows/hubble-launcher-ci.yml")
+
+
+def entry_only(path):
+ return (path in MAINTENANCE_INPUTS or path in CONTRACT_TESTS
+ or path.startswith(".github/scripts/test_") and
path.endswith(".py"))
+
+
+def optional_checks(paths):
+ checks = {name: False for name in ("legacy", "hdfs", "spark_loader",
"flink_cdc", "launcher", "upstream")}
+ for path in paths:
+ if documentation(path) or entry_only(path):
+ continue
+ name = Path(path).name.lower()
+ shared = (path in SHARED_BUILD_INPUTS or path.startswith((".mvn/",) +
SERVER_BOOTSTRAP)
+ or path.startswith(LOADER + "assembly/descriptor/")
+ or path == LOADER +
"assembly/static/bin/hugegraph-loader.sh")
+ module_workflow = (path.startswith(".github/workflows/")
+ and
bool(WORKFLOWS["toolchain"].get(Path(path).name)))
Review Comment:
Minor: A PR that edits only .github/workflows/ci.yml selects none of the
callers that now live in that file.
With this change ci.yml owns the loader_image/hubble_image, spark-loader,
flink-cdc, launcher, codeql, upstream and smoke caller jobs. For a diff of only
.github/workflows/ci.yml, create_plan returns selectedImages=[] and
spark_loader, flink_cdc, launcher, upstream, security and mainline_smoke all
false (module_workflow only feeds legacy). A wrong condition, output reference
or input value in one of those caller blocks stays unexercised until the weekly
run or the next master push.
Select the images and the optional checks (or at least the callers whose
blocks changed) when ci.yml itself changes.
##########
.github/scripts/ci-policy.py:
##########
@@ -44,10 +44,82 @@
"toolchain": {"client-ci.yml": ["client"], "client-go-ci.yml": ["go"],
"loader-ci.yml": ["loader"], "tools-ci.yml": ["tools"],
"spark-connector-ci.yml": ["spark"], "hubble-ci.yml":
["hubble"],
- "codeql-analysis.yml": [], "dependency-license.yml": [],
"license-checker.yml": []},
+ "codeql-analysis.yml": [], "dependency-license.yml": [],
"license-checker.yml": [],
+ "ci.yml": MODULES["toolchain"], "spark-loader-ci.yml":
["loader"],
+ "flink-cdc-ci.yml": ["loader"], "hubble-launcher-ci.yml": [],
+ "upstream-compatibility.yml": [], "image-ci.yml": [],
"java17-images-ci.yml": [], "labeler.yml": []},
}
IMAGES = {"loader_image", "hubble_image"}
+# Optional compatibility checks follow direct inputs, rather than consumer
fan-out.
+LOADER = "hugegraph-loader/"
+SERVER_BOOTSTRAP = (".github/actions/get-hugegraph-commit/",
".github/actions/setup-candidate-sdk/",
+ ".github/actions/setup-hugegraph-server/",
".github/actions/setup-java-env/",
+ "hugegraph-client/assembly/travis/")
+SHARED_BUILD_INPUTS = {"pom.xml", "hugegraph-client/pom.xml",
"hugegraph-loader/pom.xml",
+ ".github/configs/settings.xml"}
+MAINTENANCE_INPUTS = {".asf.yaml", ".editorconfig", ".licenserc.yaml",
".github/PULL_REQUEST_TEMPLATE.md",
+ ".github/dependabot.yml", ".github/labeler.yml"}
+CONTRACT_TESTS = {".github/actions/setup-hugegraph-server/runtime_test.py",
+ ".github/actions/get-hugegraph-commit/resolve.test.cjs",
+ "hugegraph-client/assembly/travis/checkout-server.test.cjs",
+ "hugegraph-client/assembly/travis/install-server.test.cjs"}
+LAUNCHER_INPUTS =
("hugegraph-hubble/hubble-dist/assembly/static/bin/start-hubble.sh",
+
"hugegraph-hubble/hubble-dist/assembly/static/bin/stop-hubble.sh",
+
"hugegraph-hubble/hubble-dist/assembly/travis/test_hubble_launchers.py",
+ ".github/workflows/hubble-launcher-ci.yml")
+
+
+def entry_only(path):
+ return (path in MAINTENANCE_INPUTS or path in CONTRACT_TESTS
+ or path.startswith(".github/scripts/test_") and
path.endswith(".py"))
+
+
+def optional_checks(paths):
+ checks = {name: False for name in ("legacy", "hdfs", "spark_loader",
"flink_cdc", "launcher", "upstream")}
+ for path in paths:
+ if documentation(path) or entry_only(path):
+ continue
+ name = Path(path).name.lower()
+ shared = (path in SHARED_BUILD_INPUTS or path.startswith((".mvn/",) +
SERVER_BOOTSTRAP)
+ or path.startswith(LOADER + "assembly/descriptor/")
+ or path == LOADER +
"assembly/static/bin/hugegraph-loader.sh")
+ module_workflow = (path.startswith(".github/workflows/")
+ and
bool(WORKFLOWS["toolchain"].get(Path(path).name)))
+ checks["legacy"] |= (shared or module_workflow or "legacy" in name or
"compatibility" in name
+ or path ==
"hugegraph-client/src/main/java/org/apache/hugegraph/client/RestClient.java"
+ or path ==
"hugegraph-client/src/main/java/org/apache/hugegraph/util/JsonUtil.java")
+ checks["hdfs"] |= (shared or path == ".github/workflows/loader-ci.yml"
+ or path.startswith(LOADER) and "hdfs" in
path.lower())
+ client_runtime = path.startswith("hugegraph-client/src/main/")
Review Comment:
Important: Loader production code outside the spark/flink/hdfs paths no
longer selects the Spark Loader, Flink CDC or HDFS checks, while Client
production code (one hop further away) does.
HugeGraphSparkLoader and the Flink sink import loader
builder/ElementBuilder, VertexBuilder, EdgeBuilder, mapping/InputStruct and
executor/LoadOptions, and reader/hdfs/HDFSFileReader extends
reader/file/FileReader. A PR that only edits
hugegraph-loader/.../builder/ElementBuilder.java or reader/file/FileReader.java
plans spark_loader=false, flink_cdc=false, hdfs=false (checked by calling
create_plan with that diff). loader-ci runs only unit/file/jdbc/kafka, and
master pushes now run smoke only, so a regression in those paths first shows up
in the Monday run, after merge.
Treat hugegraph-loader/src/main/ like client_runtime for spark_loader and
flink_cdc, and select hdfs for reader/file and source/file changes, plus a test
for one such path.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
