This is an automated email from the ASF dual-hosted git repository.
tiagobento pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-kie-tools.git
The following commit(s) were added to refs/heads/main by this push:
new 92544da159e kie-issues#1490: Use `turbo`'s diffing algorithm to know
which packages are affected by changes to `pnpm-lock.yaml`, avoiding FULL
builds (#2632)
92544da159e is described below
commit 92544da159e1ebe2569f7548039c80c1a6d2c70b
Author: Thiago Lugli <[email protected]>
AuthorDate: Wed Oct 2 14:51:34 2024 -0300
kie-issues#1490: Use `turbo`'s diffing algorithm to know which packages are
affected by changes to `pnpm-lock.yaml`, avoiding FULL builds (#2632)
---
.../ci/build-partitioning/build_partitioning.ts | 55 +++++++++++++++-------
.../ci/build-partitioning/globals.ts | 10 +++-
.github/workflows/ci_build.yml | 10 +++-
.gitignore | 4 ++
devbox.json | 3 +-
5 files changed, 61 insertions(+), 21 deletions(-)
diff --git
a/.github/supporting-files/ci/build-partitioning/build_partitioning.ts
b/.github/supporting-files/ci/build-partitioning/build_partitioning.ts
index ccb9c42e811..b8f4893dd53 100644
--- a/.github/supporting-files/ci/build-partitioning/build_partitioning.ts
+++ b/.github/supporting-files/ci/build-partitioning/build_partitioning.ts
@@ -20,7 +20,7 @@
*/
import { Partial, None, Full, PartitionDefinition } from "./types";
-import { __ROOT_PKG_NAME, __NON_SOURCE_FILES_PATTERNS, __PACKAGES_ROOT_DIRS,
stdoutArray } from "./globals";
+import { __ROOT_PKG_NAME, __NON_SOURCE_FILES_PATTERNS, __PACKAGES_ROOT_PATHS,
stdoutArray } from "./globals";
import {
assertLeafPackagesInPartitionDefinitionsDontOverlap,
assertLeafPackagesInPartitionsExist,
@@ -56,7 +56,7 @@ const {
allowPositionals: true,
});
-const cwd = execSync("bash -c pwd").toString().trim();
+const absolutePosixRepoRootPath = execSync("bash -c pwd").toString().trim();
const __ARG_forceFull = forceFull === "true";
@@ -142,9 +142,9 @@ async function getPartitions(): Promise<Array<None | Full |
Partial>> {
});
const allPackageDirs = new Set(
- stdoutArray(await execSync(`bash -c "pnpm -F !${__ROOT_PKG_NAME}... exec
bash -c pwd"`).toString())
- .map((pkgDir) => path.relative(cwd, pkgDir))
- .map((pkgDir) => pkgDir.split(path.sep).join(path.posix.sep))
+ stdoutArray(await execSync(`bash -c "pnpm -F !${__ROOT_PKG_NAME}... exec
bash -c pwd"`).toString()).map((pkgDir) =>
+ convertToPosixPathRelativeToRepoRoot(pkgDir)
+ )
);
await assertCompleteness({ packageDirsByName, partitions:
partitionDefinitions, allPackageDirs });
@@ -155,18 +155,31 @@ async function getPartitions(): Promise<Array<None | Full
| Partial>> {
`bash -c "git diff --name-only ${__ARG_baseSha} ${__ARG_headSha} --
${nonSourceFilesPatternsForGitDiff}"`
).toString()
);
+ const changedPackages: Array<{ path: string; name: string }> = JSON.parse(
+ execSync(`bash -c "turbo ls
--filter='[${__ARG_baseSha}...${__ARG_headSha}]' --output json"`).toString()
+ ).packages.items.map((item: { path: string; name: string }) => ({
+ path: convertToPosixPathRelativeToRepoRoot(item.path),
+ name: item.name,
+ }));
+
+ const changedPackagesDirs = changedPackages.map((item) => item.path);
+
+ const changedPackagesNames = changedPackages.map((item) => item.name);
+
console.log("[build-partitioning] Changed source paths:");
console.log(new Set(changedSourcePaths));
const changedSourcePathsInRoot = changedSourcePaths.filter((path) =>
- __PACKAGES_ROOT_DIRS.every((pkgDir) => !path.startsWith(`${pkgDir}/`))
+ __PACKAGES_ROOT_PATHS.every((rootPath) => !path.startsWith(rootPath))
);
- const affectedPackageDirsInAllPartitions = stdoutArray(
- await execSync(`pnpm -F ...[${__ARG_baseSha}] exec bash -c pwd`).toString()
+ const affectedPackageDirsInAllPartitions: Array<string> = await JSON.parse(
+ execSync(
+ `bash -c "turbo ls ${changedPackagesNames.map((packageName) =>
`--filter='...${packageName}'`).join(" ")} --output json"`
+ ).toString()
)
- .map((pkgDir) => path.relative(cwd, pkgDir))
- .map((pkgDir) => pkgDir.split(path.sep).join(path.posix.sep));
+ .packages.items.map((item: { path: string }) => item.path)
+ .map((pkgDir) => convertToPosixPathRelativeToRepoRoot(pkgDir));
return await Promise.all(
partitionDefinitions.map(async (partition) => {
@@ -183,8 +196,8 @@ async function getPartitions(): Promise<Array<None | Full |
Partial>> {
};
}
- const changedSourcePathsInPartition = changedSourcePaths.filter((path) =>
- [...partition.dirs].some((partitionDir) =>
path.startsWith(`${partitionDir}/`))
+ const changedSourcePathsInPartition = changedPackagesDirs.filter((path)
=>
+ [...partition.dirs].some((partitionDir) =>
path.startsWith(`${partitionDir}`))
);
if (changedSourcePathsInPartition.length === 0) {
@@ -204,12 +217,11 @@ async function getPartitions(): Promise<Array<None | Full
| Partial>> {
);
const relevantPackageNamesInPartition = new Set(
- [...(await getDirsOfDependencies(affectedPackageNamesInPartition,
partition.name))].map(
+ [...(await
getDirsOfDependencies(affectedPackageNamesInPartition))].map(
(pkgDir) => packageNamesByDir.get(pkgDir)!
)
);
- console.log(`[build-partitioning] 'Partial' build of
'${partition.name}'`);
console.log(
`[build-partitioning] Building
${relevantPackageNamesInPartition.size}/${partition.dirs.size}/${allPackageDirs.size}
packages.`
);
@@ -240,8 +252,17 @@ async function getPartitions(): Promise<Array<None | Full
| Partial>> {
async function getDirsOfDependencies(leafPackageNames: Set<string>) {
const packagesFilter = [...leafPackageNames].map((pkgName) => `-F
${pkgName}...`).join(" ");
return new Set(
- stdoutArray(execSync(`pnpm ${packagesFilter} exec bash -c
pwd`).toString()) //
- .map((pkgDir) => path.relative(cwd, pkgDir))
- .map((pkgDir) => pkgDir.split(path.sep).join(path.posix.sep))
+ stdoutArray(execSync(`bash -c "pnpm ${packagesFilter} exec bash -c
pwd"`).toString()) //
+ .map((pkgDir) => convertToPosixPathRelativeToRepoRoot(pkgDir))
);
}
+
+function convertToPosixPathRelativeToRepoRoot(targetPath: string) {
+ // If it's not an absolute path we can assume it's relative to the
repository root (the current directory).
+ const isTargetPathAbsolute = targetPath.startsWith(path.sep) ||
targetPath.startsWith(path.posix.sep);
+
+ // Replace separators to make sure they are posix style.
+ return `${isTargetPathAbsolute ? path.relative(absolutePosixRepoRootPath,
targetPath) : targetPath}`
+ .split(path.sep)
+ .join(path.posix.sep);
+}
diff --git a/.github/supporting-files/ci/build-partitioning/globals.ts
b/.github/supporting-files/ci/build-partitioning/globals.ts
index c052e0da5a7..6f1acdb35f7 100644
--- a/.github/supporting-files/ci/build-partitioning/globals.ts
+++ b/.github/supporting-files/ci/build-partitioning/globals.ts
@@ -22,7 +22,15 @@ import * as fs from "fs";
export const __ROOT_PKG_NAME = "kie-tools-root";
-export const __PACKAGES_ROOT_DIRS = ["packages", "examples"];
+/**
+ * Root paths that can be ignored when checking individual changed files.
+ *
+ * `packages/` for the kie-tools packages, analysed by `turbo ls`.
+ * `examples/` for the kie-tools examples, analysed by `turbo ls`.
+ * `repo/` for the dependency graph. No PR should update the repository
dependency graph by itself.
+ * `pnpm-lock.yaml` is the lockfile for the repository dependencies, also
analysed by `turbo ls`.
+ */
+export const __PACKAGES_ROOT_PATHS = ["packages/", "examples/", "repo/",
"pnpm-lock.yaml"];
export const __NON_SOURCE_FILES_PATTERNS = stdoutArray(
fs.readFileSync(path.resolve(__dirname,
"../patterns/non-source-files-patterns.txt"), "utf-8")
diff --git a/.github/workflows/ci_build.yml b/.github/workflows/ci_build.yml
index ed4d2d7237d..e5d3c694fa2 100644
--- a/.github/workflows/ci_build.yml
+++ b/.github/workflows/ci_build.yml
@@ -76,7 +76,11 @@ jobs:
shell: bash
run: |
- npm -g install [email protected]
+ npm -g install [email protected] [email protected]
+
+ echo '{}' > turbo.json
+
+ turbo telemetry disable
bun
./.github/supporting-files/ci/build-partitioning/build_partitioning.ts \
--outputPath='/tmp/partitions.json' \
@@ -87,7 +91,9 @@ jobs:
--partition='./.github/supporting-files/ci/partitions/partition0.txt' \
--partition='./.github/supporting-files/ci/partitions/partition1.txt'
- npm -g uninstall bun
+ npm -g uninstall bun turbo
+
+ rm turbo.json
echo "mode=$(jq --raw-output '.[${{ matrix.partition }}].mode'
/tmp/partitions.json)" >> $GITHUB_OUTPUT
echo "bootstrapPnpmFilterString=$(jq --raw-output '.[${{
matrix.partition }}].bootstrapPnpmFilterString' /tmp/partitions.json)" >>
$GITHUB_OUTPUT
diff --git a/.gitignore b/.gitignore
index f2f664b4bb2..0352d83b6a6 100644
--- a/.gitignore
+++ b/.gitignore
@@ -336,3 +336,7 @@ packages/python-venv/venv
# devbox
.devbox
+
+# turbo
+.turbo
+turbo.json
diff --git a/devbox.json b/devbox.json
index e76efd9adf3..0268f10a97a 100644
--- a/devbox.json
+++ b/devbox.json
@@ -16,7 +16,8 @@
"DEVBOX_COREPACK_ENABLED": "true",
"GOFLAGS": "-modcacherw",
"GOPATH": "$DEVBOX_PROJECT_ROOT/.devbox/gopkgs",
- "PATH": "$DEVBOX_PROJECT_ROOT/.devbox/gopkgs/bin:$PATH"
+ "PNPM_HOME": "$DEVBOX_PROJECT_ROOT/.devbox/pnpm-store",
+ "PATH":
"$DEVBOX_PROJECT_ROOT/.devbox/gopkgs/bin:$DEVBOX_PROJECT_ROOT/.devbox/pnpm-store:$PATH"
},
"shell": {
"init_hook": [
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
