This is an automated email from the ASF dual-hosted git repository.
jason810496 pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new 11ef1fb2824 Verify TypeScript SDK package contents (#71399)
11ef1fb2824 is described below
commit 11ef1fb2824117316fb86a4cb0d9ed3836dd6ceb
Author: Jason(Zhe-You) Liu <[email protected]>
AuthorDate: Wed Aug 26 16:05:54 2026 +0800
Verify TypeScript SDK package contents (#71399)
* Verify TypeScript SDK package contents
* Add --all-files to verify-ts-sdk-package doc command
* Smoke-test the Dag-authoring entrypoints the package publishes
The consumer smoke test asserted a registerTask export that the SDK no
longer
has, so it would have failed every release regardless of whether the tarball
was sound. It now checks the Dag, DagRegistry, and serveDags entrypoints the
documented authoring API is built from, and reports which one is missing
instead of exiting without a message.
* Import every exports subpath in the package smoke test
The check only imported the package root, so the ./coordinator subpath was
confirmed present in the tarball and never actually resolved. A subpath can
ship and still fail to import when its exports conditions are wrong or an
internal import is broken, which is the failure a consumer would hit first.
Deriving the list from the exports keys also means a new subpath is covered
without touching this script.
* Unit-test the TypeScript SDK package verification helpers
The verification ran as top-level statements, so importing the module packed
the SDK and installed it from the registry, and none of the logic that
decides
what may ship could be exercised without a full release rehearsal. Splitting
the allowlist, entry-point derivation, pack-metadata parsing, and smoke-test
construction into importable functions behind an entrypoint guard puts that
logic under test, matching the shape of the release input validator.
---
ts-sdk/.pre-commit-config.yaml | 16 ++
ts-sdk/README.md | 17 +-
ts-sdk/package.json | 1 +
ts-sdk/scripts/ci/prek/verify_ts_sdk_package.py | 37 +++++
ts-sdk/scripts/verify-package.mjs | 204 ++++++++++++++++++++++++
ts-sdk/scripts/verify-package.test.mjs | 165 +++++++++++++++++++
ts-sdk/tsconfig.build.json | 4 +-
ts-sdk/vitest.config.ts | 2 +-
8 files changed, 442 insertions(+), 4 deletions(-)
diff --git a/ts-sdk/.pre-commit-config.yaml b/ts-sdk/.pre-commit-config.yaml
index 0d9a883f3df..508a913a7ee 100644
--- a/ts-sdk/.pre-commit-config.yaml
+++ b/ts-sdk/.pre-commit-config.yaml
@@ -74,3 +74,19 @@ repos:
additional_dependencies: ['[email protected]']
pass_filenames: false
require_serial: true
+ - id: verify-ts-sdk-package
+ name: Verify TypeScript SDK package artifact
+ entry: ./scripts/ci/prek/verify_ts_sdk_package.py
+ language: node
+ stages: [manual]
+ files: |
+ (?x)
+ ^src/.*\.ts$|
+ ^scripts/verify-package\.mjs$|
+ ^package\.json$|
+ ^pnpm-lock\.yaml$|
+ ^pnpm-workspace\.yaml$|
+ ^tsconfig(\.build)?\.json$
+ additional_dependencies: ['[email protected]']
+ pass_filenames: false
+ require_serial: true
diff --git a/ts-sdk/README.md b/ts-sdk/README.md
index 65568a5e877..8508b922b11 100644
--- a/ts-sdk/README.md
+++ b/ts-sdk/README.md
@@ -273,6 +273,7 @@ pnpm install
pnpm test
pnpm run typecheck
pnpm run build
+pnpm run verify:package
```
The committed lockfile and `pnpm-workspace.yaml` define the dependency security
@@ -281,11 +282,23 @@ can enter the lockfile, transitive dependencies cannot
use Git or arbitrary
tarball sources, and only explicitly approved dependencies can run lifecycle
build scripts. Review changes to both files together when updating
dependencies.
+`verify:package` creates the npm tarball, rejects files outside the published
+runtime allowlist, installs it into a clean temporary project, and smoke-tests
+every `exports` entry point and the `bin` executable. The required paths are
+derived from `package.json`, so a new export subpath is covered automatically.
+
+`tsconfig.build.json` turns off `sourceMap` and `declarationMap` that the base
+`tsconfig.json` enables. The published tarball ships `dist` but not `src`, so
+emitted maps would point at files the consumer never receives — the allowlist
+rejects them rather than shipping dangling maps. Local `pnpm run typecheck`
+still uses the base config, so editor tooling is unaffected.
+
Without a local pnpm install, [prek](https://prek.j178.dev) can compile the SDK
-with its own managed node + pnpm toolchain:
+or verify the package with its own managed node + pnpm toolchain:
```bash
-prek run compile-ts-sdk
+prek run compile-ts-sdk --all-files
+prek run --hook-stage manual verify-ts-sdk-package --all-files
```
## API reference
diff --git a/ts-sdk/package.json b/ts-sdk/package.json
index 1d182c23215..bf3b7fec5e4 100644
--- a/ts-sdk/package.json
+++ b/ts-sdk/package.json
@@ -48,6 +48,7 @@
"test": "vitest run",
"test:watch": "vitest",
"build": "pnpm run clean && tsc -p tsconfig.build.json",
+ "verify:package": "node scripts/verify-package.mjs",
"prepack": "pnpm run build",
"generate:supervisor": "node scripts/generate-supervisor.mjs"
},
diff --git a/ts-sdk/scripts/ci/prek/verify_ts_sdk_package.py
b/ts-sdk/scripts/ci/prek/verify_ts_sdk_package.py
new file mode 100755
index 00000000000..a1aba68d1c0
--- /dev/null
+++ b/ts-sdk/scripts/ci/prek/verify_ts_sdk_package.py
@@ -0,0 +1,37 @@
+#!/usr/bin/env python3
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+from __future__ import annotations
+
+import sys
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).resolve().parents[4] / "scripts" / "ci"
/ "prek"))
+
+from common_prek_utils import AIRFLOW_ROOT_PATH, run_command
+
+if __name__ not in ("__main__", "__mp_main__"):
+ raise SystemExit(
+ "This file is intended to be executed as an executable program. You
cannot use it as a module. "
+ f"To run this script, run the ./{__file__} command"
+ )
+
+if __name__ == "__main__":
+ directory = AIRFLOW_ROOT_PATH / "ts-sdk"
+ run_command(["pnpm", "config", "set", "store-dir", ".pnpm-store"],
cwd=directory)
+ run_command(["pnpm", "install", "--frozen-lockfile",
"--config.confirmModulesPurge=false"], cwd=directory)
+ run_command(["pnpm", "run", "verify:package"], cwd=directory)
diff --git a/ts-sdk/scripts/verify-package.mjs
b/ts-sdk/scripts/verify-package.mjs
new file mode 100644
index 00000000000..975f05f033f
--- /dev/null
+++ b/ts-sdk/scripts/verify-package.mjs
@@ -0,0 +1,204 @@
+/*!
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import console from "node:console";
+import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from
"node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import process from "node:process";
+import { spawnSync } from "node:child_process";
+import { pathToFileURL } from "node:url";
+
+// A packaging check must never be the reason a static-check run hangs: npm
install reaches
+// the registry, and pnpm pack shells out to a full TypeScript build.
+const COMMAND_TIMEOUT_MS = 10 * 60 * 1000;
+
+export const REQUIRED_ROOT_FILES = ["LICENSE", "NOTICE", "README.md",
"package.json"];
+
+// The Dag-authoring entrypoints a consumer must be able to reach from the
package root.
+export const REQUIRED_ROOT_EXPORTS = ["Dag", "DagRegistry", "serveDags"];
+
+function run(command, args, options = {}) {
+ const result = spawnSync(command, args, {
+ encoding: "utf8",
+ timeout: COMMAND_TIMEOUT_MS,
+ maxBuffer: 16 * 1024 * 1024,
+ ...options,
+ });
+ if (result.error) {
+ throw new Error(`${command} ${args.join(" ")} failed:
${result.error.message}`, {
+ cause: result.error,
+ });
+ }
+ if (result.status !== 0) {
+ process.stderr.write(result.stdout ?? "");
+ process.stderr.write(result.stderr ?? "");
+ throw new Error(`${command} ${args.join(" ")} failed with exit code
${result.status}`);
+ }
+ return result;
+}
+
+export function isAllowedFile(path) {
+ return REQUIRED_ROOT_FILES.includes(path) ||
/^dist\/.+\.(?:js|d\.ts)$/.test(path);
+}
+
+/**
+ * Every path a consumer can resolve — the `exports` conditions plus the `bin`
targets. Derived
+ * from package.json so adding an export subpath extends the check instead of
silently escaping it.
+ */
+export function collectEntryPoints(packageJson) {
+ const targets = Object.values(packageJson.exports ??
{}).flatMap((conditions) =>
+ typeof conditions === "string" ? [conditions] : Object.values(conditions),
+ );
+ targets.push(...Object.values(packageJson.bin ?? {}));
+ return targets.map((target) => target.replace(/^\.\//, ""));
+}
+
+/**
+ * The `exports` subpath keys as specifier suffixes: `"."` becomes `""` and
`"./coordinator"`
+ * becomes `"/coordinator"`. Presence in the tarball is not resolution — a
subpath can ship and
+ * still fail to import if its conditions are wrong or an internal import is
broken.
+ */
+export function collectExportSubpaths(packageJson) {
+ return Object.keys(packageJson.exports ?? { ".": {} }).map((subpath) =>
+ subpath === "." ? "" : subpath.replace(/^\./, ""),
+ );
+}
+
+/** The trailing JSON object of `pnpm pack --json` stdout, past the `prepack`
build banner. */
+export function parsePackMetadata(stdout) {
+ // `--silent` does not suppress the `prepack` lifecycle banner, so stdout is
build log followed
+ // by the JSON report — take the trailing object rather than parsing the
whole stream.
+ const metadataMatch = stdout.match(/(?:^|\n)(\{[\s\S]*\})\s*$/);
+ if (!metadataMatch) {
+ throw new Error("pnpm pack did not return package metadata");
+ }
+ const metadata = JSON.parse(metadataMatch[1]);
+ if (!Array.isArray(metadata.files) || typeof metadata.filename !== "string")
{
+ throw new Error("pnpm pack returned invalid package metadata");
+ }
+ return metadata;
+}
+
+/** Required-but-absent and present-but-disallowed paths in the packed
tarball. */
+export function diffPackagedFiles(packageJson, paths) {
+ const required = [...REQUIRED_ROOT_FILES,
...collectEntryPoints(packageJson)];
+ return {
+ missing: required.filter((path) => !paths.has(path)),
+ unexpected: [...paths].filter((path) => !isAllowedFile(path)),
+ };
+}
+
+/** The module body a fresh consumer runs to prove every published entry point
resolves. */
+export function buildImportScript(packageJson) {
+ return [
+ ...collectExportSubpaths(packageJson).map(
+ (subpath) => `await import(${JSON.stringify(packageJson.name +
subpath)});`,
+ ),
+ `const sdk = await import(${JSON.stringify(packageJson.name)});`,
+ `for (const name of ${JSON.stringify(REQUIRED_ROOT_EXPORTS)}) {`,
+ ' if (typeof sdk[name] !== "function") {',
+ " throw new Error(`the root export does not expose ${name}`);",
+ " }",
+ "}",
+ ].join("\n");
+}
+
+function verifyPackage() {
+ const temporaryDirectory = mkdtempSync(join(tmpdir(),
"airflow-ts-sdk-package-"));
+
+ try {
+ const packageJson = JSON.parse(readFileSync("package.json", "utf8"));
+
+ const packed = run("pnpm", [
+ "--silent",
+ "pack",
+ "--json",
+ "--pack-destination",
+ temporaryDirectory,
+ ]);
+ const metadata = parsePackMetadata(packed.stdout);
+ const paths = new Set(metadata.files.map(({ path }) => path));
+
+ const { missing, unexpected } = diffPackagedFiles(packageJson, paths);
+ if (missing.length > 0 || unexpected.length > 0) {
+ throw new Error(
+ [
+ missing.length > 0 ? `missing required files: ${missing.join(", ")}`
: "",
+ unexpected.length > 0 ? `unexpected files: ${unexpected.join(", ")}`
: "",
+ ]
+ .filter(Boolean)
+ .join("; "),
+ );
+ }
+
+ if (metadata.name !== packageJson.name || metadata.version !==
packageJson.version) {
+ throw new Error("packed package identity does not match package.json");
+ }
+
+ const consumerDirectory = join(temporaryDirectory, "consumer");
+ mkdirSync(consumerDirectory);
+ writeFileSync(
+ join(consumerDirectory, "package.json"),
+ JSON.stringify({ name: "airflow-ts-sdk-package-smoke-test", private:
true, type: "module" }),
+ );
+ run("npm", ["install", "--ignore-scripts", "--no-audit", "--no-fund",
metadata.filename], {
+ cwd: consumerDirectory,
+ });
+ run("node", ["--input-type=module", "--eval",
buildImportScript(packageJson)], {
+ cwd: consumerDirectory,
+ });
+
+ // Spawning the installed bin shim proves it exists, is executable, and
resolves its imports.
+ // Asserting only "exited non-zero, and blamed itself" keeps this a
packaging check — the CLI's
+ // own argument handling is covered by its unit tests.
+ const [binName] = Object.keys(packageJson.bin ?? {});
+ if (!binName) {
+ throw new Error("package.json declares no bin entry to verify");
+ }
+ const executable = join(consumerDirectory, "node_modules", ".bin",
binName);
+ const cli = spawnSync(executable, [], {
+ cwd: consumerDirectory,
+ encoding: "utf8",
+ timeout: COMMAND_TIMEOUT_MS,
+ });
+ if (cli.error) {
+ throw new Error(`${binName} failed: ${cli.error.message}`, { cause:
cli.error });
+ }
+ if (cli.status === 0 || !cli.stderr.includes(binName)) {
+ throw new Error(
+ `${binName} did not report a usage error when invoked without
arguments ` +
+ `(exit ${cli.status}, stderr: ${JSON.stringify(cli.stderr.trim())})`,
+ );
+ }
+
+ console.log(`Verified ${metadata.name}@${metadata.version} (${paths.size}
files)`);
+ } finally {
+ rmSync(temporaryDirectory, { recursive: true, force: true });
+ }
+}
+
+if (process.argv[1] !== undefined && import.meta.url ===
pathToFileURL(process.argv[1]).href) {
+ try {
+ verifyPackage();
+ } catch (error) {
+ console.error(error instanceof Error ? error.message : error);
+ process.exitCode = 1;
+ }
+}
diff --git a/ts-sdk/scripts/verify-package.test.mjs
b/ts-sdk/scripts/verify-package.test.mjs
new file mode 100644
index 00000000000..a1cb7b793f5
--- /dev/null
+++ b/ts-sdk/scripts/verify-package.test.mjs
@@ -0,0 +1,165 @@
+/*!
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import { describe, expect, it } from "vitest";
+import {
+ REQUIRED_ROOT_EXPORTS,
+ buildImportScript,
+ collectEntryPoints,
+ collectExportSubpaths,
+ diffPackagedFiles,
+ isAllowedFile,
+ parsePackMetadata,
+} from "./verify-package.mjs";
+
+const PACKAGE_JSON = {
+ name: "apache-airflow-ts-sdk",
+ exports: {
+ ".": { types: "./dist/index.d.ts", import: "./dist/index.js" },
+ "./coordinator": {
+ types: "./dist/coordinator/index.d.ts",
+ import: "./dist/coordinator/index.js",
+ },
+ },
+ bin: { "airflow-ts-pack": "./dist/cli/main.js" },
+};
+
+describe("collectEntryPoints", () => {
+ it("flattens every exports condition and bin target", () => {
+ expect(collectEntryPoints(PACKAGE_JSON)).toEqual([
+ "dist/index.d.ts",
+ "dist/index.js",
+ "dist/coordinator/index.d.ts",
+ "dist/coordinator/index.js",
+ "dist/cli/main.js",
+ ]);
+ });
+
+ it("accepts a bare string export target", () => {
+ expect(collectEntryPoints({ exports: { ".": "./dist/index.js" }
})).toEqual(["dist/index.js"]);
+ });
+
+ it("returns nothing when a package declares neither exports nor bin", () => {
+ expect(collectEntryPoints({})).toEqual([]);
+ });
+});
+
+describe("collectExportSubpaths", () => {
+ it("maps the root export to a bare specifier and keeps subpaths", () => {
+ expect(collectExportSubpaths(PACKAGE_JSON)).toEqual(["", "/coordinator"]);
+ });
+
+ it("assumes a root export when exports is absent", () => {
+ expect(collectExportSubpaths({})).toEqual([""]);
+ });
+});
+
+describe("isAllowedFile", () => {
+ it("allows the required root files and compiled dist output", () => {
+ for (const path of [
+ "LICENSE",
+ "NOTICE",
+ "README.md",
+ "package.json",
+ "dist/index.js",
+ "dist/index.d.ts",
+ "dist/cli/main.js",
+ ]) {
+ expect(isAllowedFile(path)).toBe(true);
+ }
+ });
+
+ it("rejects sources, source maps, and stray files", () => {
+ for (const path of [
+ "src/index.ts",
+ "dist/index.js.map",
+ "dist/index.d.ts.map",
+ "dist/notes.txt",
+ "dist/",
+ ".npmrc",
+ "tests/public-api.test.ts",
+ ]) {
+ expect(isAllowedFile(path)).toBe(false);
+ }
+ });
+});
+
+describe("parsePackMetadata", () => {
+ it("takes the trailing JSON object after the prepack banner", () => {
+ const stdout = [
+ "> [email protected] prepack",
+ "> pnpm run build",
+ "",
+
'{"name":"apache-airflow-ts-sdk","filename":"pkg.tgz","files":[{"path":"LICENSE"}]}',
+ "",
+ ].join("\n");
+ expect(parsePackMetadata(stdout).filename).toBe("pkg.tgz");
+ });
+
+ it("rejects stdout that carries no metadata object", () => {
+ expect(() => parsePackMetadata("> pnpm run build\n")).toThrow(
+ "pnpm pack did not return package metadata",
+ );
+ });
+
+ it("rejects metadata missing the files list or the filename", () => {
+ for (const payload of ['{"filename":"pkg.tgz"}', '{"files":[]}']) {
+ expect(() => parsePackMetadata(payload)).toThrow(
+ "pnpm pack returned invalid package metadata",
+ );
+ }
+ });
+});
+
+describe("diffPackagedFiles", () => {
+ it("reports nothing for a tarball holding exactly the allowed paths", () => {
+ const paths = new Set([
+ "LICENSE",
+ "NOTICE",
+ "README.md",
+ "package.json",
+ ...collectEntryPoints(PACKAGE_JSON),
+ ]);
+ expect(diffPackagedFiles(PACKAGE_JSON, paths)).toEqual({ missing: [],
unexpected: [] });
+ });
+
+ it("reports required paths that are absent and disallowed paths that are
present", () => {
+ const paths = new Set(["LICENSE", "NOTICE", "README.md", "package.json",
"src/index.ts"]);
+ const { missing, unexpected } = diffPackagedFiles(PACKAGE_JSON, paths);
+ expect(missing).toEqual(collectEntryPoints(PACKAGE_JSON));
+ expect(unexpected).toEqual(["src/index.ts"]);
+ });
+});
+
+describe("buildImportScript", () => {
+ it("imports every exports subpath before asserting the root entrypoints", ()
=> {
+ const script = buildImportScript(PACKAGE_JSON);
+ expect(script).toContain('await import("apache-airflow-ts-sdk");');
+ expect(script).toContain('await
import("apache-airflow-ts-sdk/coordinator");');
+ for (const name of REQUIRED_ROOT_EXPORTS) {
+ expect(script).toContain(name);
+ }
+ });
+
+ it("produces a module body that parses", () => {
+ expect(
+ () => new Function(`return async () =>
{${buildImportScript(PACKAGE_JSON)}}`),
+ ).not.toThrow();
+ });
+});
diff --git a/ts-sdk/tsconfig.build.json b/ts-sdk/tsconfig.build.json
index 7685d3fb626..0f3db46650d 100644
--- a/ts-sdk/tsconfig.build.json
+++ b/ts-sdk/tsconfig.build.json
@@ -3,7 +3,9 @@
"compilerOptions": {
"noEmit": false,
"outDir": "./dist",
- "rootDir": "./src"
+ "rootDir": "./src",
+ "sourceMap": false,
+ "declarationMap": false
},
"include": ["src/**/*.ts"],
"exclude": ["src/**/*.test.ts", "tests/**/*"]
diff --git a/ts-sdk/vitest.config.ts b/ts-sdk/vitest.config.ts
index d8c2aeb8d2b..723796e6d29 100644
--- a/ts-sdk/vitest.config.ts
+++ b/ts-sdk/vitest.config.ts
@@ -21,6 +21,6 @@ import { defineConfig } from "vitest/config";
export default defineConfig({
test: {
- include: ["tests/**/*.test.ts"],
+ include: ["tests/**/*.test.ts", "scripts/**/*.test.mjs"],
},
});
