This is an automated email from the ASF dual-hosted git repository.
domoritz pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/arrow.git
The following commit(s) were added to refs/heads/main by this push:
new be5f5b8f8f GH-39722: [JS] Clean up packaging (#39723)
be5f5b8f8f is described below
commit be5f5b8f8f8294d43681c59e1e67f7434a10861c
Author: Dominik Moritz <[email protected]>
AuthorDate: Wed Apr 17 23:21:42 2024 -0400
GH-39722: [JS] Clean up packaging (#39723)
* Remove .mts since it's the same as .ts
* Use .js consistently for all packages
* Closes: #39722
---------
Signed-off-by: Matt Topol <[email protected]>
Co-authored-by: ptaylor <[email protected]>
Co-authored-by: dependabot[bot]
<49699333+dependabot[bot]@users.noreply.github.com>
---
js/.vscode/launch.json | 26 ++++++------------
js/gulp/arrow-task.js | 9 +++---
js/gulp/closure-task.js | 68 ++++++++++++++++++++++++++++++++++++++++------
js/gulp/package-task.js | 10 +++----
js/gulp/typescript-task.js | 6 +---
5 files changed, 78 insertions(+), 41 deletions(-)
diff --git a/js/.vscode/launch.json b/js/.vscode/launch.json
index 21255b61c3..189b9d1f33 100644
--- a/js/.vscode/launch.json
+++ b/js/.vscode/launch.json
@@ -5,24 +5,14 @@
"version": "0.2.0",
"inputs": [
{
- "type": "pickString",
- "default": "src",
+ "type": "command",
"id": "TEST_TARGET",
- "options": [
- "src",
- "apache-arrow",
- "ts",
- "es5.cjs",
- "es5.esm",
- "es5.umd",
- "es2015.cjs",
- "es2015.esm",
- "es2015.umd",
- "esnext.cjs",
- "esnext.esm",
- "esnext.umd",
- ],
- "description": "The JS version + Module format combination to test (or
src to test source files)",
+ "command": "shellCommand.execute",
+ "args": {
+ "cwd": "${workspaceFolder}",
+ "description": "The JS version + Module format combination to test (or
src to test source files)",
+ "command": "echo
\"src\napache-arrow\nts\nes5.cjs\nes5.esm\nes5.umd\nes2015.cjs\nes2015.esm\nes2015.umd\nesnext.cjs\nesnext.esm\nesnext.umd\""
+ }
},
{
"type": "command",
@@ -50,7 +40,7 @@
"command": "shellCommand.execute",
"args": {
"useSingleResult": "true",
- "command": "case \"${input:TEST_TARGET}\" in *cjs | *umd |
apache-arrow) echo '';; *) echo '--experimental-vm-modules';; esac"
+ "command": "case \"${input:TEST_TARGET}\" in *cjs | *umd) echo
'--no-warnings';; *) echo '--experimental-vm-modules';; esac"
}
},
],
diff --git a/js/gulp/arrow-task.js b/js/gulp/arrow-task.js
index 855cb71283..2ec2043985 100644
--- a/js/gulp/arrow-task.js
+++ b/js/gulp/arrow-task.js
@@ -30,24 +30,23 @@ import { pipeline } from 'stream/promises';
export const arrowTask = ((cache) => memoizeTask(cache, function
copyMain(target) {
const out = targetDir(target);
- const dtsGlob = `${targetDir(`es2015`, `cjs`)}/**/*.ts`;
+ const dtsGlob = `${targetDir(`es2015`, `esm`)}/**/*.ts`;
const cjsGlob = `${targetDir(`es2015`, `cjs`)}/**/*.js`;
const esmGlob = `${targetDir(`es2015`, `esm`)}/**/*.js`;
const es2015UmdGlob = `${targetDir(`es2015`, `umd`)}/*.js`;
const esnextUmdGlob = `${targetDir(`esnext`, `umd`)}/*.js`;
- const cjsSourceMapsGlob = `${targetDir(`es2015`, `cjs`)}/**/*.map`;
+ const cjsSourceMapsGlob = `${targetDir(`es2015`, `cjs`)}/**/*.js.map`;
const esmSourceMapsGlob = `${targetDir(`es2015`, `esm`)}/**/*.map`;
const es2015UmdSourceMapsGlob = `${targetDir(`es2015`, `umd`)}/*.map`;
const esnextUmdSourceMapsGlob = `${targetDir(`esnext`, `umd`)}/*.map`;
return ObservableForkJoin([
observableFromStreams(gulp.src(dtsGlob), gulp.dest(out)), // copy d.ts
files
- observableFromStreams(gulp.src(dtsGlob), gulpRename((p) => { p.extname
= '.mts'; }), gulp.dest(out)), // copy d.ts files as esm
observableFromStreams(gulp.src(cjsGlob), gulp.dest(out)), // copy
es2015 cjs files
observableFromStreams(gulp.src(cjsSourceMapsGlob), gulp.dest(out)), //
copy es2015 cjs sourcemaps
- observableFromStreams(gulp.src(esmSourceMapsGlob), gulp.dest(out)), //
copy es2015 esm sourcemaps
+ observableFromStreams(gulp.src(esmSourceMapsGlob), gulpRename((p) => {
p.basename = p.basename.replace('.js', '.mjs'); }), gulpReplace(`.js"`,
`.mjs"`), gulp.dest(out)), // copy es2015 esm sourcemaps
observableFromStreams(gulp.src(es2015UmdSourceMapsGlob),
gulp.dest(out)), // copy es2015 umd sourcemap files, but don't rename
observableFromStreams(gulp.src(esnextUmdSourceMapsGlob),
gulp.dest(out)), // copy esnext umd sourcemap files, but don't rename
- observableFromStreams(gulp.src(esmGlob), gulpRename((p) => { p.extname
= '.mjs'; }), gulpReplace(`.js'`, `.mjs'`), gulp.dest(out)), // copy es2015 esm
files and rename to `.mjs`
+ observableFromStreams(gulp.src(esmGlob), gulpRename((p) => { p.extname
= '.mjs'; }), gulpReplace(`.js'`, `.mjs'`), gulpReplace(`.js.map`, `.mjs.map`),
gulp.dest(out)), // copy es2015 esm files and rename to `.mjs`
observableFromStreams(gulp.src(es2015UmdGlob), gulpRename((p) => {
p.basename += `.es2015.min`; }), gulp.dest(out)), // copy es2015 umd files and
add `.es2015.min`
observableFromStreams(gulp.src(esnextUmdGlob), gulpRename((p) => {
p.basename += `.esnext.min`; }), gulp.dest(out)), // copy esnext umd files and
add `.esnext.min`
]).pipe(share({ connector: () => new ReplaySubject(), resetOnError: false,
resetOnComplete: false, resetOnRefCountZero: false }));
diff --git a/js/gulp/closure-task.js b/js/gulp/closure-task.js
index c620784fd1..80f841bf72 100644
--- a/js/gulp/closure-task.js
+++ b/js/gulp/closure-task.js
@@ -19,8 +19,10 @@ import { targetDir, mainExport, esmRequire,
gCCLanguageNames, publicModulePaths,
import fs from 'node:fs';
import gulp from 'gulp';
-import path from 'node:path';
+import Path from 'node:path';
+import https from 'node:https';
import { mkdirp } from 'mkdirp';
+import { PassThrough } from 'node:stream';
import sourcemaps from 'gulp-sourcemaps';
import { memoizeTask } from './memoize-task.js';
import { compileBinFiles } from './typescript-task.js';
@@ -28,6 +30,12 @@ import { compileBinFiles } from './typescript-task.js';
import closureCompiler from 'google-closure-compiler';
const compiler = closureCompiler.gulp();
+import { fileURLToPath } from 'url';
+import { dirname } from 'path';
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = dirname(__filename);
+const closureCompilerVer = JSON.parse(fs.readFileSync(Path.join(__dirname,
'..',
'package.json'))).devDependencies['google-closure-compiler'].split('.')[0];
+
export const closureTask = ((cache) => memoizeTask(cache, async function
closure(target, format) {
if (shouldRunInChildProcess(target, format)) {
@@ -35,10 +43,10 @@ export const closureTask = ((cache) => memoizeTask(cache,
async function closure
}
const src = targetDir(target, `cls`);
- const srcAbsolute = path.resolve(src);
+ const srcAbsolute = Path.resolve(src);
const out = targetDir(target, format);
- const externs = path.join(`${out}/${mainExport}.externs.js`);
- const entry_point = path.join(`${src}/${mainExport}.dom.cls.js`);
+ const externs = Path.join(`${out}/${mainExport}.externs.js`);
+ const entry_point = Path.join(`${src}/${mainExport}.dom.cls.js`);
const exportedImports = publicModulePaths(srcAbsolute).reduce((entries,
publicModulePath) => [
...entries, {
@@ -54,12 +62,42 @@ export const closureTask = ((cache) => memoizeTask(cache,
async function closure
fs.promises.writeFile(entry_point,
generateUMDExportAssignment(srcAbsolute, exportedImports))
]);
- return await Promise.all([
+ const closeCompilerPolyfills = [];
+
+ await Promise.all([
runClosureCompileAsObservable().toPromise(),
- compileBinFiles(target, format).toPromise(),
- observableFromStreams(gulp.src(`${src}/**/*.d.ts`), gulp.dest(out)),
// copy .d.ts files
+ compileBinFiles(target, format).toPromise().then(() => Promise.all([
+ observableFromStreams(gulp.src(`${src}/**/*.d.ts`),
gulp.dest(out)).toPromise(), // copy .d.ts files,
+ observableFromStreams(gulp.src(`${src}/**/*.d.ts.map`),
gulp.dest(out)).toPromise(), // copy .d.ts.map files,
+ observableFromStreams(gulp.src(`${src}/src/**/*`),
gulp.dest(`${out}/src`)).toPromise(), // copy TS source files,
+ ]))
]);
+ // Download the closure compiler polyfill sources for sourcemaps
+ await Promise.all(closeCompilerPolyfills.map(async (path) => {
+
+ await fs.promises.mkdir(
+ Path.join(out, Path.parse(path).dir),
+ { recursive: true, mode: 0o755 }
+ );
+
+ const res = new PassThrough();
+ const req = https.request(
+ new
URL(`https://raw.githubusercontent.com/google/closure-compiler/v${closureCompilerVer}/${path}`),
+ (res_) => {
+ if (res_.statusCode === 200) {
+ res_.pipe(res);
+ } else {
+ res.end();
+ }
+ }
+ );
+
+ req.on('error', (e) => res.emit('error', e)).end();
+
+ return observableFromStreams(res, fs.createWriteStream(Path.join(out,
path))).toPromise();
+ }));
+
function runClosureCompileAsObservable() {
return observableFromStreams(
gulp.src([
@@ -72,8 +110,22 @@ export const closureTask = ((cache) => memoizeTask(cache,
async function closure
compiler(createClosureArgs(entry_point, externs, target), {
platform: ['native', 'java', 'javascript']
}),
+ sourcemaps.mapSources((path) => {
+ if (path.indexOf(`${src}/`) === 0) {
+ return path.slice(`${src}/`.length);
+ }
+ if (path.includes('com/google')) {
+ closeCompilerPolyfills.push(path);
+ return path.slice(`src/`.length);
+ }
+ return path;
+ }),
// rename the sourcemaps from *.js.map files to *.min.js.map
- sourcemaps.write(`.`, { mapFile: (mapPath) =>
mapPath.replace(`.js.map`, `.${target}.min.js.map`) }),
+ sourcemaps.write(`./`, {
+ sourceRoot: './src',
+ includeContent: false,
+ mapFile: (mapPath) => mapPath.replace(`.js.map`,
`.${target}.min.js.map`),
+ }),
gulp.dest(out)
);
}
diff --git a/js/gulp/package-task.js b/js/gulp/package-task.js
index 9bc002e664..0b0f4cfa20 100644
--- a/js/gulp/package-task.js
+++ b/js/gulp/package-task.js
@@ -56,7 +56,7 @@ const createMainPackageJson = (target, format) => (orig) => ({
'.': {
node: {
import: {
- types: `./${mainExport}.node.d.mts`,
+ types: `./${mainExport}.node.d.ts`,
default: `./${mainExport}.node.mjs`,
},
require: {
@@ -65,7 +65,7 @@ const createMainPackageJson = (target, format) => (orig) => ({
},
},
import: {
- types: `./${mainExport}.dom.d.mts`,
+ types: `./${mainExport}.dom.d.ts`,
default: `./${mainExport}.dom.mjs`,
},
require: {
@@ -75,7 +75,7 @@ const createMainPackageJson = (target, format) => (orig) => ({
},
'./*': {
import: {
- types: `./*.d.mts`,
+ types: `./*.d.ts`,
default: `./*.mjs`,
},
require: {
@@ -127,8 +127,8 @@ const createScopedPackageJSON = (target, format) => (({
name, ...orig }) =>
sideEffects: format === 'esm' ? false : undefined,
// include "esm" settings for https://www.npmjs.com/package/esm if
building scoped ESM target
esm: format === `esm` ? { mode: `auto`, sourceMap: true } :
undefined,
- // set "types" (for TypeScript/VSCode)
- types: format === 'umd' ? undefined : `${mainExport}.node.d.ts`,
+ // set "types" to "Arrow.dom" if building scoped UMD target,
otherwise "Arrow.node"
+ types: format === 'umd' ? `${mainExport}.dom.d.ts`:
`${mainExport}.node.d.ts`,
}
)
);
diff --git a/js/gulp/typescript-task.js b/js/gulp/typescript-task.js
index b5a4c3232d..ae5b389f7f 100644
--- a/js/gulp/typescript-task.js
+++ b/js/gulp/typescript-task.js
@@ -61,10 +61,9 @@ function compileTypescript(out, tsconfigPath,
tsconfigOverrides, writeSourcemaps
);
const writeSources = observableFromStreams(tsProject.src(),
gulp.dest(path.join(out, 'src')));
const writeDTypes = observableFromStreams(dts, sourcemaps.write('./', {
includeContent: false, sourceRoot: './src' }), gulp.dest(out));
- const mapFile = tsProject.options.module === tsc.ModuleKind.ES2015 ?
esmMapFile : cjsMapFile;
const writeJSArgs = writeSourcemaps ? [
js,
- sourcemaps.write('./', { mapFile, includeContent: false, sourceRoot:
'./src' }),
+ sourcemaps.write('./', { includeContent: false, sourceRoot: './src' }),
gulp.dest(out)
] : [
js,
@@ -73,6 +72,3 @@ function compileTypescript(out, tsconfigPath,
tsconfigOverrides, writeSourcemaps
const writeJS = observableFromStreams(...writeJSArgs);
return ObservableForkJoin([writeSources, writeDTypes, writeJS]);
}
-
-const cjsMapFile = (mapFilePath) => mapFilePath;
-const esmMapFile = (mapFilePath) => mapFilePath.replace('.js.map', '.mjs.map');
