[AIRFLOW-2691] Manage JS dependencies via npm Closes #3572 from verdan/AIRFLOW-2691-npm-webpack
Project: http://git-wip-us.apache.org/repos/asf/incubator-airflow/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-airflow/commit/5a7f0b2e Tree: http://git-wip-us.apache.org/repos/asf/incubator-airflow/tree/5a7f0b2e Diff: http://git-wip-us.apache.org/repos/asf/incubator-airflow/diff/5a7f0b2e Branch: refs/heads/master Commit: 5a7f0b2e5333c936b39cdec06315d109ccf96934 Parents: 27fde38 Author: Verdan Mahmood <[email protected]> Authored: Sun Jul 22 09:23:57 2018 +0200 Committer: Bolke de Bruin <[email protected]> Committed: Sun Jul 22 09:24:11 2018 +0200 ---------------------------------------------------------------------- .gitignore | 6 + .rat-excludes | 2 + CONTRIBUTING.md | 57 + MANIFEST.in | 6 +- airflow/www_rbac/app.py | 3 +- airflow/www_rbac/compile_assets.sh | 23 + airflow/www_rbac/package-lock.json | 12408 ++++++++++++++ airflow/www_rbac/package.json | 55 + airflow/www_rbac/static/bootstrap-theme.css | 6494 -------- .../www_rbac/static/bootstrap-toggle.min.css | 28 - airflow/www_rbac/static/bootstrap-toggle.min.js | 9 - .../www_rbac/static/bootstrap3-typeahead.min.js | 26 - airflow/www_rbac/static/connection_form.js | 78 - airflow/www_rbac/static/css/bootstrap-theme.css | 6494 ++++++++ airflow/www_rbac/static/css/gantt.css | 57 + airflow/www_rbac/static/css/graph.css | 92 + airflow/www_rbac/static/css/main.css | 267 + airflow/www_rbac/static/css/tree.css | 96 + airflow/www_rbac/static/d3.tip.v0.6.3.js | 302 - airflow/www_rbac/static/d3.v3.min.js | 5 - airflow/www_rbac/static/dagre-d3.js | 5007 ------ airflow/www_rbac/static/dagre-d3.min.js | 2 - airflow/www_rbac/static/dagre.css | 38 - .../www_rbac/static/dataTables.bootstrap.css | 333 - airflow/www_rbac/static/gantt-chart-d3v2.js | 267 - airflow/www_rbac/static/gantt.css | 57 - airflow/www_rbac/static/graph.css | 72 - airflow/www_rbac/static/jqClock.min.js | 27 - airflow/www_rbac/static/jquery.dataTables.css | 495 - .../www_rbac/static/jquery.dataTables.min.js | 189 - airflow/www_rbac/static/js/clock.js | 41 + airflow/www_rbac/static/js/connection_form.js | 80 + airflow/www_rbac/static/js/dagre-d3.js | 5007 ++++++ airflow/www_rbac/static/js/gantt-chart-d3v2.js | 267 + airflow/www_rbac/static/js/jqClock.min.js | 27 + airflow/www_rbac/static/main.css | 267 - airflow/www_rbac/static/nv.d3.css | 788 - airflow/www_rbac/static/nv.d3.js | 14260 ----------------- airflow/www_rbac/static/tree.css | 96 - airflow/www_rbac/static_config.py | 63 + airflow/www_rbac/templates/airflow/chart.html | 11 +- airflow/www_rbac/templates/airflow/circles.html | 8 +- .../www_rbac/templates/airflow/conn_create.html | 8 +- .../www_rbac/templates/airflow/conn_edit.html | 8 +- airflow/www_rbac/templates/airflow/dag.html | 23 +- airflow/www_rbac/templates/airflow/dags.html | 17 +- .../templates/airflow/duration_chart.html | 11 +- airflow/www_rbac/templates/airflow/gantt.html | 10 +- airflow/www_rbac/templates/airflow/graph.html | 10 +- airflow/www_rbac/templates/airflow/tree.html | 7 +- .../templates/appbuilder/baselayout.html | 35 +- airflow/www_rbac/webpack.config.js | 128 + dev/README.md | 15 +- setup.py | 19 + 54 files changed, 25274 insertions(+), 28927 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/5a7f0b2e/.gitignore ---------------------------------------------------------------------- diff --git a/.gitignore b/.gitignore index 0e53aad..9749b1a 100644 --- a/.gitignore +++ b/.gitignore @@ -138,3 +138,9 @@ rat-results.txt *.generated *.tar.gz scripts/ci/kubernetes/kube/.generated/airflow.yaml + +# Node & Webpack Stuff +*.entry.js +node_modules +npm-debug.log* +static/dist http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/5a7f0b2e/.rat-excludes ---------------------------------------------------------------------- diff --git a/.rat-excludes b/.rat-excludes index e5373aa..0164f99 100644 --- a/.rat-excludes +++ b/.rat-excludes @@ -37,3 +37,5 @@ bootstrap-toggle.min.js bootstrap-toggle.min.css d3.v3.min.js ace.js +airflow/www_rbac/node_modules +.*json http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/5a7f0b2e/CONTRIBUTING.md ---------------------------------------------------------------------- diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index e45060b..3fe5204 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -242,3 +242,60 @@ $ alembic revision -m "add new field to db" Generating ~/airflow/airflow/migrations/versions/12341123_add_new_field_to_db.py ``` + +## Setting up the node / npm javascript environment (ONLY FOR www_rbac) + +`airflow/www_rbac/` contains all npm-managed, front end assets. +Flask-Appbuilder itself comes bundled with jQuery and bootstrap. +While these may be phased out over time, these packages are currently not +managed with npm. + +### Node/npm versions +Make sure you are using recent versions of node and npm. No problems have been found with node>=8.11.3 and npm>=6.1.3 + +### Using npm to generate bundled files + +#### npm +First, npm must be available in your environment. If it is not you can run the following commands +(taken from [this source](https://gist.github.com/DanHerbert/9520689)) +``` +brew install node --without-npm +echo prefix=~/.npm-packages >> ~/.npmrc +curl -L https://www.npmjs.com/install.sh | sh +``` + +The final step is to add `~/.npm-packages/bin` to your `PATH` so commands you install globally are usable. +Add something like this to your `.bashrc` file, then `source ~/.bashrc` to reflect the change. +``` +export PATH="$HOME/.npm-packages/bin:$PATH" +``` + +#### npm packages +To install third party libraries defined in `package.json`, run the +following within the `airflow/www_rbac/` directory which will install them in a +new `node_modules/` folder within `www_rbac/`. + +```bash +# from the root of the repository, move to where our JS package.json lives +cd airflow/www_rbac/ +# run npm install to fetch all the dependencies +npm install +``` + +To parse and generate bundled files for airflow, run either of the +following commands. The `dev` flag will keep the npm script running and +re-run it upon any changes within the assets directory. + +``` +# Compiles the production / optimized js & css +npm run prod + +# Start a web server that manages and updates your assets as you modify them +npm run dev +``` + +#### Upgrading npm packages + +Should you add or upgrade a npm package, which involves changing `package.json`, you'll need to re-run `npm install` +and push the newly generated `package-lock.json` file so we get the reproducible build. + http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/5a7f0b2e/MANIFEST.in ---------------------------------------------------------------------- diff --git a/MANIFEST.in b/MANIFEST.in index b069194..ec99c1f 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -6,9 +6,9 @@ # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at -# +# # http://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY @@ -24,6 +24,7 @@ include README.md graft licenses/ graft airflow/www/templates graft airflow/www/static +graft airflow/www_rbac graft airflow/www_rbac/static graft airflow/www_rbac/templates graft airflow/www_rbac/translations @@ -31,3 +32,4 @@ include airflow/alembic.ini graft scripts/systemd graft scripts/upstart graft airflow/config_templates +recursive-exclude airflow/www_rbac/node_modules * http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/5a7f0b2e/airflow/www_rbac/app.py ---------------------------------------------------------------------- diff --git a/airflow/www_rbac/app.py b/airflow/www_rbac/app.py index f419923..92e5c73 100644 --- a/airflow/www_rbac/app.py +++ b/airflow/www_rbac/app.py @@ -31,7 +31,7 @@ from werkzeug.contrib.fixers import ProxyFix from airflow import settings from airflow import configuration as conf from airflow.logging_config import configure_logging - +from airflow.www_rbac.static_config import configure_manifest_files app = None appbuilder = None @@ -64,6 +64,7 @@ def create_app(config=None, session=None, testing=False, app_name="Airflow"): app.register_blueprint(routes) configure_logging() + configure_manifest_files(app) with app.app_context(): http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/5a7f0b2e/airflow/www_rbac/compile_assets.sh ---------------------------------------------------------------------- diff --git a/airflow/www_rbac/compile_assets.sh b/airflow/www_rbac/compile_assets.sh new file mode 100755 index 0000000..2c115a4 --- /dev/null +++ b/airflow/www_rbac/compile_assets.sh @@ -0,0 +1,23 @@ +#!/bin/sh +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# first bump up package.json manually, commit and tag +rm airflow/www_rbac/static/dist/* +cd airflow/www_rbac/ +npm install +npm run build +cd ../..
