graceguo-supercat closed pull request #5543: Reduce dashboard position_json
data size
URL: https://github.com/apache/incubator-superset/pull/5543
This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:
As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):
diff --git
a/superset/assets/spec/javascripts/dashboard/util/findFirstParentContainer_spec.js
b/superset/assets/spec/javascripts/dashboard/util/findFirstParentContainer_spec.js
index 4ab29bda06..ecaca67664 100644
---
a/superset/assets/spec/javascripts/dashboard/util/findFirstParentContainer_spec.js
+++
b/superset/assets/spec/javascripts/dashboard/util/findFirstParentContainer_spec.js
@@ -10,94 +10,91 @@ import {
describe('findFirstParentContainer', () => {
const mockGridLayout = {
DASHBOARD_VERSION_KEY: 'v2',
- DASHBOARD_ROOT_ID: {
- type: 'DASHBOARD_ROOT_TYPE',
- id: 'DASHBOARD_ROOT_ID',
- children: ['DASHBOARD_GRID_ID'],
- },
- DASHBOARD_GRID_ID: {
- type: 'DASHBOARD_GRID_TYPE',
- id: 'DASHBOARD_GRID_ID',
- children: ['DASHBOARD_ROW_TYPE-Bk45URrlQ'],
- },
- 'DASHBOARD_ROW_TYPE-Bk45URrlQ': {
- type: 'DASHBOARD_ROW_TYPE',
- id: 'DASHBOARD_ROW_TYPE-Bk45URrlQ',
- children: ['DASHBOARD_CHART_TYPE-ryxVc8RHlX'],
- },
- 'DASHBOARD_CHART_TYPE-ryxVc8RHlX': {
- type: 'DASHBOARD_CHART_TYPE',
- id: 'DASHBOARD_CHART_TYPE-ryxVc8RHlX',
+ ROOT_ID: {
+ type: 'ROOT',
+ id: 'ROOT_ID',
+ children: ['GRID_ID'],
+ },
+ GRID_ID: {
+ type: 'GRID',
+ id: 'GRID_ID',
+ children: ['ROW-Bk45URrlQ'],
+ },
+ 'ROW-Bk45URrlQ': {
+ type: 'ROW',
+ id: 'ROW-Bk45URrlQ',
+ children: ['CHART-ryxVc8RHlX'],
+ },
+ 'CHART-ryxVc8RHlX': {
+ type: 'CHART',
+ id: 'CHART-ryxVc8RHlX',
children: [],
},
- DASHBOARD_HEADER_ID: {
- id: 'DASHBOARD_HEADER_ID',
- type: 'DASHBOARD_HEADER_TYPE',
+ HEADER_ID: {
+ id: 'HEADER_ID',
+ type: 'HEADER',
},
};
const mockTabsLayout = {
- 'DASHBOARD_CHART_TYPE-S1gilYABe7': {
+ 'CHART-S1gilYABe7': {
children: [],
- id: 'DASHBOARD_CHART_TYPE-S1gilYABe7',
- type: 'DASHBOARD_CHART_TYPE',
+ id: 'CHART-S1gilYABe7',
+ type: 'CHART',
},
- 'DASHBOARD_CHART_TYPE-SJli5K0HlQ': {
+ 'CHART-SJli5K0HlQ': {
children: [],
- id: 'DASHBOARD_CHART_TYPE-SJli5K0HlQ',
- type: 'DASHBOARD_CHART_TYPE',
+ id: 'CHART-SJli5K0HlQ',
+ type: 'CHART',
},
- DASHBOARD_GRID_ID: {
+ GRID_ID: {
children: [],
- id: 'DASHBOARD_GRID_ID',
- type: 'DASHBOARD_GRID_TYPE',
- },
- DASHBOARD_HEADER_ID: {
- id: 'DASHBOARD_HEADER_ID',
- type: 'DASHBOARD_HEADER_TYPE',
- },
- DASHBOARD_ROOT_ID: {
- children: ['DASHBOARD_TABS_TYPE-SkgJ5t0Bem'],
- id: 'DASHBOARD_ROOT_ID',
- type: 'DASHBOARD_ROOT_TYPE',
- },
- 'DASHBOARD_ROW_TYPE-S1B8-JLgX': {
- children: ['DASHBOARD_CHART_TYPE-SJli5K0HlQ'],
- id: 'DASHBOARD_ROW_TYPE-S1B8-JLgX',
- type: 'DASHBOARD_ROW_TYPE',
- },
- 'DASHBOARD_ROW_TYPE-S1bUb1Ilm': {
- children: ['DASHBOARD_CHART_TYPE-S1gilYABe7'],
- id: 'DASHBOARD_ROW_TYPE-S1bUb1Ilm',
- type: 'DASHBOARD_ROW_TYPE',
- },
- 'DASHBOARD_TABS_TYPE-ByeLSWyLe7': {
- children: ['DASHBOARD_TAB_TYPE-BJbLSZ1UeQ'],
- id: 'DASHBOARD_TABS_TYPE-ByeLSWyLe7',
- type: 'DASHBOARD_TABS_TYPE',
- },
- 'DASHBOARD_TABS_TYPE-SkgJ5t0Bem': {
- children: [
- 'DASHBOARD_TAB_TYPE-HkWJcFCHxQ',
- 'DASHBOARD_TAB_TYPE-ByDBbkLlQ',
- ],
- id: 'DASHBOARD_TABS_TYPE-SkgJ5t0Bem',
+ id: 'GRID_ID',
+ type: 'GRID',
+ },
+ HEADER_ID: {
+ id: 'HEADER_ID',
+ type: 'HEADER',
+ },
+ ROOT_ID: {
+ children: ['TABS-SkgJ5t0Bem'],
+ id: 'ROOT_ID',
+ type: 'ROOT',
+ },
+ 'ROW-S1B8-JLgX': {
+ children: ['CHART-SJli5K0HlQ'],
+ id: 'ROW-S1B8-JLgX',
+ type: 'ROW',
+ },
+ 'ROW-S1bUb1Ilm': {
+ children: ['CHART-S1gilYABe7'],
+ id: 'ROW-S1bUb1Ilm',
+ type: 'ROW',
+ },
+ 'TABS-ByeLSWyLe7': {
+ children: ['TAB-BJbLSZ1UeQ'],
+ id: 'TABS-ByeLSWyLe7',
+ type: 'TABS',
+ },
+ 'TABS-SkgJ5t0Bem': {
+ children: ['TAB-HkWJcFCHxQ', 'TAB-ByDBbkLlQ'],
+ id: 'TABS-SkgJ5t0Bem',
meta: {},
- type: 'DASHBOARD_TABS_TYPE',
- },
- 'DASHBOARD_TAB_TYPE-BJbLSZ1UeQ': {
- children: ['DASHBOARD_ROW_TYPE-S1bUb1Ilm'],
- id: 'DASHBOARD_TAB_TYPE-BJbLSZ1UeQ',
- type: 'DASHBOARD_TAB_TYPE',
- },
- 'DASHBOARD_TAB_TYPE-ByDBbkLlQ': {
- children: ['DASHBOARD_ROW_TYPE-S1B8-JLgX'],
- id: 'DASHBOARD_TAB_TYPE-ByDBbkLlQ',
- type: 'DASHBOARD_TAB_TYPE',
- },
- 'DASHBOARD_TAB_TYPE-HkWJcFCHxQ': {
- children: ['DASHBOARD_TABS_TYPE-ByeLSWyLe7'],
- id: 'DASHBOARD_TAB_TYPE-HkWJcFCHxQ',
- type: 'DASHBOARD_TAB_TYPE',
+ type: 'TABS',
+ },
+ 'TAB-BJbLSZ1UeQ': {
+ children: ['ROW-S1bUb1Ilm'],
+ id: 'TAB-BJbLSZ1UeQ',
+ type: 'TAB',
+ },
+ 'TAB-ByDBbkLlQ': {
+ children: ['ROW-S1B8-JLgX'],
+ id: 'TAB-ByDBbkLlQ',
+ type: 'TAB',
+ },
+ 'TAB-HkWJcFCHxQ': {
+ children: ['TABS-ByeLSWyLe7'],
+ id: 'TAB-HkWJcFCHxQ',
+ type: 'TAB',
},
DASHBOARD_VERSION_KEY: 'v2',
};
diff --git a/superset/assets/src/dashboard/components/Header.jsx
b/superset/assets/src/dashboard/components/Header.jsx
index 0c1951b8d7..aa29bea131 100644
--- a/superset/assets/src/dashboard/components/Header.jsx
+++ b/superset/assets/src/dashboard/components/Header.jsx
@@ -10,11 +10,16 @@ import UndoRedoKeylisteners from './UndoRedoKeylisteners';
import { chartPropShape } from '../util/propShapes';
import { t } from '../../locales';
-import { UNDO_LIMIT, SAVE_TYPE_OVERWRITE } from '../util/constants';
+import {
+ UNDO_LIMIT,
+ SAVE_TYPE_OVERWRITE,
+ DASHBOARD_POSITION_DATA_LIMIT,
+} from '../util/constants';
const propTypes = {
addSuccessToast: PropTypes.func.isRequired,
addDangerToast: PropTypes.func.isRequired,
+ addWarningToast: PropTypes.func.isRequired,
dashboardInfo: PropTypes.object.isRequired,
dashboardTitle: PropTypes.string.isRequired,
charts: PropTypes.objectOf(chartPropShape).isRequired,
@@ -143,7 +148,24 @@ class Header extends React.PureComponent {
default_filters: JSON.stringify(filters),
};
- this.props.onSave(data, dashboardInfo.id, SAVE_TYPE_OVERWRITE);
+ // make sure positions data less than DB storage limitation:
+ const positionJSONLength = JSON.stringify(positions).length;
+ const limit =
+ dashboardInfo.common.conf.SUPERSET_DASHBOARD_POSITION_DATA_LIMIT ||
+ DASHBOARD_POSITION_DATA_LIMIT;
+ if (positionJSONLength >= limit) {
+ this.props.addDangerToast(
+ t(
+ 'Your dashboard is too large. Please reduce the size before save
it.',
+ ),
+ );
+ } else {
+ if (positionJSONLength >= limit * 0.9) {
+ this.props.addWarningToast('Your dashboard is near the size limit.');
+ }
+
+ this.props.onSave(data, dashboardInfo.id, SAVE_TYPE_OVERWRITE);
+ }
}
render() {
diff --git a/superset/assets/src/dashboard/containers/DashboardHeader.jsx
b/superset/assets/src/dashboard/containers/DashboardHeader.jsx
index 629b916456..3e54c62469 100644
--- a/superset/assets/src/dashboard/containers/DashboardHeader.jsx
+++ b/superset/assets/src/dashboard/containers/DashboardHeader.jsx
@@ -23,7 +23,11 @@ import {
updateDashboardTitle,
} from '../actions/dashboardLayout';
-import { addSuccessToast, addDangerToast } from '../../messageToasts/actions';
+import {
+ addSuccessToast,
+ addDangerToast,
+ addWarningToast,
+} from '../../messageToasts/actions';
import { DASHBOARD_HEADER_ID } from '../util/constants';
@@ -59,6 +63,7 @@ function mapDispatchToProps(dispatch) {
{
addSuccessToast,
addDangerToast,
+ addWarningToast,
onUndo: undoLayoutAction,
onRedo: redoLayoutAction,
setEditMode,
diff --git a/superset/assets/src/dashboard/util/componentTypes.js
b/superset/assets/src/dashboard/util/componentTypes.js
index b773417983..47478e6119 100644
--- a/superset/assets/src/dashboard/util/componentTypes.js
+++ b/superset/assets/src/dashboard/util/componentTypes.js
@@ -1,15 +1,15 @@
-export const CHART_TYPE = 'DASHBOARD_CHART_TYPE';
-export const COLUMN_TYPE = 'DASHBOARD_COLUMN_TYPE';
-export const DASHBOARD_HEADER_TYPE = 'DASHBOARD_HEADER_TYPE';
-export const DASHBOARD_GRID_TYPE = 'DASHBOARD_GRID_TYPE';
-export const DASHBOARD_ROOT_TYPE = 'DASHBOARD_ROOT_TYPE';
-export const DIVIDER_TYPE = 'DASHBOARD_DIVIDER_TYPE';
-export const HEADER_TYPE = 'DASHBOARD_HEADER_TYPE';
-export const MARKDOWN_TYPE = 'DASHBOARD_MARKDOWN_TYPE';
-export const NEW_COMPONENT_SOURCE_TYPE = 'NEW_COMPONENT_SOURCE_TYPE';
-export const ROW_TYPE = 'DASHBOARD_ROW_TYPE';
-export const TABS_TYPE = 'DASHBOARD_TABS_TYPE';
-export const TAB_TYPE = 'DASHBOARD_TAB_TYPE';
+export const CHART_TYPE = 'CHART';
+export const COLUMN_TYPE = 'COLUMN';
+export const DASHBOARD_HEADER_TYPE = 'HEADER';
+export const DASHBOARD_GRID_TYPE = 'GRID';
+export const DASHBOARD_ROOT_TYPE = 'ROOT';
+export const DIVIDER_TYPE = 'DIVIDER';
+export const HEADER_TYPE = 'HEADER';
+export const MARKDOWN_TYPE = 'MARKDOWN';
+export const NEW_COMPONENT_SOURCE_TYPE = 'NEW_COMPONENT_SOURCE';
+export const ROW_TYPE = 'ROW';
+export const TABS_TYPE = 'TABS';
+export const TAB_TYPE = 'TAB';
export default {
CHART_TYPE,
diff --git a/superset/assets/src/dashboard/util/constants.js
b/superset/assets/src/dashboard/util/constants.js
index 4fd5e400ec..b26cbff85f 100644
--- a/superset/assets/src/dashboard/util/constants.js
+++ b/superset/assets/src/dashboard/util/constants.js
@@ -1,7 +1,7 @@
// Ids
-export const DASHBOARD_GRID_ID = 'DASHBOARD_GRID_ID';
-export const DASHBOARD_HEADER_ID = 'DASHBOARD_HEADER_ID';
-export const DASHBOARD_ROOT_ID = 'DASHBOARD_ROOT_ID';
+export const DASHBOARD_GRID_ID = 'GRID_ID';
+export const DASHBOARD_HEADER_ID = 'HEADER_ID';
+export const DASHBOARD_ROOT_ID = 'ROOT_ID';
export const DASHBOARD_VERSION_KEY = 'DASHBOARD_VERSION_KEY';
export const NEW_COMPONENTS_SOURCE_ID = 'NEW_COMPONENTS_SOURCE_ID';
@@ -40,3 +40,7 @@ export const UNDO_LIMIT = 50;
// save dash options
export const SAVE_TYPE_OVERWRITE = 'overwrite';
export const SAVE_TYPE_NEWDASHBOARD = 'newDashboard';
+
+// default dashboard layout data size limit
+// could be overwritten by server-side config
+export const DASHBOARD_POSITION_DATA_LIMIT = 65535;
diff --git a/superset/config.py b/superset/config.py
index 6c3c526ef9..b737580d67 100644
--- a/superset/config.py
+++ b/superset/config.py
@@ -49,6 +49,7 @@
SUPERSET_WEBSERVER_ADDRESS = '0.0.0.0'
SUPERSET_WEBSERVER_PORT = 8088
SUPERSET_WEBSERVER_TIMEOUT = 60 # deprecated
+SUPERSET_DASHBOARD_POSITION_DATA_LIMIT = 65535
EMAIL_NOTIFICATIONS = False
CUSTOM_SECURITY_MANAGER = None
SQLALCHEMY_TRACK_MODIFICATIONS = False
diff --git a/superset/migrations/versions/7fcdcde0761c_.py
b/superset/migrations/versions/7fcdcde0761c_.py
new file mode 100644
index 0000000000..793e0fb68e
--- /dev/null
+++ b/superset/migrations/versions/7fcdcde0761c_.py
@@ -0,0 +1,69 @@
+"""Reduce position_json size by remove extra space and component id prefix
+
+Revision ID: 7fcdcde0761c
+Revises: c18bd4186f15
+Create Date: 2018-08-01 11:47:02.233971
+
+"""
+
+# revision identifiers, used by Alembic.
+import json
+import re
+
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy import (
+ Table, Column,
+ Integer, String, Text, ForeignKey,
+)
+from sqlalchemy.ext.declarative import declarative_base
+from sqlalchemy.orm import relationship
+
+from superset import db
+
+revision = '7fcdcde0761c'
+down_revision = 'c18bd4186f15'
+
+Base = declarative_base()
+
+
+class Dashboard(Base):
+ """Declarative class to do query in upgrade"""
+ __tablename__ = 'dashboards'
+ id = sa.Column(sa.Integer, primary_key=True)
+ dashboard_title = sa.Column(String(500))
+ position_json = sa.Column(sa.Text)
+
+
+def is_v2_dash(positions):
+ return (
+ isinstance(positions, dict) and
+ positions.get('DASHBOARD_VERSION_KEY') == 'v2'
+ )
+
+
+def upgrade():
+ bind = op.get_bind()
+ session = db.Session(bind=bind)
+
+ dashboards = session.query(Dashboard).all()
+ for i, dashboard in enumerate(dashboards):
+ original_text = dashboard.position_json or ''
+ position_json = json.loads(original_text or '{}')
+ if is_v2_dash(position_json):
+ # re-dump the json data and remove leading and trailing white
spaces
+ text = json.dumps(
+ position_json, indent=None, separators=(',', ':'),
sort_keys=True)
+ # remove DASHBOARD_ and _TYPE prefix/suffix in all the component
ids
+ text = re.sub(r'DASHBOARD_(?!VERSION)', '', text)
+ text = text.replace('_TYPE', '')
+
+ dashboard.position_json = text
+ print('dash id:{} position_json size from {} to {}'
+ .format(dashboard.id, len(original_text), len(text)))
+ session.merge(dashboard)
+ session.commit()
+
+
+def downgrade():
+ pass
diff --git a/superset/views/base.py b/superset/views/base.py
index 25de44d26a..5eed6f6626 100644
--- a/superset/views/base.py
+++ b/superset/views/base.py
@@ -26,6 +26,7 @@
FRONTEND_CONF_KEYS = (
'SUPERSET_WEBSERVER_TIMEOUT',
+ 'SUPERSET_DASHBOARD_POSITION_DATA_LIMIT',
'ENABLE_JAVASCRIPT_CONTROLS',
)
diff --git a/superset/views/core.py b/superset/views/core.py
index 17eb34ba87..2d618c89a2 100755
--- a/superset/views/core.py
+++ b/superset/views/core.py
@@ -1642,6 +1642,9 @@ def _set_dash_metadata(dashboard, data):
session.merge(slc)
session.flush()
+ # remove leading and trailing white spaces in the dumped json
+ dashboard.position_json = json.dumps(
+ positions, indent=None, separators=(',', ':'), sort_keys=True)
dashboard.position_json = json.dumps(positions, sort_keys=True)
md = dashboard.params_dict
dashboard.css = data.get('css')
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
