This is an automated email from the ASF dual-hosted git repository.
sushuang pushed a commit to branch dataset-trans
in repository https://gitbox.apache.org/repos/asf/incubator-echarts.git
commit f8a59dfa6b2c9f651582c053e851d5286afc7a63
Author: 100pah <sushuang0...@gmail.com>
AuthorDate: Fri Jul 31 16:20:01 2020 +0800
fix: fix dimension inherit rule.
---
src/component/transform/filterTransform.ts | 6 +-
src/component/transform/sortTransform.ts | 6 +-
src/data/Source.ts | 22 ++++-
src/data/helper/sourceHelper.ts | 22 +++--
src/data/helper/sourceManager.ts | 45 ++++++---
src/data/helper/transform.ts | 74 ++++++++++-----
test/data-transform.html | 145 ++++++++++++++++++++++++++++-
7 files changed, 264 insertions(+), 56 deletions(-)
diff --git a/src/component/transform/filterTransform.ts
b/src/component/transform/filterTransform.ts
index 4b13008..4126f98 100644
--- a/src/component/transform/filterTransform.ts
+++ b/src/component/transform/filterTransform.ts
@@ -65,7 +65,7 @@ export const filterTransform:
ExternalDataTransform<FilterTransformOption> = {
if (__DEV__) {
errMsg = makePrintable(
'Can not find dimension info via: "' + dimLoose +
'".\n',
- 'Existing dimensions: ', source.dimensions, '.\n',
+ 'Existing dimensions: ',
source.getDimensionInfoAll(), '.\n',
'Illegal condition:', exprOption, '.\n'
);
}
@@ -93,9 +93,7 @@ export const filterTransform:
ExternalDataTransform<FilterTransformOption> = {
}
return {
- data: resultData,
- dimensions: source.dimensions,
- sourceHeader: sourceHeaderCount
+ data: resultData
};
}
};
diff --git a/src/component/transform/sortTransform.ts
b/src/component/transform/sortTransform.ts
index 9dfce97..e4f75e5 100644
--- a/src/component/transform/sortTransform.ts
+++ b/src/component/transform/sortTransform.ts
@@ -124,7 +124,7 @@ export const sortTransform:
ExternalDataTransform<SortTransformOption> = {
if (__DEV__) {
errMsg = makePrintable(
'Can not find dimension info via: "' + dimLoose +
'".\n',
- 'Existing dimensions: ', source.dimensions, '.\n',
+ 'Existing dimensions: ', source.getDimensionInfoAll(),
'.\n',
'Illegal config:', orderExpr, '.\n'
);
}
@@ -214,9 +214,7 @@ export const sortTransform:
ExternalDataTransform<SortTransformOption> = {
}
return {
- data: resultData,
- dimensions: source.dimensions,
- sourceHeader: sourceHeaderCount
+ data: resultData
};
}
};
diff --git a/src/data/Source.ts b/src/data/Source.ts
index bed37d6..06d5a27 100644
--- a/src/data/Source.ts
+++ b/src/data/Source.ts
@@ -17,16 +17,18 @@
* under the License.
*/
-import {createHashMap, isTypedArray, HashMap} from 'zrender/src/core/util';
+import {isTypedArray, HashMap} from 'zrender/src/core/util';
import {
SourceFormat, SeriesLayoutBy, DimensionDefinition,
- OptionEncodeValue, OptionSourceData, OptionEncode,
+ OptionEncodeValue, OptionSourceData,
SOURCE_FORMAT_ORIGINAL,
SERIES_LAYOUT_BY_COLUMN,
SOURCE_FORMAT_UNKNOWN,
SOURCE_FORMAT_KEYED_COLUMNS,
SOURCE_FORMAT_TYPED_ARRAY,
- DimensionName
+ DimensionName,
+ OptionSourceHeader,
+ DimensionDefinitionLoose
} from '../util/types';
/**
@@ -65,6 +67,12 @@ import {
* + "unknown"
*/
+export interface SourceMetaRawOption {
+ seriesLayoutBy: SeriesLayoutBy;
+ sourceHeader: OptionSourceHeader;
+ dimensions: DimensionDefinitionLoose[];
+}
+
class Source {
/**
@@ -107,6 +115,11 @@ class Source {
*/
readonly dimensionsDetectCount: number;
+ /**
+ * Raw props from user option.
+ */
+ readonly metaRawOption: SourceMetaRawOption;
+
constructor(fields: {
data: OptionSourceData,
@@ -118,6 +131,8 @@ class Source {
startIndex?: number, // default: 0
dimensionsDetectCount?: number,
+ metaRawOption?: SourceMetaRawOption,
+
// [Caveat]
// This is the raw user defined `encode` in `series`.
// If user not defined, DO NOT make a empty object or hashMap here.
@@ -136,6 +151,7 @@ class Source {
this.dimensionsDefine = fields.dimensionsDefine;
this.dimensionsDetectCount = fields.dimensionsDetectCount;
this.encodeDefine = fields.encodeDefine;
+ this.metaRawOption = fields.metaRawOption;
}
/**
diff --git a/src/data/helper/sourceHelper.ts b/src/data/helper/sourceHelper.ts
index 9ae0151..0905908 100644
--- a/src/data/helper/sourceHelper.ts
+++ b/src/data/helper/sourceHelper.ts
@@ -33,9 +33,10 @@ import {
hasOwn,
HashMap,
isNumber,
- clone
+ clone,
+ defaults
} from 'zrender/src/core/util';
-import Source from '../Source';
+import Source, { SourceMetaRawOption } from '../Source';
import {
SOURCE_FORMAT_ORIGINAL,
@@ -90,12 +91,6 @@ type SeriesEncodeInternal = {
[key in keyof OptionEncode]: DimensionIndex[];
};
-export interface SourceMetaRawOption {
- seriesLayoutBy: SeriesLayoutBy;
- sourceHeader: OptionSourceHeader;
- dimensions: DimensionDefinitionLoose[];
-}
-
export function detectSourceFormat(data: DatasetOption['source']):
SourceFormat {
let sourceFormat: SourceFormat = SOURCE_FORMAT_UNKNOWN;
@@ -170,12 +165,21 @@ export function createSource(
dimensionsDefine: dimInfo.dimensionsDefine,
startIndex: dimInfo.startIndex,
dimensionsDetectCount: dimInfo.dimensionsDetectCount,
- encodeDefine: makeEncodeDefine(encodeDefine)
+ encodeDefine: makeEncodeDefine(encodeDefine),
+ metaRawOption: clone(thisMetaRawOption)
});
return source;
}
+// See [DIMENSION_INHERIT_RULE] in `sourceManager.ts`.
+export function inheritSourceMetaRawOption(opt: {
+ parent: SourceMetaRawOption, // Can be null/undefined
+ thisNew: SourceMetaRawOption // Must be object
+}) {
+ return defaults(opt.thisNew, opt.parent);
+}
+
/**
* Clone except source data.
*/
diff --git a/src/data/helper/sourceManager.ts b/src/data/helper/sourceManager.ts
index 74ed264..5a3f9ea 100644
--- a/src/data/helper/sourceManager.ts
+++ b/src/data/helper/sourceManager.ts
@@ -28,7 +28,7 @@ import {
} from '../../util/types';
import {
querySeriesUpstreamDatasetModel, queryDatasetUpstreamDatasetModels,
- createSource, SourceMetaRawOption, cloneSourceShallow
+ createSource, SourceMetaRawOption, cloneSourceShallow,
inheritSourceMetaRawOption
} from './sourceHelper';
import { applyDataTransform } from './transform';
@@ -53,6 +53,27 @@ import { applyDataTransform } from './transform';
* They will not be calculated until `source` is about to be visited (to
prevent from
* duplicate calcuation). `source` is visited only in series and input to
transforms.
*
+ * [DIMENSION_INHERIT_RULE]:
+ * By default the dimensions are inherited from ancestors, unless a transform
return
+ * a new dimensions definition.
+ * Consider the case:
+ * ```js
+ * dataset: [{
+ * source: [ ['Product', 'Sales', 'Prise'], ['Cookies', 321, 44.21], ...]
+ * }, {
+ * transform: { type: 'filter', ... }
+ * }]
+ *
+ * dataset: [{
+ * dimension: ['Product', 'Sales', 'Prise'],
+ * source: [ ['Cookies', 321, 44.21], ...]
+ * }, {
+ * transform: { type: 'filter', ... }
+ * }]
+ * ```
+ * The two types of option should have the same behavior after transform.
+ *
+ *
* [SCENARIO]:
* (1) Provide source data directly:
* ```js
@@ -172,16 +193,15 @@ export class SourceManager {
const seriesModel = sourceHost as SeriesEncodableModel;
let data;
let sourceFormat: SourceFormat;
- let upMetaRawOption;
+ let upSource;
// Has upstream dataset
if (hasUpstream) {
const upSourceMgr = upSourceMgrList[0];
upSourceMgr.prepareSource();
- const upSource = upSourceMgr.getSource();
+ upSource = upSourceMgr.getSource();
data = upSource.data;
sourceFormat = upSource.sourceFormat;
- upMetaRawOption = upSourceMgr._getSourceMetaRawOption();
upstreamSignList = [upSourceMgr._getVersionSign()];
}
// Series data is from own.
@@ -192,11 +212,12 @@ export class SourceManager {
upstreamSignList = [];
}
- const thisMetaRawOption = defaults(
- this._getSourceMetaRawOption(),
- // See [REQUIREMENT MEMO], merge settings on series and parent
dataset if it is root.
- upMetaRawOption
- );
+ // See [REQUIREMENT_MEMO], merge settings on series and parent
dataset if it is root.
+ const thisMetaRawOption = inheritSourceMetaRawOption({
+ parent: upSource ? upSource.metaRawOption : null,
+ thisNew: this._createSourceMetaRawOption()
+ });
+
resultSourceList = [createSource(
data,
thisMetaRawOption,
@@ -218,7 +239,7 @@ export class SourceManager {
const sourceData = datasetModel.get('source', true);
resultSourceList = [createSource(
sourceData,
- this._getSourceMetaRawOption(),
+ this._createSourceMetaRawOption(),
null,
// Note: dataset option does not have `encode`.
null
@@ -329,7 +350,7 @@ export class SourceManager {
}
}
- private _getSourceMetaRawOption(): SourceMetaRawOption {
+ private _createSourceMetaRawOption(): SourceMetaRawOption {
const sourceHost = this._sourceHost;
let seriesLayoutBy: SeriesLayoutBy;
let sourceHeader: OptionSourceHeader;
@@ -339,7 +360,7 @@ export class SourceManager {
sourceHeader = sourceHost.get('sourceHeader', true);
dimensions = sourceHost.get('dimensions', true);
}
- // See [REQUIREMENT MEMO], `non-root-dataset` do not support them.
+ // See [REQUIREMENT_MEMO], `non-root-dataset` do not support them.
else if (!this._getUpstreamSourceManagers().length) {
const model = sourceHost as DatasetModel;
seriesLayoutBy = model.get('seriesLayoutBy', true);
diff --git a/src/data/helper/transform.ts b/src/data/helper/transform.ts
index e6150b1..d657d52 100644
--- a/src/data/helper/transform.ts
+++ b/src/data/helper/transform.ts
@@ -32,7 +32,7 @@ import {
getRawSourceItemGetter, getRawSourceDataCounter, getRawSourceValueGetter
} from './dataProvider';
import { parseDataValue } from './parseDataValue';
-import { createSource } from './sourceHelper';
+import { createSource, inheritSourceMetaRawOption } from './sourceHelper';
import { consoleLog, makePrintable } from '../../util/log';
@@ -76,7 +76,7 @@ export interface ExternalDataTransformResultItem {
dimensions?: DimensionDefinitionLoose[];
sourceHeader?: OptionSourceHeader;
}
-export interface ExternalDimensionDefinition extends DimensionDefinition {
+interface ExternalDimensionDefinition extends DimensionDefinition {
// Mandatory
index: DimensionIndex;
}
@@ -95,13 +95,16 @@ class ExternalSource {
data: OptionSourceData;
sourceFormat: SourceFormat;
- dimensions: ExternalDimensionDefinition[];
sourceHeaderCount: number;
getDimensionInfo(dim: DimensionLoose): ExternalDimensionDefinition {
return;
}
+ getDimensionInfoAll(): ExternalDimensionDefinition[] {
+ return;
+ }
+
getRawDataItem(dataIndex: number): OptionDataItem {
return;
}
@@ -140,8 +143,13 @@ function createExternalSource(
extSource.sourceFormat = sourceFormat;
extSource.sourceHeaderCount = sourceHeaderCount;
+ // [MEMO]
// Create a new dimensions structure for exposing.
- const dimensions = extSource.dimensions = [] as
ExternalDimensionDefinition[];
+ // Do not expose all dimension info to users directly.
+ // Becuase the dimension is probably auto detected from data and not might
reliable.
+ // Should not lead the transformers to think that is relialbe and return
it.
+ // See [DIMENSION_INHERIT_RULE] in `sourceManager.ts`.
+ const dimensions = [] as ExternalDimensionDefinition[];
const dimsByName = {} as Dictionary<ExternalDimensionDefinition>;
each(dimsDef, function (dimDef, idx) {
const name = dimDef.name;
@@ -179,7 +187,7 @@ function createExternalSource(
if (rawItem == null) {
return;
}
- const dimDef = extSource.dimensions[dimIndex];
+ const dimDef = dimensions[dimIndex];
// When `dimIndex` is `null`, `rawValueGetter` return the whole item.
if (dimDef) {
return rawValueGetter(rawItem, dimIndex, dimDef.name) as
OptionDataValue;
@@ -187,6 +195,7 @@ function createExternalSource(
};
extSource.getDimensionInfo = bind(getDimensionInfo, null, dimensions,
dimsByName);
+ extSource.getDimensionInfoAll = bind(getDimensionInfoAll, null,
dimensions);
return extSource;
}
@@ -212,6 +221,12 @@ function getDimensionInfo(
}
}
+function getDimensionInfoAll(
+ dimensions: ExternalDimensionDefinition[]
+): ExternalDimensionDefinition[] {
+ return dimensions;
+}
+
const externalTransformMap = createHashMap<ExternalDataTransform, string>();
@@ -241,22 +256,13 @@ export function applyDataTransform(
for (let i = 0, len = pipedTransOption.length; i < len; i++) {
const transOption = pipedTransOption[i];
- sourceList = applySingleDataTransform(transOption, sourceList);
+ const isFinal = i === len - 1;
+ sourceList = applySingleDataTransform(transOption, sourceList,
infoForPrint, isFinal);
// piped transform only support single input, except the fist one.
// piped transform only support single output, except the last one.
- if (i < len - 1) {
+ if (!isFinal) {
sourceList.length = Math.max(sourceList.length, 1);
}
-
- if (__DEV__) {
- if (transOption.print) {
- const printStrArr = map(sourceList, source => {
- return '--- datasetIndex: ' + infoForPrint.datasetIndex +
', transform result: ---\n'
- + makePrintable(source.data);
- }).join('\n');
- consoleLog(printStrArr);
- }
- }
}
return sourceList;
@@ -264,7 +270,9 @@ export function applyDataTransform(
function applySingleDataTransform(
rawTransOption: DataTransformOption,
- upSourceList: Source[]
+ upSourceList: Source[],
+ infoForPrint: { datasetIndex: number },
+ isFinal: boolean
): Source[] {
assert(upSourceList.length, 'Must have at least one upstream dataset.');
@@ -292,6 +300,23 @@ function applySingleDataTransform(
})
);
+ if (__DEV__) {
+ if (isFinal && transOption.print) {
+ const printStrArr = map(resultList, extSource => {
+ return [
+ '--- datasetIndex: ' + infoForPrint.datasetIndex + '---',
+ '- transform result data:',
+ makePrintable(extSource.data),
+ '- transform result dimensions:',
+ makePrintable(extSource.dimensions),
+ '- transform result sourceHeader: ' +
extSource.sourceHeader,
+ '------'
+ ].join('\n');
+ }).join('\n');
+ consoleLog(printStrArr);
+ }
+ }
+
return map(resultList, function (result) {
assert(
isObject(result),
@@ -302,13 +327,18 @@ function applySingleDataTransform(
'Result data should be object or array in data transform.'
);
- return createSource(
- result.data,
- {
+ const resultMetaRawOption = inheritSourceMetaRawOption({
+ parent: upSourceList[0].metaRawOption,
+ thisNew: {
seriesLayoutBy: SERIES_LAYOUT_BY_COLUMN,
sourceHeader: result.sourceHeader,
dimensions: result.dimensions
- },
+ }
+ });
+
+ return createSource(
+ result.data,
+ resultMetaRawOption,
null,
null
);
diff --git a/test/data-transform.html b/test/data-transform.html
index 05ada21..9a11dc7 100644
--- a/test/data-transform.html
+++ b/test/data-transform.html
@@ -37,10 +37,12 @@ under the License.
- <!-- <div id="main_simplest_pies"></div>
+ <div id="main_simplest_pies"></div>
<div id="main_pies_encode_price"></div>
- <div id="main_cartesian_parse_trim_time_reg"></div> -->
+ <div id="main_cartesian_parse_trim_time_reg"></div>
<div id="main_cartesian_sort"></div>
+ <div id="main_update_condition"></div>
+ <div id="main_update_source_no_dim_inside_data"></div>
@@ -656,6 +658,145 @@ under the License.
+
+
+ <script>
+ require(['echarts'], function (echarts) {
+ var currentYear = 2011
+ var option = {
+ title: {
+ text: currentYear,
+ left: 'center'
+ },
+ dataset: [{
+ source: FOOD_SALES_PRICE_WITH_HEADER
+ }, {
+ id: 'one_year',
+ transform: {
+ type: 'filter',
+ config: { dimension: 'Year', value: currentYear }
+ }
+ }],
+ tooltip: {},
+ xAxis: { type: 'category' },
+ yAxis: {},
+ series: [{
+ name: 'one year',
+ type: 'bar',
+ datasetIndex: 1,
+ }]
+ };
+
+ var chart = testHelper.create(echarts, 'main_update_condition', {
+ title: [
+ 'click "next year", check the bar change.'
+ ],
+ height: 300,
+ option: option,
+ buttons: [{
+ text: 'next year',
+ onclick: function () {
+ currentYear++;
+ if (currentYear >= 2014) {
+ currentYear = 2011
+ }
+ chart.setOption({
+ title: {
+ text: currentYear
+ },
+ dataset: {
+ id: 'one_year',
+ transform: {
+ type: 'filter',
+ config: { dimension: 'Year', value:
currentYear }
+ }
+ }
+ });
+ }
+ }]
+ });
+ });
+ </script>
+
+
+
+
+
+ <script>
+ require(['echarts'], function (echarts) {
+ var currData = FOOD_SALES_PRICE_NO_HEADER;
+
+ var option = {
+ dataset: [{
+ id: 'all_data',
+ dimensions: FOOD_SALES_PRICE_HEADER,
+ source: currData
+ }, {
+ transform: {
+ type: 'filter',
+ print: true,
+ config: { dimension: 'Price', '<': 40 }
+ }
+ }],
+ tooltip: {},
+ legend: {},
+ xAxis: { type: 'category' },
+ yAxis: { scale: true },
+ series: [{
+ name: 'all data',
+ type: 'scatter',
+ symbolSize: 15,
+ encode: {
+ itemId: 'Product',
+ y: 'Price',
+ label: [0, 1, 2, 3]
+ },
+ itemStyle: {
+ color: '#999'
+ }
+ }, {
+ name: 'Price < 40',
+ type: 'scatter',
+ encode: {
+ itemId: 'Product',
+ x: 0,
+ y: 'Price',
+ label: [0, 1, 2, 3]
+ },
+ datasetIndex: 1
+ }]
+ };
+
+ var chart = testHelper.create(echarts,
'main_update_source_no_dim_inside_data', {
+ title: [
+ 'Init: all items that Price < 40',
+ 'click "add price 10", check the bar change.'
+ ],
+ height: 300,
+ option: option,
+ buttons: [{
+ text: 'add price 10',
+ onclick: function () {
+ currData = echarts.util.clone(currData);
+ echarts.util.each(currData, function (line) {
+ line[2] += 10;
+ });
+
+ chart.setOption({
+ dataset: {
+ id: 'all_data',
+ source: currData
+ }
+ });
+ }
+ }]
+ });
+ });
+ </script>
+
+
+
+
</body>
</html>
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@echarts.apache.org
For additional commands, e-mail: commits-h...@echarts.apache.org
