feature: data transform

src/chart/helper/createListFromArray.ts

@@ -28,13 +28,13 @@ import Source from '../../data/Source';
 import {enableDataStack} from '../../data/helper/dataStackHelper';
 import {makeSeriesEncodeForAxisCoordSys} from '../../data/helper/sourceHelper';
 import {
-    SOURCE_FORMAT_ORIGINAL, DimensionDefinitionLoose, DimensionDefinition, OptionSourceData
+    SOURCE_FORMAT_ORIGINAL, DimensionDefinitionLoose, DimensionDefinition, OptionSourceData, EncodeDefaulter
 } from '../../util/types';
 import SeriesModel from '../../model/Series';
 
 function createListFromArray(source: Source | OptionSourceData, seriesModel: SeriesModel, opt?: {
     generateCoord?: string
-    useEncodeDefaulter?: boolean
+    useEncodeDefaulter?: boolean | EncodeDefaulter
 }): List {
     opt = opt || {};
 
@@ -73,10 +73,13 @@ function createListFromArray(source: Source | OptionSourceData, seriesModel: Ser
         )) || ['x', 'y'];
     }
 
+    const useEncodeDefaulter = opt.useEncodeDefaulter;
     const dimInfoList = createDimensions(source, {
         coordDimensions: coordSysDimDefs,
         generateCoord: opt.generateCoord,
-        encodeDefaulter: opt.useEncodeDefaulter
+        encodeDefaulter: zrUtil.isFunction(useEncodeDefaulter)
+            ? useEncodeDefaulter
+            : useEncodeDefaulter
             ? zrUtil.curry(makeSeriesEncodeForAxisCoordSys, coordSysDimDefs, seriesModel)
             : null
     });

src/chart/parallel/ParallelSeries.ts

@@ -18,7 +18,7 @@
 */
 
 
-import {each, createHashMap} from 'zrender/src/core/util';
+import {each, bind} from 'zrender/src/core/util';
 import SeriesModel from '../../model/Series';
 import createListFromArray from '../helper/createListFromArray';
 import {
@@ -29,13 +29,15 @@ import {
     SeriesTooltipOption,
     DimensionName,
     OptionDataValue,
-    StatesOptionMixin
+    StatesOptionMixin,
+    OptionEncodeValue,
+    Dictionary,
+    OptionEncode
  } from '../../util/types';
 import GlobalModel from '../../model/Global';
 import List from '../../data/List';
 import { ParallelActiveState, ParallelAxisOption } from '../../coord/parallel/AxisModel';
 import Parallel from '../../coord/parallel/Parallel';
-import Source from '../../data/Source';
 import ParallelModel from '../../coord/parallel/ParallelModel';
 
 type ParallelSeriesDataValue = OptionDataValue[];
@@ -89,12 +91,10 @@ class ParallelSeriesModel extends SeriesModel<ParallelSeriesOption> {
     coordinateSystem: Parallel;
 
 
-    getInitialData(option: ParallelSeriesOption, ecModel: GlobalModel): List {
-        const source = this.getSource();
-
-        setEncodeAndDimensions(source, this);
-
-        return createListFromArray(source, this);
+    getInitialData(this: ParallelSeriesModel, option: ParallelSeriesOption, ecModel: GlobalModel): List {
+        return createListFromArray(this.getSource(), this, {
+            useEncodeDefaulter: bind(makeDefaultEncode, null, this)
+        });
     }
 
     /**
@@ -151,7 +151,7 @@ class ParallelSeriesModel extends SeriesModel<ParallelSeriesOption> {
 
 SeriesModel.registerClass(ParallelSeriesModel);
 
-function setEncodeAndDimensions(source: Source, seriesModel: ParallelSeriesModel): void {
+function makeDefaultEncode(seriesModel: ParallelSeriesModel): OptionEncode {
     // The mapping of parallelAxis dimension to data dimension can
     // be specified in parallelAxis.option.dim. For example, if
     // parallelAxis.option.dim is 'dim3', it mapping to the third
@@ -159,10 +159,6 @@ function setEncodeAndDimensions(source: Source, seriesModel: ParallelSeriesModel
     // Moreover, parallelModel.dimension should not be regarded as data
     // dimensions. Consider dimensions = ['dim4', 'dim2', 'dim6'];
 
-    if (source.encodeDefine) {
-        return;
-    }
-
     const parallelModel = seriesModel.ecModel.getComponent(
         'parallel', seriesModel.get('parallelIndex')
     ) as ParallelModel;
@@ -170,11 +166,13 @@ function setEncodeAndDimensions(source: Source, seriesModel: ParallelSeriesModel
         return;
     }
 
-    const encodeDefine = source.encodeDefine = createHashMap();
+    const encodeDefine: Dictionary<OptionEncodeValue> = {};
     each(parallelModel.dimensions, function (axisDim) {
         const dataDimIndex = convertDimNameToNumber(axisDim);
-        encodeDefine.set(axisDim, dataDimIndex);
+        encodeDefine[axisDim] = dataDimIndex;
     });
+
+    return encodeDefine;
 }
 
 function convertDimNameToNumber(dimName: DimensionName): number {

src/component/dataset.ts

@@ -28,22 +28,33 @@
 
 import ComponentModel from '../model/Component';
 import ComponentView from '../view/Component';
-import {detectSourceFormat} from '../data/helper/sourceHelper';
 import {
-    SERIES_LAYOUT_BY_COLUMN, ComponentOption, SeriesEncodeOptionMixin, OptionSourceData, SeriesLayoutBy
+    SERIES_LAYOUT_BY_COLUMN, ComponentOption, SeriesEncodeOptionMixin,
+    OptionSourceData, SeriesLayoutBy, OptionSourceHeader
 } from '../util/types';
+import { DataTransformOption, PipedDataTransformOption } from '../data/helper/transform';
+import GlobalModel from '../model/Global';
+import Model from '../model/Model';
+import { disableTransformOptionMerge, SourceManager } from '../data/helper/sourceManager';
 
 
-interface DatasetOption extends
+export interface DatasetOption extends
         Pick<ComponentOption, 'type' | 'id' | 'name'>,
         Pick<SeriesEncodeOptionMixin, 'dimensions'> {
     seriesLayoutBy?: SeriesLayoutBy;
-    // null/undefined/'auto': auto detect header, see "src/data/helper/sourceHelper".
-    sourceHeader?: boolean | 'auto';
-    data?: OptionSourceData;
+    sourceHeader?: OptionSourceHeader;
+    source?: OptionSourceData;
+
+    fromDatasetIndex?: number;
+    fromDatasetId?: string;
+    transform?: DataTransformOption | PipedDataTransformOption;
+    // When a transform result more than on results, the results can be referenced only by:
+    // Using `fromDatasetIndex`/`fromDatasetId` and `transfromResultIndex` to retrieve
+    // the results from other dataset.
+    fromTransformResult?: number;
 }
 
-class DatasetModel extends ComponentModel {
+export class DatasetModel<Opts extends DatasetOption = DatasetOption> extends ComponentModel<Opts> {
 
     type = 'dataset';
     static type = 'dataset';
@@ -52,18 +63,33 @@ class DatasetModel extends ComponentModel {
         seriesLayoutBy: SERIES_LAYOUT_BY_COLUMN
     };
 
+    private _sourceManager: SourceManager;
+
+    init(option: Opts, parentModel: Model, ecModel: GlobalModel): void {
+        super.init(option, parentModel, ecModel);
+        this._sourceManager = new SourceManager(this);
+        disableTransformOptionMerge(this);
+    }
+
+    mergeOption(newOption: Opts, ecModel: GlobalModel): void {
+        super.mergeOption(newOption, ecModel);
+        disableTransformOptionMerge(this);
+    }
+
     optionUpdated() {
-        detectSourceFormat(this);
+        this._sourceManager.dirty();
+    }
+
+    getSourceManager() {
+        return this._sourceManager;
     }
 }
 
 ComponentModel.registerClass(DatasetModel);
 
-
 class DatasetView extends ComponentView {
     static type = 'dataset';
     type = 'dataset';
 }
 
 ComponentView.registerClass(DatasetView);
-

src/component/transform.ts

+/*
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements.  See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership.  The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License.  You may obtain a copy of the License at
+*
+*   http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing,
+* software distributed under the License is distributed on an
+* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+* KIND, either express or implied.  See the License for the
+* specific language governing permissions and limitations
+* under the License.
+*/
+
+import * as echarts from '../echarts';
+import {filterTransform} from './transform/filterTransform';
+import {sortTransform} from './transform/sortTransform';
+
+echarts.registerTransform(filterTransform);
+echarts.registerTransform(sortTransform);

src/component/transform/filterTransform.ts

+/*
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements.  See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership.  The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License.  You may obtain a copy of the License at
+*
+*   http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing,
+* software distributed under the License is distributed on an
+* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+* KIND, either express or implied.  See the License for the
+* specific language governing permissions and limitations
+* under the License.
+*/
+
+import { DataTransformOption, ExternalDataTransform } from '../../data/helper/transform';
+import { DimensionIndex, OptionDataItem } from '../../util/types';
+import { parseConditionalExpression, ConditionalExpressionOption } from '../../util/conditionalExpression';
+import { hasOwn, createHashMap } from 'zrender/src/core/util';
+import { makePrintable, throwError } from '../../util/log';
+
+
+export interface FilterTransformOption extends DataTransformOption {
+    type: 'filter';
+    config: ConditionalExpressionOption;
+}
+
+export const filterTransform: ExternalDataTransform<FilterTransformOption> = {
+
+    type: 'echarts:filter',
+
+    // PEDING: enhance to filter by index rather than create new data
+    transform: function transform(params) {
+        // [Caveat] Fail-Fast:
+        // Do not return the whole dataset unless user config indicate it explicitly.
+        // For example, if no condition specified by mistake, return an empty result
+        // is better than return the entire raw soruce for user to find the mistake.
+
+        const source = params.source;
+        let rawItem: OptionDataItem;
+
+        const condition = parseConditionalExpression<{ dimIdx: DimensionIndex }>(params.config, {
+
+            valueGetterAttrMap: createHashMap<boolean, string>({ dimension: true }),
+
+            prepareGetValue: function (exprOption) {
+                let errMsg = '';
+                const dimLoose = exprOption.dimension;
+                if (!hasOwn(exprOption, 'dimension')) {
+                    if (__DEV__) {
+                        errMsg = makePrintable(
+                            'Relation condition must has prop "dimension" specified.',
+                            'Illegal condition:', exprOption
+                        );
+                    }
+                    throwError(errMsg);
+                }
+
+                const dimInfo = source.getDimensionInfo(dimLoose);
+                if (!dimInfo) {
+                    if (__DEV__) {
+                        errMsg = makePrintable(
+                            'Can not find dimension info via: "' + dimLoose + '".\n',
+                            'Existing dimensions: ', source.dimensions, '.\n',
+                            'Illegal condition:', exprOption, '.\n'
+                        );
+                    }
+                    throwError(errMsg);
+                }
+
+                return { dimIdx: dimInfo.index };
+            },
+
+            getValue: function (param) {
+                return source.retrieveItemValue(rawItem, param.dimIdx);
+            }
+        });
+
+        const sourceHeaderCount = source.sourceHeaderCount;
+        const resultData = [];
+        for (let i = 0; i < sourceHeaderCount; i++) {
+            resultData.push(source.getRawHeaderItem(i));
+        }
+        for (let i = 0, len = source.count(); i < len; i++) {
+            rawItem = source.getRawDataItem(i);
+            if (condition.evaluate()) {
+                resultData.push(rawItem);
+            }
+        }
+
+        return {
+            data: resultData,
+            dimensions: source.dimensions,
+            sourceHeader: sourceHeaderCount
+        };
+    }
+};

src/component/transform/sortTransform.ts

+/*
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements.  See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership.  The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License.  You may obtain a copy of the License at
+*
+*   http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing,
+* software distributed under the License is distributed on an
+* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+* KIND, either express or implied.  See the License for the
+* specific language governing permissions and limitations
+* under the License.
+*/
+
+import { DataTransformOption, ExternalDataTransform } from '../../data/helper/transform';
+import {
+    DimensionLoose, SOURCE_FORMAT_KEYED_COLUMNS, DimensionIndex, OptionDataValue
+} from '../../util/types';
+import { makePrintable, throwError } from '../../util/log';
+import { isArray, each, hasOwn } from 'zrender/src/core/util';
+import { normalizeToArray } from '../../util/model';
+import { parseDate } from '../../util/number';
+
+/**
+ * @usage
+ *
+ * ```js
+ * transform: {
+ *     type: 'sort',
+ *     config: { dimension: 'score', order: 'asc' }
+ * }
+ * transform: {
+ *     type: 'sort',
+ *     config: [
+ *         { dimension: 1, order: 'asc' },
+ *         { dimension: 'age', order: 'desc' }
+ *     ]
+ * }
+ * ```
+ */
+
+export interface SortTransformOption extends DataTransformOption {
+    type: 'sort';
+    config: OrderExpression | OrderExpression[];
+}
+
+// PENDING: whether support { dimension: 'score', order: 'asc' } ?
+type OrderExpression = {
+    dimension: DimensionLoose;
+    order: SortOrder;
+    parse?: 'time'
+};
+
+type SortOrder = 'asc' | 'desc';
+const SortOrderValidMap = { asc: true, desc: true } as const;
+
+let sampleLog = '';
+if (__DEV__) {
+    sampleLog = [
+        'Valid config is like:',
+        '{ dimension: "age", order: "asc" }',
+        'or [{ dimension: "age", order: "asc"], { dimension: "date", order: "desc" }]'
+    ].join('');
+}
+
+const timeParser = function (val: OptionDataValue): number {
+    return +parseDate(val);
+};
+
+
+export const sortTransform: ExternalDataTransform<SortTransformOption> = {
+
+    type: 'echarts:sort',
+
+    transform: function transform(params) {
+        const source = params.source;
+        const config = params.config;
+        let errMsg = '';
+
+        // Normalize
+        // const orderExprList: OrderExpression[] = isArray(config[0])
+        //     ? config as OrderExpression[]
+        //     : [config as OrderExpression];
+        const orderExprList: OrderExpression[] = normalizeToArray(config);
+
+        if (!orderExprList.length) {
+            if (__DEV__) {
+                errMsg = 'Empty `config` in sort transform.';
+            }
+            throwError(errMsg);
+        }
+
+        const orderDefList: {
+            dimIdx: DimensionIndex;
+            orderReturn: -1 | 1;
+            parser: (val: OptionDataValue) => number;
+        }[] = [];
+        each(orderExprList, function (orderExpr) {
+            const dimLoose = orderExpr.dimension;
+            const order = orderExpr.order;
+            const parserName = orderExpr.parse;
+
+            if (dimLoose == null) {
+                if (__DEV__) {
+                    errMsg = 'Sort transform config must has "dimension" specified.' + sampleLog;
+                }
+                throwError(errMsg);
+            }
+
+            if (!hasOwn(SortOrderValidMap, order)) {
+                if (__DEV__) {
+                    errMsg = 'Sort transform config must has "order" specified.' + sampleLog;
+                }
+                throwError(errMsg);
+            }
+
+            const dimInfo = source.getDimensionInfo(dimLoose);
+            if (!dimInfo) {
+                if (__DEV__) {
+                    errMsg = makePrintable(
+                        'Can not find dimension info via: "' + dimLoose + '".\n',
+                        'Existing dimensions: ', source.dimensions, '.\n',
+                        'Illegal config:', orderExpr, '.\n'
+                    );
+                }
+                throwError(errMsg);
+            }
+
+            let parser;
+            if (parserName) {
+                if (parserName !== 'time') {
+                    if (__DEV__) {
+                        errMsg = makePrintable(
+                            'Invalid parser name' + parserName + '.\n',
+                            'Illegal config:', orderExpr, '.\n'
+                        );
+                    }
+                    throwError(errMsg);
+                }
+                parser = timeParser;
+            }
+
+            orderDefList.push({
+                dimIdx: dimInfo.index,
+                orderReturn: order === 'asc' ? -1 : 1,
+                parser: parser
+            });
+        });
+
+        // TODO: support it?
+        if (!isArray(source.data)) {
+            if (__DEV__) {
+                errMsg = source.sourceFormat === SOURCE_FORMAT_KEYED_COLUMNS
+                    ? 'sourceFormat ' + SOURCE_FORMAT_KEYED_COLUMNS + ' is not supported yet'
+                    : source.data == null
+                    ? 'Upstream source data is null/undefined'
+                    : 'Unsupported source format.';
+            }
+            throwError(errMsg);
+        }
+
+        // Other source format are all array.
+        const sourceHeaderCount = source.sourceHeaderCount;
+        const resultData = [];
+        const headerPlaceholder = {};
+        for (let i = 0; i < sourceHeaderCount; i++) {
+            resultData.push(headerPlaceholder);
+        }
+        for (let i = 0, len = source.count(); i < len; i++) {
+            resultData.push(source.getRawDataItem(i));
+        }
+
+        resultData.sort(function (item0, item1) {
+            if (item0 === headerPlaceholder) {
+                return -1;
+            }
+            if (item1 === headerPlaceholder) {
+                return 1;
+            }
+            // FIXME: check other empty?
+            // Always put empty item last?
+            if (item0 == null) {
+                return 1;
+            }
+            if (item1 == null) {
+                return -1;
+            }
+            // TODO Optimize a little: manually loop unrolling?
+            for (let i = 0; i < orderDefList.length; i++) {
+                const orderDef = orderDefList[i];
+                let val0 = source.retrieveItemValue(item0, orderDef.dimIdx);
+                let val1 = source.retrieveItemValue(item1, orderDef.dimIdx);
+                if (orderDef.parser) {
+                    val0 = orderDef.parser(val0);
+                    val1 = orderDef.parser(val1);
+                }
+                if (val0 < val1) {
+                    return orderDef.orderReturn;
+                }
+                else if (val0 > val1) {
+                    return -orderDef.orderReturn;
+                }
+            }
+            return 0;
+        });
+
+        for (let i = 0; i < sourceHeaderCount; i++) {
+            resultData[i] = source.getRawHeaderItem(i);
+        }
+
+        return {
+            data: resultData,
+            dimensions: source.dimensions,
+            sourceHeader: sourceHeaderCount
+        };
+    }
+};
+

src/component/visualMap/PiecewiseModel.ts

@@ -28,6 +28,7 @@ import ComponentModel from '../../model/Component';
 import { inheritDefaultOption } from '../../util/component';
 
 
+// TODO: use `relationExpression.ts` instead
 interface VisualPiece extends VisualOptionPiecewise {
     min?: number
     max?: number

src/data/List.ts

@@ -36,13 +36,13 @@ import {
     DimensionIndex, DimensionName, DimensionLoose, OptionDataItem,
     ParsedValue, ParsedValueNumeric, OrdinalNumber, DimensionUserOuput, ModelOption, SeriesDataType
 } from '../util/types';
-import {parseDate} from '../util/number';
 import {isDataItemOption} from '../util/model';
 import { getECData } from '../util/ecData';
 import { PathStyleProps } from 'zrender/src/graphic/Path';
 import type Graph from './Graph';
 import type Tree from './Tree';
 import type { VisualMeta } from '../component/visualMap/VisualMapModel';
+import { parseDataValue } from './helper/parseDataValue';
 
 
 const isObject = zrUtil.isObject;
@@ -1916,7 +1916,7 @@ class List<
             objectRows: function (
                 this: List, dataItem: Dictionary<any>, dimName: string, dataIndex: number, dimIndex: number
             ): ParsedValue {
-                return convertDataValue(dataItem[dimName], this._dimensionInfos[dimName]);
+                return parseDataValue(dataItem[dimName], this._dimensionInfos[dimName]);
             },
 
             keyedColumns: getDimValueSimply,
@@ -1934,7 +1934,7 @@ class List<
                 if (!this._rawData.pure && isDataItemOption(dataItem)) {
                     this.hasItemOption = true;
                 }
-                return convertDataValue(
+                return parseDataValue(
                     (value instanceof Array)
                         ? value[dimIndex]
                         // If value is a single number or something else not array.
@@ -1954,44 +1954,9 @@ class List<
         function getDimValueSimply(
             this: List, dataItem: any, dimName: string, dataIndex: number, dimIndex: number
         ): ParsedValue {
-            return convertDataValue(dataItem[dimIndex], this._dimensionInfos[dimName]);
+            return parseDataValue(dataItem[dimIndex], this._dimensionInfos[dimName]);
         }
 
-        /**
-         * Convert raw the value in to inner value in List.
-         * [Caution]: this is the key logic of user value parser.
-         * For backward compatibiliy, do not modify it until have to.
-         */
-        function convertDataValue(value: any, dimInfo: DataDimensionInfo): ParsedValue {
-            // Performance sensitive.
-            const dimType = dimInfo && dimInfo.type;
-            if (dimType === 'ordinal') {
-                // If given value is a category string
-                const ordinalMeta = dimInfo && dimInfo.ordinalMeta;
-                return ordinalMeta
-                    ? ordinalMeta.parseAndCollect(value)
-                    : value;
-            }
-
-            if (dimType === 'time'
-                // spead up when using timestamp
-                && typeof value !== 'number'
-                && value != null
-                && value !== '-'
-            ) {
-                value = +parseDate(value);
-            }
-
-            // dimType defaults 'number'.
-            // If dimType is not ordinal and value is null or undefined or NaN or '-',
-            // parse to NaN.
-            return (value == null || value === '')
-                ? NaN
-                // If string (like '-'), using '+' parse to NaN
-                // If object, also parse to NaN
-                : +value;
-        };
-
         prepareInvertedIndex = function (list: List): void {
             const invertedIndicesMap = list._invertedIndicesMap;
             zrUtil.each(invertedIndicesMap, function (invertedIndices, dim) {

src/data/Source.ts

@@ -67,11 +67,8 @@ import {
 
 class Source {
 
-    readonly fromDataset: boolean;
-
     /**
      * Not null/undefined.
-     * @type {Array|Object}
      */
     readonly data: OptionSourceData;
 
@@ -98,7 +95,7 @@ class Source {
      * can be null/undefined.
      * Might be specified outside.
      */
-    encodeDefine: HashMap<OptionEncodeValue, DimensionName>;
+    readonly encodeDefine: HashMap<OptionEncodeValue, DimensionName>;
 
     /**
      * Not null/undefined, uint.
@@ -112,27 +109,33 @@ class Source {
 
 
     constructor(fields: {
-        fromDataset: boolean,
-        data?: OptionSourceData,
-        sourceFormat?: SourceFormat, // default: SOURCE_FORMAT_UNKNOWN
+        data: OptionSourceData,
+        sourceFormat: SourceFormat, // default: SOURCE_FORMAT_UNKNOWN
+
+        // Visit config are optional:
         seriesLayoutBy?: SeriesLayoutBy, // default: 'column'
         dimensionsDefine?: DimensionDefinition[],
-        encodeDefine?: OptionEncode,
         startIndex?: number, // default: 0
-        dimensionsDetectCount?: number
+        dimensionsDetectCount?: number,
+
+        // [Caveat]
+        // This is the raw user defined `encode` in `series`.
+        // If user not defined, DO NOT make a empty object or hashMap here.
+        // An empty object or hashMap will prevent from auto generating encode.
+        encodeDefine?: HashMap<OptionEncodeValue, DimensionName>
     }) {
 
-        this.fromDataset = fields.fromDataset;
         this.data = fields.data || (
             fields.sourceFormat === SOURCE_FORMAT_KEYED_COLUMNS ? {} : []
         );
         this.sourceFormat = fields.sourceFormat || SOURCE_FORMAT_UNKNOWN;
+
+        // Visit config
         this.seriesLayoutBy = fields.seriesLayoutBy || SERIES_LAYOUT_BY_COLUMN;
-        this.dimensionsDefine = fields.dimensionsDefine;
-        this.encodeDefine = fields.encodeDefine
-            && createHashMap<OptionEncodeValue, DimensionName>(fields.encodeDefine);
         this.startIndex = fields.startIndex || 0;
+        this.dimensionsDefine = fields.dimensionsDefine;
         this.dimensionsDetectCount = fields.dimensionsDetectCount;
+        this.encodeDefine = fields.encodeDefine;
     }
 
     /**
@@ -143,8 +146,7 @@ class Source {
             data: data,
             sourceFormat: isTypedArray(data)
                 ? SOURCE_FORMAT_TYPED_ARRAY
-                : SOURCE_FORMAT_ORIGINAL,
-            fromDataset: false
+                : SOURCE_FORMAT_ORIGINAL
         });
     };
 }

src/data/helper/dataProvider.ts

@@ -21,7 +21,7 @@
 // ??? refactor? check the outer usage of data provider.
 // merge with defaultDimValueGetter?
 
-import {isTypedArray, extend, assert, each, isObject} from 'zrender/src/core/util';
+import {isTypedArray, extend, assert, each, isObject, bind} from 'zrender/src/core/util';
 import {getDataItemValue} from '../../util/model';
 import Source from '../Source';
 import {ArrayLike, Dictionary} from 'zrender/src/core/types';
@@ -34,7 +34,7 @@ import {
     SERIES_LAYOUT_BY_COLUMN,
     SERIES_LAYOUT_BY_ROW,
     DimensionName, DimensionIndex, OptionSourceData,
-    DimensionIndexLoose, OptionDataItem, OptionDataValue
+    DimensionIndexLoose, OptionDataItem, OptionDataValue, DimensionDefinition, SourceFormat, SeriesLayoutBy
 } from '../../util/types';
 import List from '../List';
 
@@ -54,6 +54,7 @@ export interface DataProvider {
 
 
 let providerMethods: Dictionary<any>;
+let mountMethods: (provider: DefaultDataProvider, data: OptionSourceData, source: Source) => void;
 
 /**
  * If normal array used, mutable chunk size is supported.
@@ -90,12 +91,10 @@ export class DefaultDataProvider implements DataProvider {
 
         // declare source is Source;
         this._source = source;
-
         const data = this._data = source.data;
-        const sourceFormat = source.sourceFormat;
 
         // Typed array. TODO IE10+?
-        if (sourceFormat === SOURCE_FORMAT_TYPED_ARRAY) {
+        if (source.sourceFormat === SOURCE_FORMAT_TYPED_ARRAY) {
             if (__DEV__) {
                 if (dimSize == null) {
                     throw new Error('Typed array data must specify dimension size');
@@ -106,17 +105,7 @@ export class DefaultDataProvider implements DataProvider {
             this._data = data;
         }
 
-        const methods = providerMethods[
-            sourceFormat === SOURCE_FORMAT_ARRAY_ROWS
-            ? sourceFormat + '_' + source.seriesLayoutBy
-            : sourceFormat
-        ];
-
-        if (__DEV__) {
-            assert(methods, 'Invalide sourceFormat: ' + sourceFormat);
-        }
-
-        extend(this, methods);
+        mountMethods(this, data, source);
     }
 
     getSource(): Source {
@@ -127,7 +116,7 @@ export class DefaultDataProvider implements DataProvider {
         return 0;
     }
 
-    getItem(idx: number): OptionDataItem {
+    getItem(idx: number, out?: ArrayLike<number>): OptionDataItem {
         return;
     }
 
@@ -139,35 +128,58 @@ export class DefaultDataProvider implements DataProvider {
 
     private static internalField = (function () {
 
+        mountMethods = function (provider, data, source) {
+            const sourceFormat = source.sourceFormat;
+            const seriesLayoutBy = source.seriesLayoutBy;
+            const startIndex = source.startIndex;
+            const dimsDef = source.dimensionsDefine;
+
+            const methods = providerMethods[getMethodMapKey(sourceFormat, seriesLayoutBy)];
+            if (__DEV__) {
+                assert(methods, 'Invalide sourceFormat: ' + sourceFormat);
+            }
+
+            extend(provider, methods);
+
+            if (sourceFormat === SOURCE_FORMAT_TYPED_ARRAY) {
+                provider.getItem = getItemForTypedArray;
+                provider.count = countForTypedArray;
+            }
+            else {
+                const rawItemGetter = getRawSourceItemGetter(sourceFormat, seriesLayoutBy);
+                provider.getItem = bind(rawItemGetter, null, data, startIndex, dimsDef);
+                const rawCounter = getRawSourceDataCounter(sourceFormat, seriesLayoutBy);
+                provider.count = bind(rawCounter, null, data, startIndex, dimsDef);
+            }
+        };
+
+        const getItemForTypedArray: DefaultDataProvider['getItem'] = function (
+            this: DefaultDataProvider, idx: number, out: ArrayLike<number>
+        ): ArrayLike<number> {
+            idx = idx - this._offset;
+            out = out || [];
+            const offset = this._dimSize * idx;
+            for (let i = 0; i < this._dimSize; i++) {
+                out[i] = (this._data as ArrayLike<number>)[offset + i];
+            }
+            return out;
+        };
+
+        const countForTypedArray: DefaultDataProvider['count'] = function (
+            this: DefaultDataProvider
+        ) {
+            return this._data ? ((this._data as ArrayLike<number>).length / this._dimSize) : 0;
+        };
+
         providerMethods = {
 
             [SOURCE_FORMAT_ARRAY_ROWS + '_' + SERIES_LAYOUT_BY_COLUMN]: {
                 pure: true,
-                count: function (this: DefaultDataProvider): number {
-                    return Math.max(0, (this._data as OptionDataItem[][]).length - this._source.startIndex);
-                },
-                getItem: function (this: DefaultDataProvider, idx: number): OptionDataValue[] {
-                    return (this._data as OptionDataValue[][])[idx + this._source.startIndex];
-                },
                 appendData: appendDataSimply
             },
 
             [SOURCE_FORMAT_ARRAY_ROWS + '_' + SERIES_LAYOUT_BY_ROW]: {
                 pure: true,
-                count: function (this: DefaultDataProvider): number {
-                    const row = (this._data as OptionDataValue[][])[0];
-                    return row ? Math.max(0, row.length - this._source.startIndex) : 0;
-                },
-                getItem: function (this: DefaultDataProvider, idx: number): OptionDataValue[] {
-                    idx += this._source.startIndex;
-                    const item = [];
-                    const data = this._data as OptionDataValue[][];
-                    for (let i = 0; i < data.length; i++) {
-                        const row = data[i];
-                        item.push(row ? row[idx] : null);
-                    }
-                    return item;
-                },
                 appendData: function () {
                     throw new Error('Do not support appendData when set seriesLayoutBy: "row".');
                 }
@@ -175,27 +187,11 @@ export class DefaultDataProvider implements DataProvider {
 
             [SOURCE_FORMAT_OBJECT_ROWS]: {
                 pure: true,
-                count: countSimply,
-                getItem: getItemSimply,
                 appendData: appendDataSimply
             },
 
             [SOURCE_FORMAT_KEYED_COLUMNS]: {
                 pure: true,
-                count: function (this: DefaultDataProvider): number {
-                    const dimName = this._source.dimensionsDefine[0].name;
-                    const col = (this._data as Dictionary<OptionDataValue[]>)[dimName];
-                    return col ? col.length : 0;
-                },
-                getItem: function (this: DefaultDataProvider, idx: number): OptionDataValue[] {
-                    const item = [];
-                    const dims = this._source.dimensionsDefine;
-                    for (let i = 0; i < dims.length; i++) {
-                        const col = (this._data as Dictionary<OptionDataValue[]>)[dims[i].name];
-                        item.push(col ? col[idx] : null);
-                    }
-                    return item;
-                },
                 appendData: function (this: DefaultDataProvider, newData: Dictionary<OptionDataValue[]>) {
                     const data = this._data as Dictionary<OptionDataValue[]>;
                     each(newData, function (newCol, key) {
@@ -208,26 +204,12 @@ export class DefaultDataProvider implements DataProvider {
             },
 
             [SOURCE_FORMAT_ORIGINAL]: {
-                count: countSimply,
-                getItem: getItemSimply,
                 appendData: appendDataSimply
             },
 
             [SOURCE_FORMAT_TYPED_ARRAY]: {
                 persistent: false,
                 pure: true,
-                count: function (this: DefaultDataProvider): number {
-                    return this._data ? ((this._data as ArrayLike<number>).length / this._dimSize) : 0;
-                },
-                getItem: function (this: DefaultDataProvider, idx: number, out: ArrayLike<number>): ArrayLike<number> {
-                    idx = idx - this._offset;
-                    out = out || [];
-                    const offset = this._dimSize * idx;
-                    for (let i = 0; i < this._dimSize; i++) {
-                        out[i] = (this._data as ArrayLike<number>)[offset + i];
-                    }
-                    return out;
-                },
                 appendData: function (this: DefaultDataProvider, newData: ArrayLike<number>): void {
                     if (__DEV__) {
                         assert(
@@ -235,7 +217,6 @@ export class DefaultDataProvider implements DataProvider {
                             'Added data must be TypedArray if data in initialization is TypedArray'
                         );
                     }
-
                     this._data = newData;
                 },
 
@@ -248,12 +229,6 @@ export class DefaultDataProvider implements DataProvider {
             }
         };
 
-        function countSimply(this: DefaultDataProvider): number {
-            return (this._data as []).length;
-        }
-        function getItemSimply(this: DefaultDataProvider, idx: number): OptionDataItem {
-            return (this._data as [])[idx];
-        }
         function appendDataSimply(this: DefaultDataProvider, newData: ArrayLike<OptionDataItem>): void {
             for (let i = 0; i < newData.length; i++) {
                 (this._data as any[]).push(newData[i]);
@@ -262,23 +237,136 @@ export class DefaultDataProvider implements DataProvider {
 
     })();
 }
+
+
+
+type RawSourceItemGetter = (
+    rawData: OptionSourceData,
+    startIndex: number,
+    dimsDef: DimensionDefinition[],
+    idx: number
+) => OptionDataItem;
+
+const getItemSimply: RawSourceItemGetter = function (
+    rawData, startIndex, dimsDef, idx
+): OptionDataItem {
+    return (rawData as [])[idx];
+};
+
+const rawSourceItemGetterMap: Dictionary<RawSourceItemGetter> = {
+    [SOURCE_FORMAT_ARRAY_ROWS + '_' + SERIES_LAYOUT_BY_COLUMN]: function (
+        rawData, startIndex, dimsDef, idx
+    ): OptionDataValue[] {
+        return (rawData as OptionDataValue[][])[idx + startIndex];
+    },
+    [SOURCE_FORMAT_ARRAY_ROWS + '_' + SERIES_LAYOUT_BY_ROW]: function (
+        rawData, startIndex, dimsDef, idx
+    ): OptionDataValue[] {
+        idx += startIndex;
+        const item = [];
+        const data = rawData as OptionDataValue[][];
+        for (let i = 0; i < data.length; i++) {
+            const row = data[i];
+            item.push(row ? row[idx] : null);
+        }
+        return item;
+    },
+    [SOURCE_FORMAT_OBJECT_ROWS]: getItemSimply,
+    [SOURCE_FORMAT_KEYED_COLUMNS]: function (
+        rawData, startIndex, dimsDef, idx
+    ): OptionDataValue[] {
+        const item = [];
+        for (let i = 0; i < dimsDef.length; i++) {
+            const col = (rawData as Dictionary<OptionDataValue[]>)[dimsDef[i].name];
+            item.push(col ? col[idx] : null);
+        }
+        return item;
+    },
+    [SOURCE_FORMAT_ORIGINAL]: getItemSimply
+};
+
+export function getRawSourceItemGetter(
+    sourceFormat: SourceFormat, seriesLayoutBy: SeriesLayoutBy
+): RawSourceItemGetter {
+    const method = rawSourceItemGetterMap[getMethodMapKey(sourceFormat, seriesLayoutBy)];
+    if (__DEV__) {
+        assert(method, 'Do not suppport get item on "' + sourceFormat + '", "' + seriesLayoutBy + '".');
+    }
+    return method;
+}
+
+
+
+
+type RawSourceDataCounter = (
+    rawData: OptionSourceData,
+    startIndex: number,
+    dimsDef: DimensionDefinition[]
+) => number;
+
+const countSimply: RawSourceDataCounter = function (
+    rawData, startIndex, dimsDef
+) {
+    return (rawData as []).length;
+};
+
+const rawSourceDataCounterMap: Dictionary<RawSourceDataCounter> = {
+    [SOURCE_FORMAT_ARRAY_ROWS + '_' + SERIES_LAYOUT_BY_COLUMN]: function (
+        rawData, startIndex, dimsDef
+    ) {
+        return Math.max(0, (rawData as OptionDataItem[][]).length - startIndex);
+    },
+    [SOURCE_FORMAT_ARRAY_ROWS + '_' + SERIES_LAYOUT_BY_ROW]: function (
+        rawData, startIndex, dimsDef
+    ) {
+        const row = (rawData as OptionDataValue[][])[0];
+        return row ? Math.max(0, row.length - startIndex) : 0;
+    },
+    [SOURCE_FORMAT_OBJECT_ROWS]: countSimply,
+    [SOURCE_FORMAT_KEYED_COLUMNS]: function (
+        rawData, startIndex, dimsDef
+    ) {
+        const dimName = dimsDef[0].name;
+        const col = (rawData as Dictionary<OptionDataValue[]>)[dimName];
+        return col ? col.length : 0;
+    },
+    [SOURCE_FORMAT_ORIGINAL]: countSimply
+};
+
+export function getRawSourceDataCounter(
+    sourceFormat: SourceFormat, seriesLayoutBy: SeriesLayoutBy
+): RawSourceDataCounter {
+    const method = rawSourceDataCounterMap[getMethodMapKey(sourceFormat, seriesLayoutBy)];
+    if (__DEV__) {
+        assert(method, 'Do not suppport count on "' + sourceFormat + '", "' + seriesLayoutBy + '".');
+    }
+    return method;
+}
+
+
+
 // TODO
 // merge it to dataProvider?
-type RawValueGetter = (
+type RawSourceValueGetter = (
     dataItem: OptionDataItem,
-    dataIndex: number,
     dimIndex: DimensionIndex,
     dimName: DimensionName
     // If dimIndex not provided, return OptionDataItem.
     // If dimIndex provided, return OptionDataPrimitive.
 ) => OptionDataValue | OptionDataItem;
 
-const rawValueGetters: {[sourceFormat: string]: RawValueGetter} = {
+const getRawValueSimply = function (
+    dataItem: ArrayLike<OptionDataValue>, dimIndex: number, dimName: string
+): OptionDataValue | ArrayLike<OptionDataValue> {
+    return dimIndex != null ? dataItem[dimIndex] : dataItem;
+};
+
+const rawSourceValueGetterMap: {[sourceFormat: string]: RawSourceValueGetter} = {
 
     [SOURCE_FORMAT_ARRAY_ROWS]: getRawValueSimply,
 
     [SOURCE_FORMAT_OBJECT_ROWS]: function (
-        dataItem: Dictionary<OptionDataValue>, dataIndex: number, dimIndex: number, dimName: string
+        dataItem: Dictionary<OptionDataValue>, dimIndex: number, dimName: string
     ): OptionDataValue | Dictionary<OptionDataValue> {
         return dimIndex != null ? dataItem[dimName] : dataItem;
     },
@@ -286,7 +374,7 @@ const rawValueGetters: {[sourceFormat: string]: RawValueGetter} = {
     [SOURCE_FORMAT_KEYED_COLUMNS]: getRawValueSimply,
 
     [SOURCE_FORMAT_ORIGINAL]: function (
-        dataItem: OptionDataItem, dataIndex: number, dimIndex: number, dimName: string
+        dataItem: OptionDataItem, dimIndex: number, dimName: string
     ): OptionDataValue | OptionDataItem {
         // FIXME: In some case (markpoint in geo (geo-map.html)),
         // dataItem is {coord: [...]}
@@ -299,12 +387,22 @@ const rawValueGetters: {[sourceFormat: string]: RawValueGetter} = {
     [SOURCE_FORMAT_TYPED_ARRAY]: getRawValueSimply
 };
 
-function getRawValueSimply(
-    dataItem: ArrayLike<OptionDataValue>, dataIndex: number, dimIndex: number, dimName: string
-): OptionDataValue | ArrayLike<OptionDataValue> {
-    return dimIndex != null ? dataItem[dimIndex] : dataItem;
+export function getRawSourceValueGetter(sourceFormat: SourceFormat): RawSourceValueGetter {
+    const method = rawSourceValueGetterMap[sourceFormat];
+    if (__DEV__) {
+        assert(method, 'Do not suppport get value on "' + sourceFormat + '".');
+    }
+    return method;
 }
 
+
+function getMethodMapKey(sourceFormat: SourceFormat, seriesLayoutBy: SeriesLayoutBy): string {
+    return sourceFormat === SOURCE_FORMAT_ARRAY_ROWS
+        ? sourceFormat + '_' + seriesLayoutBy
+        : sourceFormat;
+}
+
+
 // ??? FIXME can these logic be more neat: getRawValue, getRawDataItem,
 // Consider persistent.
 // Caution: why use raw value to display on label or tooltip?
@@ -336,9 +434,10 @@ export function retrieveRawValue(
         dimIndex = dimInfo.index;
     }
 
-    return rawValueGetters[sourceFormat](dataItem, dataIndex, dimIndex, dimName);
+    return getRawSourceValueGetter(sourceFormat)(dataItem, dimIndex, dimName);
 }
 
+
 /**
  * Compatible with some cases (in pie, map) like:
  * data: [{name: 'xx', value: 5, selected: true}, ...]

src/data/helper/parseDataValue.ts

+/*
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements.  See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership.  The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License.  You may obtain a copy of the License at
+*
+*   http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing,
+* software distributed under the License is distributed on an
+* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+* KIND, either express or implied.  See the License for the
+* specific language governing permissions and limitations
+* under the License.
+*/
+
+import { ParsedValue, DimensionType } from '../../util/types';
+import OrdinalMeta from '../OrdinalMeta';
+import { parseDate } from '../../util/number';
+
+
+/**
+ * Convert raw the value in to inner value in List.
+ *
+ * [Performance sensitive]
+ *
+ * [Caution]: this is the key logic of user value parser.
+ * For backward compatibiliy, do not modify it until have to !
+ */
+export function parseDataValue(
+    value: any,
+    // For high performance, do not omit the second param.
+    opt: {
+        // Default type: 'number'. There is no 'unknown' type. That is, a string
+        // will be parsed to NaN if do not set `type` as 'ordinal'. It has been
+        // the logic in `List.ts` for long time. Follow the same way if you need
+        // to get same result as List did from a raw value.
+        type?: DimensionType,
+        ordinalMeta?: OrdinalMeta
+    }
+): ParsedValue {
+    // Performance sensitive.
+    const dimType = opt && opt.type;
+    if (dimType === 'ordinal') {
+        // If given value is a category string
+        const ordinalMeta = opt && opt.ordinalMeta;
+        return ordinalMeta
+            ? ordinalMeta.parseAndCollect(value)
+            : value;
+    }
+
+    if (dimType === 'time'
+        // spead up when using timestamp
+        && typeof value !== 'number'
+        && value != null
+        && value !== '-'
+    ) {
+        value = +parseDate(value);
+    }
+
+    // dimType defaults 'number'.
+    // If dimType is not ordinal and value is null or undefined or NaN or '-',
+    // parse to NaN.
+    return (value == null || value === '')
+        ? NaN
+        // If string (like '-'), using '+' parse to NaN
+        // If object, also parse to NaN
+        : +value;
+};
+

src/data/helper/sourceHelper.ts

@@ -18,7 +18,7 @@
 */
 
 
-import {makeInner, getDataItemValue} from '../../util/model';
+import {makeInner, getDataItemValue, queryReferringComponents, SINGLE_REFERRING} from '../../util/model';
 import {
     createHashMap,
     each,
@@ -31,7 +31,9 @@ import {
     extend,
     assert,
     hasOwn,
-    HashMap
+    HashMap,
+    isNumber,
+    clone
 } from 'zrender/src/core/util';
 import Source from '../Source';
 
@@ -45,8 +47,6 @@ import {
     SOURCE_FORMAT_UNKNOWN,
     SourceFormat,
     Dictionary,
-    SeriesEncodeOptionMixin,
-    SeriesOption,
     OptionSourceData,
     SeriesLayoutBy,
     OptionSourceHeader,
@@ -59,9 +59,11 @@ import {
     OptionSourceDataOriginal,
     OptionSourceDataObjectRows,
     OptionEncode,
-    DimensionIndex
+    DimensionIndex,
+    SeriesEncodableModel,
+    OptionEncodeValue
 } from '../../util/types';
-import { DatasetModel } from '../../component/dataset';
+import { DatasetModel, DatasetOption } from '../../component/dataset';
 import SeriesModel from '../../model/Series';
 import GlobalModel from '../../model/Global';
 import { CoordDimensionDefinition } from './createDimensions';
@@ -74,16 +76,11 @@ export const BE_ORDINAL = {
 };
 type BeOrdinalValue = (typeof BE_ORDINAL)[keyof typeof BE_ORDINAL];
 
-const innerDatasetModel = makeInner<{
-    sourceFormat: SourceFormat;
-}, DatasetModel>();
-const innerSeriesModel = makeInner<{
-    source: Source;
-}, SeriesModel>();
 const innerGlobalModel = makeInner<{
     datasetMap: HashMap<DatasetRecord, string>
 }, GlobalModel>();
 
+
 interface DatasetRecord {
     categoryWayDim: number;
     valueWayDim: number;
@@ -93,11 +90,13 @@ type SeriesEncodeInternal = {
     [key in keyof OptionEncode]: DimensionIndex[];
 };
 
-type SeriesEncodableModel = SeriesModel<SeriesOption & SeriesEncodeOptionMixin>;
-
+export interface SourceMetaRawOption {
+    seriesLayoutBy: SeriesLayoutBy;
+    sourceHeader: OptionSourceHeader;
+    dimensions: DimensionDefinitionLoose[];
+}
 
-export function detectSourceFormat(datasetModel: DatasetModel): void {
-    const data = datasetModel.option.source;
+export function detectSourceFormat(data: DatasetOption['source']): SourceFormat {
     let sourceFormat: SourceFormat = SOURCE_FORMAT_UNKNOWN;
 
     if (isTypedArray(data)) {
@@ -137,33 +136,7 @@ export function detectSourceFormat(datasetModel: DatasetModel): void {
         throw new Error('Invalid data');
     }
 
-    innerDatasetModel(datasetModel).sourceFormat = sourceFormat;
-}
-
-/**
- * [Scenarios]:
- * (1) Provide source data directly:
- *     series: {
- *         encode: {...},
- *         dimensions: [...]
- *         seriesLayoutBy: 'row',
- *         data: [[...]]
- *     }
- * (2) Refer to datasetModel.
- *     series: [{
- *         encode: {...}
- *         // Ignore datasetIndex means `datasetIndex: 0`
- *         // and the dimensions defination in dataset is used
- *     }, {
- *         encode: {...},
- *         seriesLayoutBy: 'column',
- *         datasetIndex: 1
- *     }]
- *
- * Get data from series itself or datset.
- */
-export function getSource(seriesModel: SeriesModel): Source {
-    return innerSeriesModel(seriesModel).source;
+    return sourceFormat;
 }
 
 /**
@@ -174,65 +147,63 @@ export function resetSourceDefaulter(ecModel: GlobalModel): void {
     innerGlobalModel(ecModel).datasetMap = createHashMap();
 }
 
-/**
- * [Caution]:
- * MUST be called after series option merged and
- * before "series.getInitailData()" called.
- *
- * [The rule of making default encode]:
- * Category axis (if exists) alway map to the first dimension.
- * Each other axis occupies a subsequent dimension.
- *
- * [Why make default encode]:
- * Simplify the typing of encode in option, avoiding the case like that:
- * series: [{encode: {x: 0, y: 1}}, {encode: {x: 0, y: 2}}, {encode: {x: 0, y: 3}}],
- * where the "y" have to be manually typed as "1, 2, 3, ...".
- */
-export function prepareSource(seriesModel: SeriesEncodableModel): void {
-    const seriesOption = seriesModel.option;
-
-    let data = seriesOption.data as OptionSourceData;
-    let sourceFormat: SourceFormat = isTypedArray(data)
-        ? SOURCE_FORMAT_TYPED_ARRAY : SOURCE_FORMAT_ORIGINAL;
-    let fromDataset = false;
-
-    let seriesLayoutBy = seriesOption.seriesLayoutBy;
-    let sourceHeader = seriesOption.sourceHeader;
-    let dimensionsDefine = seriesOption.dimensions;
-
-    const datasetModel = getDatasetModel(seriesModel);
-    if (datasetModel) {
-        const datasetOption = datasetModel.option;
-
-        data = datasetOption.source;
-        sourceFormat = innerDatasetModel(datasetModel).sourceFormat;
-        fromDataset = true;
+export function createSource(
+    sourceData: OptionSourceData,
+    thisMetaRawOption: SourceMetaRawOption,
+    // can be null. If not provided, auto detect it from `sourceData`.
+    sourceFormat: SourceFormat,
+    encodeDefine: OptionEncode  // can be null
+): Source {
+    sourceFormat = sourceFormat || detectSourceFormat(sourceData);
+    const dimInfo = determineSourceDimensions(
+        sourceData,
+        sourceFormat,
+        thisMetaRawOption.seriesLayoutBy,
+        thisMetaRawOption.sourceHeader,
+        thisMetaRawOption.dimensions
+    );
+    const source = new Source({
+        data: sourceData,
+        sourceFormat: sourceFormat,
 
-        // These settings from series has higher priority.
-        seriesLayoutBy = seriesLayoutBy || datasetOption.seriesLayoutBy;
-        sourceHeader == null && (sourceHeader = datasetOption.sourceHeader);
-        dimensionsDefine = dimensionsDefine || datasetOption.dimensions;
-    }
+        seriesLayoutBy: thisMetaRawOption.seriesLayoutBy,
+        dimensionsDefine: dimInfo.dimensionsDefine,
+        startIndex: dimInfo.startIndex,
+        dimensionsDetectCount: dimInfo.dimensionsDetectCount,
+        encodeDefine: makeEncodeDefine(encodeDefine)
+    });
 
-    const completeResult = completeBySourceData(
-        data, sourceFormat, seriesLayoutBy, sourceHeader, dimensionsDefine
-    );
+    return source;
+}
 
-    innerSeriesModel(seriesModel).source = new Source({
-        data: data,
-        fromDataset: fromDataset,
-        seriesLayoutBy: seriesLayoutBy,
-        sourceFormat: sourceFormat,
-        dimensionsDefine: completeResult.dimensionsDefine,
-        startIndex: completeResult.startIndex,
-        dimensionsDetectCount: completeResult.dimensionsDetectCount,
-        // Note: dataset option does not have `encode`.
-        encodeDefine: seriesOption.encode
+/**
+ * Clone except source data.
+ */
+export function cloneSourceShallow(source: Source) {
+    return new Source({
+        data: source.data,
+        sourceFormat: source.sourceFormat,
+
+        seriesLayoutBy: source.seriesLayoutBy,
+        dimensionsDefine: clone(source.dimensionsDefine),
+        startIndex: source.startIndex,
+        dimensionsDetectCount: source.dimensionsDetectCount,
+        encodeDefine: makeEncodeDefine(source.encodeDefine)
     });
 }
 
+function makeEncodeDefine(
+    encodeDefine: OptionEncode | HashMap<OptionEncodeValue, DimensionName>
+): HashMap<OptionEncodeValue, DimensionName> {
+    // null means user not specify `series.encode`.
+    return encodeDefine
+        ? createHashMap<OptionEncodeValue, DimensionName>(encodeDefine)
+        : null;
+}
+
+
 // return {startIndex, dimensionsDefine, dimensionsCount}
-function completeBySourceData(
+export function determineSourceDimensions(
     data: OptionSourceData,
     sourceFormat: SourceFormat,
     seriesLayoutBy: SeriesLayoutBy,
@@ -248,7 +219,7 @@ function completeBySourceData(
 
     if (!data) {
         return {
-            dimensionsDefine: normalizeDimensionsDefine(dimensionsDefine),
+            dimensionsDefine: normalizeDimensionsOption(dimensionsDefine),
             startIndex,
             dimensionsDetectCount
         };
@@ -275,7 +246,7 @@ function completeBySourceData(
             }, seriesLayoutBy, dataArrayRows, 10);
         }
         else {
-            startIndex = sourceHeader ? 1 : 0;
+            startIndex = isNumber(sourceHeader) ? sourceHeader : sourceHeader ? 1 : 0;
         }
 
         if (!dimensionsDefine && startIndex === 1) {
@@ -318,7 +289,7 @@ function completeBySourceData(
 
     return {
         startIndex: startIndex,
-        dimensionsDefine: normalizeDimensionsDefine(dimensionsDefine),
+        dimensionsDefine: normalizeDimensionsOption(dimensionsDefine),
         dimensionsDetectCount: dimensionsDetectCount
     };
 }
@@ -326,7 +297,7 @@ function completeBySourceData(
 // Consider dimensions defined like ['A', 'price', 'B', 'price', 'C', 'price'],
 // which is reasonable. But dimension name is duplicated.
 // Returns undefined or an array contains only object without null/undefiend or string.
-function normalizeDimensionsDefine(dimensionsDefine: DimensionDefinitionLoose[]): DimensionDefinition[] {
+function normalizeDimensionsOption(dimensionsDefine: DimensionDefinitionLoose[]): DimensionDefinition[] {
     if (!dimensionsDefine) {
         // The meaning of null/undefined is different from empty array.
         return;
@@ -419,7 +390,7 @@ export function makeSeriesEncodeForAxisCoordSys(
 ): SeriesEncodeInternal {
     const encode: SeriesEncodeInternal = {};
 
-    const datasetModel = getDatasetModel(seriesModel);
+    const datasetModel = querySeriesUpstreamDatasetModel(seriesModel);
     // Currently only make default when using dataset, util more reqirements occur.
     if (!datasetModel || !coordDimensions) {
         return encode;
@@ -512,7 +483,7 @@ export function makeSeriesEncodeForNameBased(
 ): SeriesEncodeInternal {
     const encode: SeriesEncodeInternal = {};
 
-    const datasetModel = getDatasetModel(seriesModel);
+    const datasetModel = querySeriesUpstreamDatasetModel(seriesModel);
     // Currently only make default when using dataset, util more reqirements occur.
     if (!datasetModel) {
         return encode;
@@ -600,21 +571,55 @@ export function makeSeriesEncodeForNameBased(
 }
 
 /**
- * If return null/undefined, indicate that should not use datasetModel.
+ * @return If return null/undefined, indicate that should not use datasetModel.
  */
-function getDatasetModel(seriesModel: SeriesEncodableModel): DatasetModel {
-    const option = seriesModel.option;
+export function querySeriesUpstreamDatasetModel(
+    seriesModel: SeriesEncodableModel
+): DatasetModel {
     // Caution: consider the scenario:
     // A dataset is declared and a series is not expected to use the dataset,
     // and at the beginning `setOption({series: { noData })` (just prepare other
     // option but no data), then `setOption({series: {data: [...]}); In this case,
     // the user should set an empty array to avoid that dataset is used by default.
-    const thisData = option.data;
+    const thisData = seriesModel.get('data', true);
     if (!thisData) {
-        return seriesModel.ecModel.getComponent('dataset', option.datasetIndex || 0) as DatasetModel;
+        return queryReferringComponents(
+            seriesModel.ecModel,
+            'dataset',
+            {
+                index: seriesModel.get('datasetIndex', true),
+                id: seriesModel.get('datasetId', true)
+            },
+            SINGLE_REFERRING
+        ).models[0] as DatasetModel;
     }
 }
 
+/**
+ * @return Always return an array event empty.
+ */
+export function queryDatasetUpstreamDatasetModels(
+    datasetModel: DatasetModel
+): DatasetModel[] {
+    // Only these attributes declared, we by defualt reference to `datasetIndex: 0`.
+    // Otherwise, no reference.
+    if (!datasetModel.get('transform', true)
+        && !datasetModel.get('fromTransformResult', true)
+    ) {
+        return [];
+    }
+
+    return queryReferringComponents(
+        datasetModel.ecModel,
+        'dataset',
+        {
+            index: datasetModel.get('fromDatasetIndex', true),
+            id: datasetModel.get('fromDatasetId', true)
+        },
+        SINGLE_REFERRING
+    ).models as DatasetModel[];
+}
+
 /**
  * The rule should not be complex, otherwise user might not
  * be able to known where the data is wrong.

src/data/helper/sourceManager.ts

+/*
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements.  See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership.  The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License.  You may obtain a copy of the License at
+*
+*   http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing,
+* software distributed under the License is distributed on an
+* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+* KIND, either express or implied.  See the License for the
+* specific language governing permissions and limitations
+* under the License.
+*/
+
+import { DatasetModel } from '../../component/dataset';
+import SeriesModel from '../../model/Series';
+import { setAsPrimitive, map, isTypedArray, defaults, assert, each } from 'zrender/src/core/util';
+import Source from '../Source';
+import {
+    SeriesEncodableModel, OptionSourceData,
+    SOURCE_FORMAT_TYPED_ARRAY, SOURCE_FORMAT_ORIGINAL,
+    SourceFormat, SeriesLayoutBy, OptionSourceHeader, DimensionDefinitionLoose
+} from '../../util/types';
+import {
+    querySeriesUpstreamDatasetModel, queryDatasetUpstreamDatasetModels,
+    createSource, SourceMetaRawOption, cloneSourceShallow
+} from './sourceHelper';
+import { applyDataTransform } from './transform';
+
+
+/**
+ * [REQUIREMENT_MEMO]:
+ * (0) `metaRawOption` means `dimensions`/`sourceHeader`/`seriesLayoutBy` in raw option.
+ * (1) Keep support the feature: `metaRawOption` can be specified both on `series` and
+ * `root-dataset`. Them on `series` has higher priority.
+ * (2) Do not support to set `metaRawOption` on a `non-root-dataset`, because it might
+ * confuse users: whether those props indicate how to visit the upstream source or visit
+ * the transform result source, and some transforms has nothing to do with these props,
+ * and some transforms might have multiple upstream.
+ * (3) Transforms should specify `metaRawOption` in each output, just like they can be
+ * declared in `root-dataset`.
+ * (4) At present only support visit source in `SERIES_LAYOUT_BY_COLUMN` in transforms.
+ * That is for reducing complexity in transfroms.
+ * PENDING: Whether to provide transposition transform?
+ *
+ * [IMPLEMENTAION_MEMO]:
+ * "sourceVisitConfig" are calculated from `metaRawOption` and `data`.
+ * They will not be calculated until `source` is about to be visited (to prevent from
+ * duplicate calcuation). `source` is visited only in series and input to transforms.
+ *
+ * [SCENARIO]:
+ * (1) Provide source data directly:
+ * ```js
+ * series: {
+ *     encode: {...},
+ *     dimensions: [...]
+ *     seriesLayoutBy: 'row',
+ *     data: [[...]]
+ * }
+ * ```
+ * (2) Series refer to dataset.
+ * ```js
+ * series: [{
+ *     encode: {...}
+ *     // Ignore datasetIndex means `datasetIndex: 0`
+ *     // and the dimensions defination in dataset is used
+ * }, {
+ *     encode: {...},
+ *     seriesLayoutBy: 'column',
+ *     datasetIndex: 1
+ * }]
+ * ```
+ * (3) dataset transform
+ * ```js
+ * dataset: [{
+ *     source: [...]
+ * }, {
+ *     source: [...]
+ * }, {
+ *     // By default from 0.
+ *     transform: { type: 'filter', config: {...} }
+ * }, {
+ *     // Piped.
+ *     transform: [
+ *         { type: 'filter', config: {...} },
+ *         { type: 'sort', config: {...} }
+ *     ]
+ * }, {
+ *     id: 'regressionData',
+ *     fromDatasetIndex: 1,
+ *     // Third-party transform
+ *     transform: { type: 'ecStat:regression', config: {...} }
+ * }, {
+ *     // retrieve the extra result.
+ *     id: 'regressionFormula',
+ *     fromDatasetId: 'regressionData',
+ *     fromTransformResult: 1
+ * }]
+ * ```
+ */
+
+export class SourceManager {
+
+    // Currently only datasetModel can host `transform`
+    private _sourceHost: DatasetModel | SeriesModel;
+
+    // Cached source. Do not repeat calculating if not dirty.
+    private _sourceList: Source[] = [];
+
+    // version sign of each upstream source manager.
+    private _upstreamSignList: string[] = [];
+
+    private _versionSignBase = 0;
+
+    constructor(sourceHost: DatasetModel | SeriesModel) {
+        this._sourceHost = sourceHost;
+    }
+
+    /**
+     * Mark dirty.
+     */
+    dirty() {
+        this._setLocalSource([], []);
+    }
+
+    private _setLocalSource(
+        sourceList: Source[],
+        upstreamSignList: string[]
+    ): void {
+        this._sourceList = sourceList;
+        this._upstreamSignList = upstreamSignList;
+        this._versionSignBase++;
+        if (this._versionSignBase > 9e10) {
+            this._versionSignBase = 0;
+        }
+    }
+
+    /**
+     * For detecting whether the upstream source is dirty, so that
+     * the local cached source (in `_sourceList`) should be discarded.
+     */
+    private _getVersionSign(): string {
+        return this._sourceHost.uid + '_' + this._versionSignBase;
+    }
+
+    /**
+     * Always return a source instance. Otherwise throw error.
+     */
+    prepareSource(): void {
+        // For the case that call `setOption` multiple time but no data changed,
+        // cache the result source to prevent from repeating transform.
+        if (this._isDirty()) {
+            this._createSource();
+        }
+    }
+
+    private _createSource(): void {
+        this._setLocalSource([], []);
+        const sourceHost = this._sourceHost;
+
+        const upSourceMgrList = this._getUpstreamSourceManagers();
+        const hasUpstream = !!upSourceMgrList.length;
+        let resultSourceList: Source[];
+        let upstreamSignList: string[];
+
+        if (isSeries(sourceHost)) {
+            const seriesModel = sourceHost as SeriesEncodableModel;
+            let data;
+            let sourceFormat: SourceFormat;
+            let upMetaRawOption;
+
+            // Has upstream dataset
+            if (hasUpstream) {
+                const upSourceMgr = upSourceMgrList[0];
+                upSourceMgr.prepareSource();
+                const upSource = upSourceMgr.getSource();
+                data = upSource.data;
+                sourceFormat = upSource.sourceFormat;
+                upMetaRawOption = upSourceMgr._getSourceMetaRawOption();
+                upstreamSignList = [upSourceMgr._getVersionSign()];
+            }
+            // Series data is from own.
+            else {
+                data = seriesModel.get('data', true) as OptionSourceData;
+                sourceFormat = isTypedArray(data)
+                    ? SOURCE_FORMAT_TYPED_ARRAY : SOURCE_FORMAT_ORIGINAL;
+                upstreamSignList = [];
+            }
+
+            const thisMetaRawOption = defaults(
+                this._getSourceMetaRawOption(),
+                // See [REQUIREMENT MEMO], merge settings on series and parent dataset if it is root.
+                upMetaRawOption
+            );
+            resultSourceList = [createSource(
+                data,
+                thisMetaRawOption,
+                sourceFormat,
+                seriesModel.get('encode', true)
+            )];
+        }
+        else {
+            const datasetModel = sourceHost as DatasetModel;
+
+            // Has upstream dataset.
+            if (hasUpstream) {
+                const result = this._applyTransform(upSourceMgrList);
+                resultSourceList = result.sourceList;
+                upstreamSignList = result.upstreamSignList;
+            }
+            // Is root dataset.
+            else {
+                const sourceData = datasetModel.get('source', true);
+                resultSourceList = [createSource(
+                    sourceData,
+                    this._getSourceMetaRawOption(),
+                    null,
+                    // Note: dataset option does not have `encode`.
+                    null
+                )];
+                upstreamSignList = [];
+            }
+        }
+
+        if (__DEV__) {
+            assert(resultSourceList && upstreamSignList);
+        }
+
+        this._setLocalSource(resultSourceList, upstreamSignList);
+    }
+
+    private _applyTransform(
+        upMgrList: SourceManager[]
+    ): {
+        sourceList: Source[],
+        upstreamSignList: string[]
+    } {
+        const datasetModel = this._sourceHost as DatasetModel;
+        const transformOption = datasetModel.get('transform', true);
+        const fromTransformResult = datasetModel.get('fromTransformResult', true);
+        let sourceList: Source[];
+        let upstreamSignList: string[];
+
+        if (transformOption) {
+            const upSourceList: Source[] = [];
+            upstreamSignList = [];
+            each(upMgrList, upMgr => {
+                upMgr.prepareSource();
+                upSourceList.push(upMgr.getSource());
+                upstreamSignList.push(upMgr._getVersionSign());
+            });
+            sourceList = applyDataTransform(
+                transformOption,
+                upSourceList,
+                { datasetIndex: datasetModel.componentIndex }
+            );
+        }
+        else if (fromTransformResult != null) {
+            if (upMgrList.length !== 1) {
+                let errMsg = '';
+                if (__DEV__) {
+                    errMsg = 'When using `fromTransformResult`, there should be only one upstream dataset';
+                }
+                doThrow(errMsg);
+            }
+            const upMgr = upMgrList[0];
+            upMgr.prepareSource();
+            const upSource = upMgr.getSource(fromTransformResult);
+            upstreamSignList = [upMgr._getVersionSign()];
+            sourceList = [cloneSourceShallow(upSource)];
+        }
+
+        return { sourceList, upstreamSignList };
+    }
+
+    private _isDirty(): boolean {
+        const sourceList = this._sourceList;
+        if (!sourceList.length) {
+            return true;
+        }
+
+        // All sourceList is from the some upsteam.
+        const upSourceMgrList = this._getUpstreamSourceManagers();
+        for (let i = 0; i < upSourceMgrList.length; i++) {
+            const upSrcMgr = upSourceMgrList[i];
+            if (
+                // Consider the case that there is ancestor diry, call it recursively.
+                // The performance is probably not an issue because usually the chain is not long.
+                upSrcMgr._isDirty()
+                || this._upstreamSignList[i] !== upSrcMgr._getVersionSign()
+            ) {
+                return true;
+            }
+        }
+    }
+
+    /**
+     * @param sourceIndex By defualt 0, means "main source".
+     *                    Most cases there is only one source.
+     */
+    getSource(sourceIndex?: number) {
+        return this._sourceList[sourceIndex || 0];
+    }
+
+    /**
+     * PEDING: Is it fast enough?
+     * If no upstream, return empty array.
+     */
+    private _getUpstreamSourceManagers(): SourceManager[] {
+        // Always get the relationship from the raw option.
+        // Do not cache the link of the dependency graph, so that
+        // no need to update them when change happen.
+        const sourceHost = this._sourceHost;
+
+        if (isSeries(sourceHost)) {
+            const datasetModel = querySeriesUpstreamDatasetModel(sourceHost);
+            return !datasetModel ? [] : [datasetModel.getSourceManager()];
+        }
+        else {
+            return map(
+                queryDatasetUpstreamDatasetModels(sourceHost as DatasetModel),
+                datasetModel => datasetModel.getSourceManager()
+            );
+        }
+    }
+
+    private _getSourceMetaRawOption(): SourceMetaRawOption {
+        const sourceHost = this._sourceHost;
+        let seriesLayoutBy: SeriesLayoutBy;
+        let sourceHeader: OptionSourceHeader;
+        let dimensions: DimensionDefinitionLoose[];
+        if (isSeries(sourceHost)) {
+            seriesLayoutBy = sourceHost.get('seriesLayoutBy', true);
+            sourceHeader = sourceHost.get('sourceHeader', true);
+            dimensions = sourceHost.get('dimensions', true);
+        }
+        // See [REQUIREMENT MEMO], `non-root-dataset` do not support them.
+        else if (!this._getUpstreamSourceManagers().length) {
+            const model = sourceHost as DatasetModel;
+            seriesLayoutBy = model.get('seriesLayoutBy', true);
+            sourceHeader = model.get('sourceHeader', true);
+            dimensions = model.get('dimensions', true);
+        }
+        return { seriesLayoutBy, sourceHeader, dimensions };
+    }
+
+}
+
+// Call this method after `super.init` and `super.mergeOption` to
+// disable the transform merge, but do not disable transfrom clone from rawOption.
+export function disableTransformOptionMerge(datasetModel: DatasetModel): void {
+    const transformOption = datasetModel.option.transform;
+    transformOption && setAsPrimitive(datasetModel.option.transform);
+}
+
+function isSeries(sourceHost: SourceManager['_sourceHost']): sourceHost is SeriesEncodableModel {
+    // Avoid circular dependency with Series.ts
+    return (sourceHost as SeriesModel).mainType === 'series';
+}
+
+function doThrow(errMsg: string): void {
+    throw new Error(errMsg);
+}

src/data/helper/transform.ts

+/*
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements.  See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership.  The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License.  You may obtain a copy of the License at
+*
+*   http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing,
+* software distributed under the License is distributed on an
+* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+* KIND, either express or implied.  See the License for the
+* specific language governing permissions and limitations
+* under the License.
+*/
+
+import {
+    Dictionary, OptionSourceData, DimensionDefinitionLoose, OptionSourceHeader,
+    SourceFormat, DimensionDefinition, OptionDataItem, DimensionIndex,
+    OptionDataValue, DimensionLoose, DimensionName, ParsedValue, SERIES_LAYOUT_BY_COLUMN
+} from '../../util/types';
+import Source from '../Source';
+import { normalizeToArray } from '../../util/model';
+import {
+    assert, createHashMap, bind, each, hasOwn, map, clone, isObject,
+    isArrayLike
+} from 'zrender/src/core/util';
+import {
+    getRawSourceItemGetter, getRawSourceDataCounter, getRawSourceValueGetter
+} from './dataProvider';
+import { parseDataValue } from './parseDataValue';
+import { createSource } from './sourceHelper';
+import { consoleLog, makePrintable } from '../../util/log';
+
+
+export type PipedDataTransformOption = DataTransformOption[];
+export type DataTransformType = string;
+export type DataTransformConfig = unknown;
+
+export interface DataTransformOption {
+    type: DataTransformType;
+    config: DataTransformConfig;
+    // Print the result via `console.log` when transform performed. Only work in dev mode for debug.
+    print?: boolean;
+}
+
+export interface DataTransformResult {
+    source: Source;
+}
+
+export interface DataTransform {
+    (sourceList: Source[], config: DataTransformConfig): {
+    }
+}
+
+export interface ExternalDataTransform<TO extends DataTransformOption = DataTransformOption> {
+    // Must include namespace like: 'ecStat:regression'
+    type: string,
+    transform?: (
+        param: ExternalDataTransformParam<TO>
+    ) => ExternalDataTransformResultItem | ExternalDataTransformResultItem[]
+}
+
+interface ExternalDataTransformParam<TO extends DataTransformOption = DataTransformOption> {
+    // This is the first source in sourceList. In most cases,
+    // there is only one upstream source.
+    source: ExternalSource;
+    sourceList: ExternalSource[];
+    config: TO['config'];
+}
+export interface ExternalDataTransformResultItem {
+    data: OptionSourceData;
+    dimensions?: DimensionDefinitionLoose[];
+    sourceHeader?: OptionSourceHeader;
+}
+export interface ExternalDimensionDefinition extends DimensionDefinition {
+    // Mandatory
+    index: DimensionIndex;
+}
+
+/**
+ * TODO: disable writable.
+ * This structure will be exposed to users.
+ */
+class ExternalSource {
+    /**
+     * [Caveat]
+     * This instance is to be exposed to users.
+     * DO NOT mount private members on this instance directly.
+     * If we have to use private members, we can make them in closure or use `makeInner`.
+     */
+
+    data: OptionSourceData;
+    sourceFormat: SourceFormat;
+    dimensions: ExternalDimensionDefinition[];
+    sourceHeaderCount: number;
+
+    getDimensionInfo(dim: DimensionLoose): ExternalDimensionDefinition {
+        return;
+    }
+
+    getRawDataItem(dataIndex: number): OptionDataItem {
+        return;
+    }
+
+    getRawHeaderItem(dataIndex: number): OptionDataItem {
+        return;
+    }
+
+    count(): number {
+        return;
+    }
+
+    /**
+     * Only support by dimension index.
+     * No need to support by dimension name in transform function,
+     * becuase transform function is not case-specific, no need to use name literally.
+     */
+    retrieveItemValue(rawItem: OptionDataItem, dimIndex: DimensionIndex): OptionDataValue {
+        return;
+    }
+
+    convertDataValue(rawVal: unknown, dimInfo: ExternalDimensionDefinition): ParsedValue {
+        return parseDataValue(rawVal, dimInfo);
+    }
+}
+
+function createExternalSource(
+    data: OptionSourceData,
+    sourceFormat: SourceFormat,
+    dimsDef: DimensionDefinition[],
+    sourceHeaderCount: number
+): ExternalSource {
+    const extSource = new ExternalSource();
+
+    extSource.data = data;
+    extSource.sourceFormat = sourceFormat;
+    extSource.sourceHeaderCount = sourceHeaderCount;
+
+    // Create a new dimensions structure for exposing.
+    const dimensions = extSource.dimensions = [] as ExternalDimensionDefinition[];
+    const dimsByName = {} as Dictionary<ExternalDimensionDefinition>;
+    each(dimsDef, function (dimDef, idx) {
+        const name = dimDef.name;
+        const dimDefExt = {
+            index: idx,
+            name: name,
+            displayName: dimDef.displayName
+        };
+        dimensions.push(dimDefExt);
+        // Users probably not sepcify dimension name. For simplicity, data transform
+        // do not generate dimension name.
+        if (name != null) {
+            // Dimension name should not be duplicated.
+            // For simplicity, data transform forbid name duplication, do not generate
+            // new name like module `completeDimensions.ts` did, but just tell users.
+            assert(!hasOwn(dimsByName, name), 'dimension name "' + name + '" duplicated.');
+            dimsByName[name] = dimDefExt;
+        }
+    });
+
+    // Implement public methods:
+    const rawItemGetter = getRawSourceItemGetter(sourceFormat, SERIES_LAYOUT_BY_COLUMN);
+    extSource.getRawDataItem = bind(rawItemGetter, null, data, sourceHeaderCount, dimensions);
+    extSource.getRawHeaderItem = function (dataIndex: number) {
+        if (dataIndex < sourceHeaderCount) {
+            return rawItemGetter(data, 0, dimensions, dataIndex);
+        }
+    };
+
+    const rawCounter = getRawSourceDataCounter(sourceFormat, SERIES_LAYOUT_BY_COLUMN);
+    extSource.count = bind(rawCounter, null, data, sourceHeaderCount, dimensions);
+
+    const rawValueGetter = getRawSourceValueGetter(sourceFormat);
+    extSource.retrieveItemValue = function (rawItem, dimIndex) {
+        if (rawItem == null) {
+            return;
+        }
+        const dimDef = extSource.dimensions[dimIndex];
+        // When `dimIndex` is `null`, `rawValueGetter` return the whole item.
+        if (dimDef) {
+            return rawValueGetter(rawItem, dimIndex, dimDef.name) as OptionDataValue;
+        }
+    };
+
+    extSource.getDimensionInfo = bind(getDimensionInfo, null, dimensions, dimsByName);
+
+    return extSource;
+}
+
+
+function getDimensionInfo(
+    dimensions: ExternalDimensionDefinition[],
+    dimsByName: Dictionary<ExternalDimensionDefinition>,
+    dim: DimensionLoose
+): ExternalDimensionDefinition {
+    if (dim == null) {
+        return;
+    }
+    // Keep the same logic as `List::getDimension` did.
+    if (typeof dim === 'number'
+        // If being a number-like string but not being defined a dimension name.
+        || (!isNaN(dim as any) && !hasOwn(dimsByName, dim))
+    ) {
+        return dimensions[dim as DimensionIndex];
+    }
+    else if (hasOwn(dimsByName, dim)) {
+        return dimsByName[dim as DimensionName];
+    }
+}
+
+
+
+const externalTransformMap = createHashMap<ExternalDataTransform, string>();
+
+export function registerExternalTransform(
+    externalTransform: ExternalDataTransform
+): void {
+    externalTransform = clone(externalTransform);
+    let type = externalTransform.type;
+    assert(type, 'Must have a `type` when `registerTransform`.');
+    const typeParsed = type.split(':');
+    assert(typeParsed.length === 2, 'Name must include namespace like "ns:regression".');
+    // Namespace 'echarts:xxx' is official namespace, where the transforms should
+    // be called directly via 'xxx' rather than 'echarts:xxx'.
+    if (typeParsed[0] === 'echarts') {
+        type = typeParsed[1];
+    }
+    externalTransformMap.set(type, externalTransform);
+}
+
+export function applyDataTransform(
+    rawTransOption: DataTransformOption | PipedDataTransformOption,
+    sourceList: Source[],
+    infoForPrint: { datasetIndex: number }
+): Source[] {
+    const pipedTransOption: PipedDataTransformOption = normalizeToArray(rawTransOption);
+
+    for (let i = 0, len = pipedTransOption.length; i < len; i++) {
+        const transOption = pipedTransOption[i];
+        sourceList = applySingleDataTransform(transOption, sourceList);
+        // piped transform only support single input, except the fist one.
+        // piped transform only support single output, except the last one.
+        if (i < len - 1) {
+            sourceList.length = Math.max(sourceList.length, 1);
+        }
+
+        if (__DEV__) {
+            if (transOption.print) {
+                const printStrArr = map(sourceList, source => {
+                    return '--- datasetIndex: ' + infoForPrint.datasetIndex + ', transform result: ---\n'
+                        + makePrintable(source.data);
+                }).join('\n');
+                consoleLog(printStrArr);
+            }
+        }
+    }
+
+    return sourceList;
+}
+
+function applySingleDataTransform(
+    rawTransOption: DataTransformOption,
+    upSourceList: Source[]
+): Source[] {
+    assert(upSourceList.length, 'Must have at least one upstream dataset.');
+
+    const transOption = rawTransOption;
+    const transType = transOption.type;
+    const externalTransform = externalTransformMap.get(transType);
+
+    assert(externalTransform, 'Can not find transform on type "' + transType + '".');
+
+    // Prepare source
+    const sourceList = map(upSourceList, function (source) {
+        return createExternalSource(
+            source.data,
+            source.sourceFormat,
+            source.dimensionsDefine,
+            source.startIndex
+        );
+    });
+
+    const resultList = normalizeToArray(
+        externalTransform.transform({
+            source: sourceList[0],
+            sourceList: sourceList,
+            config: clone(transOption.config)
+        })
+    );
+
+    return map(resultList, function (result) {
+        assert(
+            isObject(result),
+            'A transform should not return some empty results.'
+        );
+        assert(
+            isObject(result.data) || isArrayLike(result.data),
+            'Result data should be object or array in data transform.'
+        );
+
+        return createSource(
+            result.data,
+            {
+                seriesLayoutBy: SERIES_LAYOUT_BY_COLUMN,
+                sourceHeader: result.sourceHeader,
+                dimensions: result.dimensions
+            },
+            null,
+            null
+        );
+    });
+}
+

src/echarts.ts

@@ -100,6 +100,7 @@ import { handleLegacySelectEvents } from './legacy/dataSelectAction';
 
 // At least canvas renderer.
 import 'zrender/src/canvas/canvas';
+import { registerExternalTransform } from './data/helper/transform';
 
 declare let global: any;
 type ModelFinder = modelUtil.ModelFinder;
@@ -2716,6 +2717,8 @@ export function getMap(mapName: string) {
     };
 }
 
+export const registerTransform = registerExternalTransform;
+
 /**
  * Globa dispatchAction to a specified chart instance.
  */

src/model/Component.ts

@@ -29,7 +29,9 @@ import {
     ClassManager,
     mountExtend
 } from '../util/clazz';
-import {makeInner, ModelFinderIndexQuery, queryReferringComponents, ModelFinderIdQuery, QueryReferringOpt} from '../util/model';
+import {
+    makeInner, ModelFinderIndexQuery, queryReferringComponents, ModelFinderIdQuery, QueryReferringOpt
+} from '../util/model';
 import * as layout from '../util/layout';
 import GlobalModel from './Global';
 import {

src/model/OptionManager.ts

@@ -36,6 +36,7 @@ import {
     each, clone, map, isTypedArray, setAsPrimitive
     // , HashMap , createHashMap, extend, merge,
 } from 'zrender/src/core/util';
+import { DatasetOption } from '../component/dataset';
 
 const QUERY_REG = /^(min|max)?(.+)$/;
 
@@ -98,6 +99,9 @@ class OptionManager {
             each(normalizeToArray((rawOption as ECUnitOption).series), function (series: SeriesOption) {
                 series && series.data && isTypedArray(series.data) && setAsPrimitive(series.data);
             });
+            each(normalizeToArray((rawOption as ECUnitOption).dataset), function (dataset: DatasetOption) {
+                dataset && dataset.source && isTypedArray(dataset.source) && setAsPrimitive(dataset.source);
+            });
         }
 
         // Caution: some series modify option data, if do not clone,

src/model/Series.ts

@@ -41,10 +41,6 @@ import {
     fetchLayoutMode
 } from '../util/layout';
 import {createTask} from '../stream/task';
-import {
-    prepareSource,
-    getSource
-} from '../data/helper/sourceHelper';
 import {retrieveRawValue} from '../data/helper/dataProvider';
 import GlobalModel from './Global';
 import { CoordinateSystem } from '../coord/CoordinateSystem';
@@ -57,10 +53,12 @@ import Axis from '../coord/Axis';
 import { GradientObject } from 'zrender/src/graphic/Gradient';
 import type { BrushCommonSelectorsForSeries, BrushSelectableArea } from '../component/brush/selector';
 import makeStyleMapper from './mixin/makeStyleMapper';
+import { SourceManager } from '../data/helper/sourceManager';
 
 const inner = modelUtil.makeInner<{
     data: List
     dataBeforeProcessed: List
+    sourceManager: SourceManager
 }, SeriesModel>();
 
 function getSelectionKey(data: List, dataIndex: number): string {
@@ -139,7 +137,6 @@ class SeriesModel<Opt extends SeriesOption = SeriesOption> extends ComponentMode
     // Injected outside
     pipelineContext: PipelineContext;
 
-
     // ---------------------------------------
     // Props to tell visual/style.ts about how to do visual encoding.
     // ---------------------------------------
@@ -197,7 +194,8 @@ class SeriesModel<Opt extends SeriesOption = SeriesOption> extends ComponentMode
 
         this.mergeDefaultAndTheme(option, ecModel);
 
-        prepareSource(this);
+        const sourceManager = inner(this).sourceManager = new SourceManager(this);
+        sourceManager.prepareSource();
 
         const data = this.getInitialData(option, ecModel);
         wrapData(data, this);
@@ -273,7 +271,9 @@ class SeriesModel<Opt extends SeriesOption = SeriesOption> extends ComponentMode
             );
         }
 
-        prepareSource(this);
+        const sourceManager = inner(this).sourceManager;
+        sourceManager.dirty();
+        sourceManager.prepareSource();
 
         const data = this.getInitialData(newSeriesOption, ecModel);
         wrapData(data, this);
@@ -377,7 +377,7 @@ class SeriesModel<Opt extends SeriesOption = SeriesOption> extends ComponentMode
     }
 
     getSource(): Source {
-        return getSource(this);
+        return inner(this).sourceManager.getSource();
     }
 
     /**

src/util/conditionalExpression.ts

+/*
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements.  See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership.  The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License.  You may obtain a copy of the License at
+*
+*   http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing,
+* software distributed under the License is distributed on an
+* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+* KIND, either express or implied.  See the License for the
+* specific language governing permissions and limitations
+* under the License.
+*/
+
+import { OptionDataValue, DimensionLoose, Dictionary } from './types';
+import {
+    createHashMap, keys, isArray, map, isObject, isString, trim, HashMap, isRegExp, isArrayLike
+} from 'zrender/src/core/util';
+import { throwError, makePrintable } from './log';
+import { parseDate } from './number';
+
+
+// PENDING:
+// (1) Support more parser like: `parse: 'trim'`, `parse: 'lowerCase'`, `parse: 'year'`, `parse: 'dayOfWeek'`?
+// (2) Support piped parser ?
+// (3) Support callback parser or callback condition?
+// (4) At present do not support string expression yet but only stuctured expression.
+
+
+/**
+ * The structured expression considered:
+ * (1) Literal simplicity
+ * (2) Sementic displayed clearly
+ *
+ * Sementic supports:
+ * (1) relational expression
+ * (2) logical expression
+ *
+ * For example:
+ * ```js
+ * {
+ *     and: [{
+ *         or: [{
+ *             dimension: 'Year', gt: 2012, lt: 2019
+ *         }, {
+ *             dimension: 'Year', '>': 2002, '<=': 2009
+ *         }]
+ *     }, {
+ *         dimension: 'Product', eq: 'Tofu'
+ *     }]
+ * }
+ *
+ * { dimension: 'Product', eq: 'Tofu' }
+ *
+ * {
+ *     or: [
+ *         { dimension: 'Product', value: 'Tofu' },
+ *         { dimension: 'Product', value: 'Biscuit' }
+ *     ]
+ * }
+ *
+ * {
+ *     and: [true]
+ * }
+ * ```
+ *
+ * [PARSER]
+ * In an relation expression object, we can specify some built-in parsers:
+ * ```js
+ * // Trim if string
+ * {
+ *     parse: 'trim',
+ *     eq: 'Flowers'
+ * }
+ * // Parse as time and enable arithmetic relation comparison.
+ * {
+ *     parse: 'time',
+ *     lt: '2012-12-12'
+ * }
+ * // RegExp, include the feature in SQL: `like '%xxx%'`.
+ * {
+ *     reg: /^asdf$/
+ * }
+ * {
+ *     reg: '^asdf$' // Serializable reg exp, will be `new RegExp(...)`
+ * }
+ * ```
+ *
+ *
+ * [EMPTY_RULE]
+ * (1) If a relational expression set value as `null`/`undefined` like:
+ * `{ dimension: 'Product', lt: undefined }`,
+ * The result will be `false` rather than `true`.
+ * Consider the case like "filter condition", return all result when null/undefined
+ * is probably not expected and even dangours.
+ * (2) If a relational expression has no operator like:
+ * `{ dimension: 'Product' }`,
+ * An error will be thrown. Because it is probably a mistake.
+ * (3) If a logical expression has no children like
+ * `{ and: undefined }` or `{ and: [] }`,
+ * An error will be thrown. Because it is probably an mistake.
+ * (4) If intending have a condition that always `true` or always `false`,
+ * Use `true` or `flase`.
+ * The entire condition can be `true`/`false`,
+ * or also can be `{ and: [true] }`, `{ or: [false] }`
+ */
+
+
+// --------------------------------------------------
+// --- Relational Expression --------------------------
+// --------------------------------------------------
+
+/**
+ * Date string and ordinal string can be accepted.
+ */
+interface RelationalExpressionOptionByOp {
+    lt?: OptionDataValue; // less than
+    lte?: OptionDataValue; // less than or equal
+    gt?: OptionDataValue; // greater than
+    gte?: OptionDataValue; // greater than or equal
+    eq?: OptionDataValue; // equal
+    ne?: OptionDataValue; // not equal
+    reg?: RegExp | string; // RegExp
+};
+interface RelationalExpressionOptionByOpAlias {
+    value?: RelationalExpressionOptionByOp['eq'];
+
+    '<'?: OptionDataValue; // lt
+    '<='?: OptionDataValue; // lte
+    '>'?: OptionDataValue; // gt
+    '>='?: OptionDataValue; // gte
+    '='?: OptionDataValue; // eq
+    '!='?: OptionDataValue; // ne
+    '<>'?: OptionDataValue; // ne (SQL style)
+
+    // '=='?: OptionDataValue; // eq
+    // '==='?: OptionDataValue; // eq
+    // '!=='?: OptionDataValue; // eq
+
+    // ge: RelationalExpressionOptionByOp['gte'];
+    // le: RelationalExpressionOptionByOp['lte'];
+    // neq: RelationalExpressionOptionByOp['ne'];
+};
+const aliasToOpMap = createHashMap<RelationalExpressionOp, RelationalExpressionOpAlias>({
+    value: 'eq',
+
+    // PENDING: not good for literal semantic?
+    '<': 'lt',
+    '<=': 'lte',
+    '>': 'gt',
+    '>=': 'gte',
+    '=': 'eq',
+    '!=': 'ne',
+    '<>': 'ne'
+
+    // Might mileading for sake of the different between '==' and '===',
+    // So dont support them.
+    // '==': 'eq',
+    // '===': 'seq',
+    // '!==': 'sne'
+
+    // PENDING: Whether support some common alias "ge", "le", "neq"?
+    // ge: 'gte',
+    // le: 'lte',
+    // neq: 'ne',
+});
+
+type RelationalExpressionOp = keyof RelationalExpressionOptionByOp;
+type RelationalExpressionOpAlias = keyof RelationalExpressionOptionByOpAlias;
+
+interface RelationalExpressionOption extends
+        RelationalExpressionOptionByOp, RelationalExpressionOptionByOpAlias {
+    dimension?: DimensionLoose;
+    parse?: RelationalExpressionValueParserType;
+}
+
+type RelationalExpressionOpEvaluate = (tarVal: unknown, condVal: unknown) => boolean;
+
+const relationalOpEvaluateMap = createHashMap<RelationalExpressionOpEvaluate, RelationalExpressionOp>({
+    // PENDING: should keep supporting string compare?
+    lt: function (tarVal, condVal) {
+        return tarVal < condVal;
+    },
+    lte: function (tarVal, condVal) {
+        return tarVal <= condVal;
+    },
+    gt: function (tarVal, condVal) {
+        return tarVal > condVal;
+    },
+    gte: function (tarVal, condVal) {
+        return tarVal >= condVal;
+    },
+    eq: function (tarVal, condVal) {
+        // eq is probably most used, DO NOT use JS ==,
+        // the rule is too complicated.
+        return tarVal === condVal;
+    },
+    ne: function (tarVal, condVal) {
+        return tarVal !== condVal;
+    },
+    reg: function (tarVal, condVal: RegExp) {
+        const type = typeof tarVal;
+        return type === 'string' ? condVal.test(tarVal as string)
+            : type === 'number' ? condVal.test(tarVal + '')
+            : false;
+    }
+});
+
+function parseRegCond(condVal: unknown): RegExp {
+    // Support condVal: RegExp | string
+    return isString(condVal) ? new RegExp(condVal)
+        : isRegExp(condVal) ? condVal as RegExp
+        : null;
+}
+
+type RelationalExpressionValueParserType = 'time' | 'trim';
+type RelationalExpressionValueParser = (val: unknown) => unknown;
+const valueParserMap = createHashMap<RelationalExpressionValueParser, RelationalExpressionValueParserType>({
+    time: function (val): number {
+        // return timestamp.
+        return +parseDate(val);
+    },
+    trim: function (val) {
+        return typeof val === 'string' ? trim(val) : val;
+    }
+});
+
+
+// --------------------------------------------------
+// --- Logical Expression ---------------------------
+// --------------------------------------------------
+
+
+interface LogicalExpressionOption {
+    and?: LogicalExpressionSubOption[];
+    or?: LogicalExpressionSubOption[];
+    not?: LogicalExpressionSubOption;
+}
+type LogicalExpressionSubOption =
+    LogicalExpressionOption | RelationalExpressionOption | TrueFalseExpressionOption;
+
+
+
+// -----------------------------------------------------
+// --- Conditional Expression --------------------------
+// -----------------------------------------------------
+
+
+export type TrueExpressionOption = true;
+export type FalseExpressionOption = false;
+export type TrueFalseExpressionOption = TrueExpressionOption | FalseExpressionOption;
+
+export type ConditionalExpressionOption =
+    LogicalExpressionOption
+    | RelationalExpressionOption
+    | TrueFalseExpressionOption;
+
+type ValueGetterParam = Dictionary<unknown>;
+export interface ConditionalExpressionValueGetterParamGetter<VGP extends ValueGetterParam = ValueGetterParam> {
+    (relExpOption: RelationalExpressionOption): VGP
+}
+export interface ConditionalExpressionValueGetter<VGP extends ValueGetterParam = ValueGetterParam> {
+    (param: VGP): OptionDataValue
+}
+
+interface ParsedConditionInternal {
+    evaluate(): boolean;
+}
+class ConstConditionInternal implements ParsedConditionInternal {
+    value: boolean;
+    evaluate(): boolean {
+        return this.value;
+    }
+}
+class AndConditionInternal implements ParsedConditionInternal {
+    children: ParsedConditionInternal[];
+    evaluate() {
+        const children = this.children;
+        for (let i = 0; i < children.length; i++) {
+            if (!children[i].evaluate()) {
+                return false;
+            }
+        }
+        return true;
+    }
+}
+class OrConditionInternal implements ParsedConditionInternal {
+    children: ParsedConditionInternal[];
+    evaluate() {
+        const children = this.children;
+        for (let i = 0; i < children.length; i++) {
+            if (children[i].evaluate()) {
+                return true;
+            }
+        }
+        return false;
+    }
+}
+class NotConditionInternal implements ParsedConditionInternal {
+    child: ParsedConditionInternal;
+    evaluate() {
+        return !this.child.evaluate();
+    }
+}
+class RelationalConditionInternal implements ParsedConditionInternal {
+    valueGetterParam: ValueGetterParam;
+    valueParser: RelationalExpressionValueParser;
+    // If no parser, be null/undefined.
+    getValue: ConditionalExpressionValueGetter;
+    subCondList: {
+        condValue: unknown;
+        evaluate: RelationalExpressionOpEvaluate;
+    }[];
+
+    evaluate() {
+        const getValue = this.getValue;
+        const needParse = !!this.valueParser;
+        // Call getValue with no `this`.
+        const tarValRaw = getValue(this.valueGetterParam);
+        const tarValParsed = needParse ? this.valueParser(tarValRaw) : null;
+
+        // Relational cond follow "and" logic internally.
+        for (let i = 0; i < this.subCondList.length; i++) {
+            const subCond = this.subCondList[i];
+            if (
+                !subCond.evaluate(
+                    needParse ? tarValParsed : tarValRaw,
+                    subCond.condValue
+                )
+            ) {
+                return false;
+            }
+        }
+        return true;
+    }
+}
+
+function parseOption(
+    exprOption: ConditionalExpressionOption,
+    getters: ConditionalGetters
+): ParsedConditionInternal {
+    if (exprOption === true || exprOption === false) {
+        const cond = new ConstConditionInternal();
+        cond.value = exprOption as boolean;
+        return cond;
+    }
+
+    let errMsg = '';
+    if (!isObjectNotArray(exprOption)) {
+        if (__DEV__) {
+            errMsg = makePrintable(
+                'Illegal config. Expect a plain object but actually', exprOption
+            );
+        }
+        throwError(errMsg);
+    }
+
+    if ((exprOption as LogicalExpressionOption).and) {
+        return parseAndOrOption('and', exprOption as LogicalExpressionOption, getters);
+    }
+    else if ((exprOption as LogicalExpressionOption).or) {
+        return parseAndOrOption('or', exprOption as LogicalExpressionOption, getters);
+    }
+    else if ((exprOption as LogicalExpressionOption).not) {
+        return parseNotOption(exprOption as LogicalExpressionOption, getters);
+    }
+
+    return parseRelationalOption(exprOption as RelationalExpressionOption, getters);
+}
+
+function parseAndOrOption(
+    op: 'and' | 'or',
+    exprOption: LogicalExpressionOption,
+    getters: ConditionalGetters
+): ParsedConditionInternal {
+    const subOptionArr = exprOption[op] as ConditionalExpressionOption[];
+    let errMsg = '';
+    if (__DEV__) {
+        errMsg = makePrintable(
+            '"and"/"or" condition should only be `' + op + ': [...]` and must not be empty array.',
+            'Illegal condition:', exprOption
+        );
+    }
+    if (!isArray(subOptionArr)) {
+        throwError(errMsg);
+    }
+    if (!(subOptionArr as []).length) {
+        throwError(errMsg);
+    }
+    const cond = op === 'and' ? new AndConditionInternal() : new OrConditionInternal();
+    cond.children = map(subOptionArr, subOption => parseOption(subOption, getters));
+    if (!cond.children.length) {
+        throwError(errMsg);
+    }
+    return cond;
+}
+
+function parseNotOption(
+    exprOption: LogicalExpressionOption,
+    getters: ConditionalGetters
+): ParsedConditionInternal {
+    const subOption = exprOption.not as ConditionalExpressionOption;
+    let errMsg = '';
+    if (__DEV__) {
+        errMsg = makePrintable(
+            '"not" condition should only be `not: {}`.',
+            'Illegal condition:', exprOption
+        );
+    }
+    if (!isObjectNotArray(subOption)) {
+        throwError(errMsg);
+    }
+    const cond = new NotConditionInternal();
+    cond.child = parseOption(subOption, getters);
+    if (!cond.child) {
+        throwError(errMsg);
+    }
+    return cond;
+}
+
+function parseRelationalOption(
+    exprOption: RelationalExpressionOption,
+    getters: ConditionalGetters
+): ParsedConditionInternal {
+    let errMsg = '';
+
+    const valueGetterParam = getters.prepareGetValue(exprOption);
+
+    const subCondList = [] as RelationalConditionInternal['subCondList'];
+    const exprKeys = keys(exprOption);
+
+    const parserName = exprOption.parse;
+    const valueParser = parserName ? valueParserMap.get(parserName) : null;
+
+    for (let i = 0; i < exprKeys.length; i++) {
+        const keyRaw = exprKeys[i];
+        if (keyRaw === 'parse' || getters.valueGetterAttrMap.get(keyRaw)) {
+            continue;
+        }
+
+        const op: RelationalExpressionOp = aliasToOpMap.get(keyRaw as RelationalExpressionOpAlias)
+            || (keyRaw as RelationalExpressionOp);
+        const evaluateHandler = relationalOpEvaluateMap.get(op);
+
+        if (!evaluateHandler) {
+            if (__DEV__) {
+                errMsg = makePrintable(
+                    'Illegal relational operation: "' + keyRaw + '" in condition:', exprOption
+                );
+            }
+            throwError(errMsg);
+        }
+
+        const condValueRaw = exprOption[keyRaw];
+        let condValue;
+        if (keyRaw === 'reg') {
+            condValue = parseRegCond(condValueRaw);
+            if (condValue == null) {
+                let errMsg = '';
+                if (__DEV__) {
+                    errMsg = makePrintable('Illegal regexp', condValueRaw, 'in', exprOption);
+                }
+                throwError(errMsg);
+            }
+        }
+        else {
+            // At present, all other operators are applicable `RelationalExpressionValueParserType`.
+            // But if adding new parser, we should check it again.
+            condValue = valueParser ? valueParser(condValueRaw) : condValueRaw;
+        }
+
+        subCondList.push({
+            condValue: condValue,
+            evaluate: evaluateHandler
+        });
+    }
+
+    if (!subCondList.length) {
+        if (__DEV__) {
+            errMsg = makePrintable(
+                'Relational condition must have at least one operator.',
+                'Illegal condition:', exprOption
+            );
+        }
+        // No relational operator always disabled in case of dangers result.
+        throwError(errMsg);
+    }
+
+    const cond = new RelationalConditionInternal();
+    cond.valueGetterParam = valueGetterParam;
+    cond.valueParser = valueParser;
+    cond.getValue = getters.getValue;
+    cond.subCondList = subCondList;
+
+    return cond;
+}
+
+function isObjectNotArray(val: unknown): boolean {
+    return isObject(val) && !isArrayLike(val);
+}
+
+
+class ConditionalExpressionParsed {
+
+    private _cond: ParsedConditionInternal;
+
+    constructor(
+        exprOption: ConditionalExpressionOption,
+        getters: ConditionalGetters
+    ) {
+        this._cond = parseOption(exprOption, getters);
+    }
+
+    evaluate(): boolean {
+        return this._cond.evaluate();
+    }
+};
+
+interface ConditionalGetters<VGP extends ValueGetterParam = ValueGetterParam> {
+    prepareGetValue: ConditionalExpressionValueGetterParamGetter<VGP>;
+    getValue: ConditionalExpressionValueGetter<VGP>;
+    valueGetterAttrMap: HashMap<boolean, string>;
+}
+
+export function parseConditionalExpression<VGP extends ValueGetterParam = ValueGetterParam>(
+    exprOption: ConditionalExpressionOption,
+    getters: ConditionalGetters<VGP>
+): ConditionalExpressionParsed {
+    return new ConditionalExpressionParsed(exprOption, getters);
+}
+

src/util/ecData.ts

@@ -18,7 +18,7 @@
 */
 
 import Element from 'zrender/src/Element';
-import { DataModel, ECEventData, BlurScope, InnerFocus } from './types';
+import { DataModel, ECEventData, BlurScope, InnerFocus, SeriesDataType } from './types';
 import { makeInner } from './model';
 /**
  * ECData stored on graphic element
@@ -28,7 +28,7 @@ export interface ECData {
     dataModel?: DataModel;
     eventData?: ECEventData;
     seriesIndex?: number;
-    dataType?: string;
+    dataType?: SeriesDataType;
     focus?: InnerFocus;
     blurScope?: BlurScope;
 }

src/util/log.ts

@@ -18,6 +18,7 @@
 */
 
 import { Dictionary } from './types';
+import { map, isString, isFunction, eqNaN, isRegExp } from 'zrender/src/core/util';
 
 const storedLogs: Dictionary<boolean> = {};
 
@@ -38,3 +39,67 @@ export function deprecateReplaceLog(oldOpt: string, newOpt: string, scope?: stri
         deprecateLog((scope ? `[${scope}]` : '') + `${oldOpt} is deprecated, use ${newOpt} instead.`);
     }
 }
+
+export function consoleLog(...args: unknown[]) {
+    if (__DEV__) {
+        /* eslint-disable no-console */
+        if (typeof console !== 'undefined' && console.log) {
+            console.log.apply(console, args);
+        }
+        /* eslint-enable no-console */
+    }
+}
+
+/**
+ * If in __DEV__ environment, get console printable message for users hint.
+ * Parameters are separated by ' '.
+ * @usuage
+ * makePrintable('This is an error on', someVar, someObj);
+ *
+ * @param hintInfo anything about the current execution context to hint users.
+ * @throws Error
+ */
+export function makePrintable(...hintInfo: unknown[]) {
+    let msg = '';
+
+    if (__DEV__) {
+        // Fuzzy stringify for print.
+        // This code only exist in dev environment.
+        msg = map(hintInfo, arg => {
+            if (isString(arg)) {
+                // Print without quotation mark for some statement.
+                return arg;
+            }
+            else if (typeof JSON !== 'undefined' && JSON.stringify) {
+                try {
+                    return JSON.stringify(arg, function (n, val) {
+                        return val === void 0 ? 'undefined'
+                            : val === Infinity ? 'Infinity'
+                            : val === -Infinity ? '-Infinity'
+                            : eqNaN(val) ? 'NaN'
+                            : val instanceof Date ? 'Date(' + val.toISOString() + ')'
+                            : isFunction(val) ? 'function () { ... }'
+                            : isRegExp(val) ? val + ''
+                            : val;
+                    });
+                    // In most cases the info object is small, so do not line break.
+                }
+                catch (err) {
+                    return '?';
+                }
+            }
+            else {
+                return '?';
+            }
+        }).join(' ');
+    }
+
+    return msg;
+}
+
+/**
+ * @throws Error
+ */
+export function throwError(msg?: string) {
+    throw new Error(msg);
+}

src/util/model.ts

@@ -791,7 +791,7 @@ export function parseFinder(
     }
 
     const defaultMainType = opt ? opt.defaultMainType : null;
-    const queryOptionMap = createHashMap<QueryReferringOption, ComponentMainType>();
+    const queryOptionMap = createHashMap<QueryReferringUserOption, ComponentMainType>();
     const result = {} as ParsedModelFinder;
 
     each(finder, function (value, key) {
@@ -803,7 +803,7 @@ export function parseFinder(
 
         const parsedKey = key.match(/^(\w+)(Index|Id|Name)$/) || [];
         const mainType = parsedKey[1];
-        const queryType = (parsedKey[2] || '').toLowerCase() as keyof QueryReferringOption;
+        const queryType = (parsedKey[2] || '').toLowerCase() as keyof QueryReferringUserOption;
 
         if (
             !mainType
@@ -836,7 +836,7 @@ export function parseFinder(
     return result;
 }
 
-type QueryReferringOption = {
+export type QueryReferringUserOption = {
     index?: ModelFinderIndexQuery,
     id?: ModelFinderIdQuery,
     name?: ModelFinderNameQuery,
@@ -857,7 +857,7 @@ export type QueryReferringOpt = {
 export function queryReferringComponents(
     ecModel: GlobalModel,
     mainType: ComponentMainType,
-    userOption: QueryReferringOption,
+    userOption: QueryReferringUserOption,
     opt: QueryReferringOpt
 ): {
     // Always be array rather than null/undefined, which is convenient to use.

src/util/number.ts

@@ -286,7 +286,8 @@ export function isRadianAroundZero(val: number): boolean {
 const TIME_REG = /^(?:(\d{4})(?:[-\/](\d{1,2})(?:[-\/](\d{1,2})(?:[T ](\d{1,2})(?::(\d{1,2})(?::(\d{1,2})(?:[.,](\d+))?)?)?(Z|[\+\-]\d\d:?\d\d)?)?)?)?)?$/; // jshint ignore:line
 
 /**
- * @param value These values can be accepted:
+ * @param value valid type: number | string | Date, otherwise return `new Date(NaN)`
+ *   These values can be accepted:
  *   + An instance of Date, represent a time in its own time zone.
  *   + Or string in a subset of ISO 8601, only including:
  *     + only year, month, date: '2012-03', '2012-03-01', '2012-03-01 05', '2012-03-01 05:06',
@@ -298,9 +299,9 @@ const TIME_REG = /^(?:(\d{4})(?:[-\/](\d{1,2})(?:[-\/](\d{1,2})(?:[T ](\d{1,2})(
  *     '2012', '2012-3-1', '2012/3/1', '2012/03/01',
  *     '2009/6/12 2:00', '2009/6/12 2:05:08', '2009/6/12 2:05:08.123'
  *   + a timestamp, which represent a time in UTC.
- * @return date
+ * @return date Never be null/undefined. If invalid, return `new Date(NaN)`.
  */
-export function parseDate(value: number | string | Date): Date {
+export function parseDate(value: unknown): Date {
     if (value instanceof Date) {
         return value;
     }
@@ -358,7 +359,7 @@ export function parseDate(value: number | string | Date): Date {
         return new Date(NaN);
     }
 
-    return new Date(Math.round(value));
+    return new Date(Math.round(value as number));
 }
 
 /**

src/util/types.ts

@@ -372,7 +372,9 @@ export const SERIES_LAYOUT_BY_ROW = 'row' as const;
 
 export type SeriesLayoutBy = typeof SERIES_LAYOUT_BY_COLUMN | typeof SERIES_LAYOUT_BY_ROW;
 // null/undefined/'auto': auto detect header, see "src/data/helper/sourceHelper".
-export type OptionSourceHeader = boolean | 'auto';
+// If number, means header lines count, or say, `startIndex`.
+// Like `sourceHeader: 2`, means line 0 and line 1 are header, data start from line 2.
+export type OptionSourceHeader = boolean | 'auto' | number;
 
 export type SeriesDataType = 'main' | 'node' | 'edge';
 
@@ -1395,8 +1397,11 @@ export interface SeriesSamplingOptionMixin {
 
 export interface SeriesEncodeOptionMixin {
     datasetIndex?: number;
+    datasetId?: string | number;
     seriesLayoutBy?: SeriesLayoutBy;
     sourceHeader?: OptionSourceHeader;
     dimensions?: DimensionDefinitionLoose[];
     encode?: OptionEncode
 }
+
+export type SeriesEncodableModel = SeriesModel<SeriesOption & SeriesEncodeOptionMixin>;