From e9a2b0f5aee017be1bf41726e6531c38530ce28e Mon Sep 17 00:00:00 2001 From: 100pah Date: Fri, 31 Jul 2020 13:02:59 +0800 Subject: [PATCH] feature: data transform --- src/chart/helper/createListFromArray.ts | 9 +- src/chart/parallel/ParallelSeries.ts | 30 +- src/component/dataset.ts | 46 +- src/component/transform.ts | 25 + src/component/transform/filterTransform.ts | 101 ++++ src/component/transform/sortTransform.ts | 223 +++++++ src/component/visualMap/PiecewiseModel.ts | 1 + src/data/List.ts | 45 +- src/data/Source.ts | 32 +- src/data/helper/dataProvider.ts | 267 ++++++--- src/data/helper/parseDataValue.ts | 73 +++ src/data/helper/sourceHelper.ts | 219 +++---- src/data/helper/sourceManager.ts | 368 ++++++++++++ src/data/helper/transform.ts | 317 ++++++++++ src/echarts.ts | 3 + src/model/Component.ts | 4 +- src/model/OptionManager.ts | 4 + src/model/Series.ts | 16 +- src/util/conditionalExpression.ts | 536 +++++++++++++++++ src/util/ecData.ts | 4 +- src/util/log.ts | 67 ++- src/util/model.ts | 8 +- src/util/number.ts | 9 +- src/util/types.ts | 7 +- test/data-transform.html | 661 +++++++++++++++++++++ 25 files changed, 2779 insertions(+), 296 deletions(-) create mode 100644 src/component/transform.ts create mode 100644 src/component/transform/filterTransform.ts create mode 100644 src/component/transform/sortTransform.ts create mode 100644 src/data/helper/parseDataValue.ts create mode 100644 src/data/helper/sourceManager.ts create mode 100644 src/data/helper/transform.ts create mode 100644 src/util/conditionalExpression.ts create mode 100644 test/data-transform.html diff --git a/src/chart/helper/createListFromArray.ts b/src/chart/helper/createListFromArray.ts index e451cd3f6..c12ac24e5 100644 --- a/src/chart/helper/createListFromArray.ts +++ b/src/chart/helper/createListFromArray.ts @@ -28,13 +28,13 @@ import Source from '../../data/Source'; import {enableDataStack} from '../../data/helper/dataStackHelper'; import {makeSeriesEncodeForAxisCoordSys} from '../../data/helper/sourceHelper'; import { - SOURCE_FORMAT_ORIGINAL, DimensionDefinitionLoose, DimensionDefinition, OptionSourceData + SOURCE_FORMAT_ORIGINAL, DimensionDefinitionLoose, DimensionDefinition, OptionSourceData, EncodeDefaulter } from '../../util/types'; import SeriesModel from '../../model/Series'; function createListFromArray(source: Source | OptionSourceData, seriesModel: SeriesModel, opt?: { generateCoord?: string - useEncodeDefaulter?: boolean + useEncodeDefaulter?: boolean | EncodeDefaulter }): List { opt = opt || {}; @@ -73,10 +73,13 @@ function createListFromArray(source: Source | OptionSourceData, seriesModel: Ser )) || ['x', 'y']; } + const useEncodeDefaulter = opt.useEncodeDefaulter; const dimInfoList = createDimensions(source, { coordDimensions: coordSysDimDefs, generateCoord: opt.generateCoord, - encodeDefaulter: opt.useEncodeDefaulter + encodeDefaulter: zrUtil.isFunction(useEncodeDefaulter) + ? useEncodeDefaulter + : useEncodeDefaulter ? zrUtil.curry(makeSeriesEncodeForAxisCoordSys, coordSysDimDefs, seriesModel) : null }); diff --git a/src/chart/parallel/ParallelSeries.ts b/src/chart/parallel/ParallelSeries.ts index d0b5a38fb..287bc2fac 100644 --- a/src/chart/parallel/ParallelSeries.ts +++ b/src/chart/parallel/ParallelSeries.ts @@ -18,7 +18,7 @@ */ -import {each, createHashMap} from 'zrender/src/core/util'; +import {each, bind} from 'zrender/src/core/util'; import SeriesModel from '../../model/Series'; import createListFromArray from '../helper/createListFromArray'; import { @@ -29,13 +29,15 @@ import { SeriesTooltipOption, DimensionName, OptionDataValue, - StatesOptionMixin + StatesOptionMixin, + OptionEncodeValue, + Dictionary, + OptionEncode } from '../../util/types'; import GlobalModel from '../../model/Global'; import List from '../../data/List'; import { ParallelActiveState, ParallelAxisOption } from '../../coord/parallel/AxisModel'; import Parallel from '../../coord/parallel/Parallel'; -import Source from '../../data/Source'; import ParallelModel from '../../coord/parallel/ParallelModel'; type ParallelSeriesDataValue = OptionDataValue[]; @@ -89,12 +91,10 @@ class ParallelSeriesModel extends SeriesModel { coordinateSystem: Parallel; - getInitialData(option: ParallelSeriesOption, ecModel: GlobalModel): List { - const source = this.getSource(); - - setEncodeAndDimensions(source, this); - - return createListFromArray(source, this); + getInitialData(this: ParallelSeriesModel, option: ParallelSeriesOption, ecModel: GlobalModel): List { + return createListFromArray(this.getSource(), this, { + useEncodeDefaulter: bind(makeDefaultEncode, null, this) + }); } /** @@ -151,7 +151,7 @@ class ParallelSeriesModel extends SeriesModel { SeriesModel.registerClass(ParallelSeriesModel); -function setEncodeAndDimensions(source: Source, seriesModel: ParallelSeriesModel): void { +function makeDefaultEncode(seriesModel: ParallelSeriesModel): OptionEncode { // The mapping of parallelAxis dimension to data dimension can // be specified in parallelAxis.option.dim. For example, if // parallelAxis.option.dim is 'dim3', it mapping to the third @@ -159,10 +159,6 @@ function setEncodeAndDimensions(source: Source, seriesModel: ParallelSeriesModel // Moreover, parallelModel.dimension should not be regarded as data // dimensions. Consider dimensions = ['dim4', 'dim2', 'dim6']; - if (source.encodeDefine) { - return; - } - const parallelModel = seriesModel.ecModel.getComponent( 'parallel', seriesModel.get('parallelIndex') ) as ParallelModel; @@ -170,11 +166,13 @@ function setEncodeAndDimensions(source: Source, seriesModel: ParallelSeriesModel return; } - const encodeDefine = source.encodeDefine = createHashMap(); + const encodeDefine: Dictionary = {}; each(parallelModel.dimensions, function (axisDim) { const dataDimIndex = convertDimNameToNumber(axisDim); - encodeDefine.set(axisDim, dataDimIndex); + encodeDefine[axisDim] = dataDimIndex; }); + + return encodeDefine; } function convertDimNameToNumber(dimName: DimensionName): number { diff --git a/src/component/dataset.ts b/src/component/dataset.ts index 8e16f47a8..26b77ec00 100644 --- a/src/component/dataset.ts +++ b/src/component/dataset.ts @@ -28,22 +28,33 @@ import ComponentModel from '../model/Component'; import ComponentView from '../view/Component'; -import {detectSourceFormat} from '../data/helper/sourceHelper'; import { - SERIES_LAYOUT_BY_COLUMN, ComponentOption, SeriesEncodeOptionMixin, OptionSourceData, SeriesLayoutBy + SERIES_LAYOUT_BY_COLUMN, ComponentOption, SeriesEncodeOptionMixin, + OptionSourceData, SeriesLayoutBy, OptionSourceHeader } from '../util/types'; +import { DataTransformOption, PipedDataTransformOption } from '../data/helper/transform'; +import GlobalModel from '../model/Global'; +import Model from '../model/Model'; +import { disableTransformOptionMerge, SourceManager } from '../data/helper/sourceManager'; -interface DatasetOption extends +export interface DatasetOption extends Pick, Pick { seriesLayoutBy?: SeriesLayoutBy; - // null/undefined/'auto': auto detect header, see "src/data/helper/sourceHelper". - sourceHeader?: boolean | 'auto'; - data?: OptionSourceData; + sourceHeader?: OptionSourceHeader; + source?: OptionSourceData; + + fromDatasetIndex?: number; + fromDatasetId?: string; + transform?: DataTransformOption | PipedDataTransformOption; + // When a transform result more than on results, the results can be referenced only by: + // Using `fromDatasetIndex`/`fromDatasetId` and `transfromResultIndex` to retrieve + // the results from other dataset. + fromTransformResult?: number; } -class DatasetModel extends ComponentModel { +export class DatasetModel extends ComponentModel { type = 'dataset'; static type = 'dataset'; @@ -52,18 +63,33 @@ class DatasetModel extends ComponentModel { seriesLayoutBy: SERIES_LAYOUT_BY_COLUMN }; + private _sourceManager: SourceManager; + + init(option: Opts, parentModel: Model, ecModel: GlobalModel): void { + super.init(option, parentModel, ecModel); + this._sourceManager = new SourceManager(this); + disableTransformOptionMerge(this); + } + + mergeOption(newOption: Opts, ecModel: GlobalModel): void { + super.mergeOption(newOption, ecModel); + disableTransformOptionMerge(this); + } + optionUpdated() { - detectSourceFormat(this); + this._sourceManager.dirty(); + } + + getSourceManager() { + return this._sourceManager; } } ComponentModel.registerClass(DatasetModel); - class DatasetView extends ComponentView { static type = 'dataset'; type = 'dataset'; } ComponentView.registerClass(DatasetView); - diff --git a/src/component/transform.ts b/src/component/transform.ts new file mode 100644 index 000000000..d679f8c31 --- /dev/null +++ b/src/component/transform.ts @@ -0,0 +1,25 @@ +/* +* Licensed to the Apache Software Foundation (ASF) under one +* or more contributor license agreements. See the NOTICE file +* distributed with this work for additional information +* regarding copyright ownership. The ASF licenses this file +* to you under the Apache License, Version 2.0 (the +* "License"); you may not use this file except in compliance +* with the License. You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an +* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +* KIND, either express or implied. See the License for the +* specific language governing permissions and limitations +* under the License. +*/ + +import * as echarts from '../echarts'; +import {filterTransform} from './transform/filterTransform'; +import {sortTransform} from './transform/sortTransform'; + +echarts.registerTransform(filterTransform); +echarts.registerTransform(sortTransform); diff --git a/src/component/transform/filterTransform.ts b/src/component/transform/filterTransform.ts new file mode 100644 index 000000000..4b13008c5 --- /dev/null +++ b/src/component/transform/filterTransform.ts @@ -0,0 +1,101 @@ +/* +* Licensed to the Apache Software Foundation (ASF) under one +* or more contributor license agreements. See the NOTICE file +* distributed with this work for additional information +* regarding copyright ownership. The ASF licenses this file +* to you under the Apache License, Version 2.0 (the +* "License"); you may not use this file except in compliance +* with the License. You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an +* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +* KIND, either express or implied. See the License for the +* specific language governing permissions and limitations +* under the License. +*/ + +import { DataTransformOption, ExternalDataTransform } from '../../data/helper/transform'; +import { DimensionIndex, OptionDataItem } from '../../util/types'; +import { parseConditionalExpression, ConditionalExpressionOption } from '../../util/conditionalExpression'; +import { hasOwn, createHashMap } from 'zrender/src/core/util'; +import { makePrintable, throwError } from '../../util/log'; + + +export interface FilterTransformOption extends DataTransformOption { + type: 'filter'; + config: ConditionalExpressionOption; +} + +export const filterTransform: ExternalDataTransform = { + + type: 'echarts:filter', + + // PEDING: enhance to filter by index rather than create new data + transform: function transform(params) { + // [Caveat] Fail-Fast: + // Do not return the whole dataset unless user config indicate it explicitly. + // For example, if no condition specified by mistake, return an empty result + // is better than return the entire raw soruce for user to find the mistake. + + const source = params.source; + let rawItem: OptionDataItem; + + const condition = parseConditionalExpression<{ dimIdx: DimensionIndex }>(params.config, { + + valueGetterAttrMap: createHashMap({ dimension: true }), + + prepareGetValue: function (exprOption) { + let errMsg = ''; + const dimLoose = exprOption.dimension; + if (!hasOwn(exprOption, 'dimension')) { + if (__DEV__) { + errMsg = makePrintable( + 'Relation condition must has prop "dimension" specified.', + 'Illegal condition:', exprOption + ); + } + throwError(errMsg); + } + + const dimInfo = source.getDimensionInfo(dimLoose); + if (!dimInfo) { + if (__DEV__) { + errMsg = makePrintable( + 'Can not find dimension info via: "' + dimLoose + '".\n', + 'Existing dimensions: ', source.dimensions, '.\n', + 'Illegal condition:', exprOption, '.\n' + ); + } + throwError(errMsg); + } + + return { dimIdx: dimInfo.index }; + }, + + getValue: function (param) { + return source.retrieveItemValue(rawItem, param.dimIdx); + } + }); + + const sourceHeaderCount = source.sourceHeaderCount; + const resultData = []; + for (let i = 0; i < sourceHeaderCount; i++) { + resultData.push(source.getRawHeaderItem(i)); + } + for (let i = 0, len = source.count(); i < len; i++) { + rawItem = source.getRawDataItem(i); + if (condition.evaluate()) { + resultData.push(rawItem); + } + } + + return { + data: resultData, + dimensions: source.dimensions, + sourceHeader: sourceHeaderCount + }; + } +}; diff --git a/src/component/transform/sortTransform.ts b/src/component/transform/sortTransform.ts new file mode 100644 index 000000000..9dfce973b --- /dev/null +++ b/src/component/transform/sortTransform.ts @@ -0,0 +1,223 @@ +/* +* Licensed to the Apache Software Foundation (ASF) under one +* or more contributor license agreements. See the NOTICE file +* distributed with this work for additional information +* regarding copyright ownership. The ASF licenses this file +* to you under the Apache License, Version 2.0 (the +* "License"); you may not use this file except in compliance +* with the License. You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an +* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +* KIND, either express or implied. See the License for the +* specific language governing permissions and limitations +* under the License. +*/ + +import { DataTransformOption, ExternalDataTransform } from '../../data/helper/transform'; +import { + DimensionLoose, SOURCE_FORMAT_KEYED_COLUMNS, DimensionIndex, OptionDataValue +} from '../../util/types'; +import { makePrintable, throwError } from '../../util/log'; +import { isArray, each, hasOwn } from 'zrender/src/core/util'; +import { normalizeToArray } from '../../util/model'; +import { parseDate } from '../../util/number'; + +/** + * @usage + * + * ```js + * transform: { + * type: 'sort', + * config: { dimension: 'score', order: 'asc' } + * } + * transform: { + * type: 'sort', + * config: [ + * { dimension: 1, order: 'asc' }, + * { dimension: 'age', order: 'desc' } + * ] + * } + * ``` + */ + +export interface SortTransformOption extends DataTransformOption { + type: 'sort'; + config: OrderExpression | OrderExpression[]; +} + +// PENDING: whether support { dimension: 'score', order: 'asc' } ? +type OrderExpression = { + dimension: DimensionLoose; + order: SortOrder; + parse?: 'time' +}; + +type SortOrder = 'asc' | 'desc'; +const SortOrderValidMap = { asc: true, desc: true } as const; + +let sampleLog = ''; +if (__DEV__) { + sampleLog = [ + 'Valid config is like:', + '{ dimension: "age", order: "asc" }', + 'or [{ dimension: "age", order: "asc"], { dimension: "date", order: "desc" }]' + ].join(''); +} + +const timeParser = function (val: OptionDataValue): number { + return +parseDate(val); +}; + + +export const sortTransform: ExternalDataTransform = { + + type: 'echarts:sort', + + transform: function transform(params) { + const source = params.source; + const config = params.config; + let errMsg = ''; + + // Normalize + // const orderExprList: OrderExpression[] = isArray(config[0]) + // ? config as OrderExpression[] + // : [config as OrderExpression]; + const orderExprList: OrderExpression[] = normalizeToArray(config); + + if (!orderExprList.length) { + if (__DEV__) { + errMsg = 'Empty `config` in sort transform.'; + } + throwError(errMsg); + } + + const orderDefList: { + dimIdx: DimensionIndex; + orderReturn: -1 | 1; + parser: (val: OptionDataValue) => number; + }[] = []; + each(orderExprList, function (orderExpr) { + const dimLoose = orderExpr.dimension; + const order = orderExpr.order; + const parserName = orderExpr.parse; + + if (dimLoose == null) { + if (__DEV__) { + errMsg = 'Sort transform config must has "dimension" specified.' + sampleLog; + } + throwError(errMsg); + } + + if (!hasOwn(SortOrderValidMap, order)) { + if (__DEV__) { + errMsg = 'Sort transform config must has "order" specified.' + sampleLog; + } + throwError(errMsg); + } + + const dimInfo = source.getDimensionInfo(dimLoose); + if (!dimInfo) { + if (__DEV__) { + errMsg = makePrintable( + 'Can not find dimension info via: "' + dimLoose + '".\n', + 'Existing dimensions: ', source.dimensions, '.\n', + 'Illegal config:', orderExpr, '.\n' + ); + } + throwError(errMsg); + } + + let parser; + if (parserName) { + if (parserName !== 'time') { + if (__DEV__) { + errMsg = makePrintable( + 'Invalid parser name' + parserName + '.\n', + 'Illegal config:', orderExpr, '.\n' + ); + } + throwError(errMsg); + } + parser = timeParser; + } + + orderDefList.push({ + dimIdx: dimInfo.index, + orderReturn: order === 'asc' ? -1 : 1, + parser: parser + }); + }); + + // TODO: support it? + if (!isArray(source.data)) { + if (__DEV__) { + errMsg = source.sourceFormat === SOURCE_FORMAT_KEYED_COLUMNS + ? 'sourceFormat ' + SOURCE_FORMAT_KEYED_COLUMNS + ' is not supported yet' + : source.data == null + ? 'Upstream source data is null/undefined' + : 'Unsupported source format.'; + } + throwError(errMsg); + } + + // Other source format are all array. + const sourceHeaderCount = source.sourceHeaderCount; + const resultData = []; + const headerPlaceholder = {}; + for (let i = 0; i < sourceHeaderCount; i++) { + resultData.push(headerPlaceholder); + } + for (let i = 0, len = source.count(); i < len; i++) { + resultData.push(source.getRawDataItem(i)); + } + + resultData.sort(function (item0, item1) { + if (item0 === headerPlaceholder) { + return -1; + } + if (item1 === headerPlaceholder) { + return 1; + } + // FIXME: check other empty? + // Always put empty item last? + if (item0 == null) { + return 1; + } + if (item1 == null) { + return -1; + } + // TODO Optimize a little: manually loop unrolling? + for (let i = 0; i < orderDefList.length; i++) { + const orderDef = orderDefList[i]; + let val0 = source.retrieveItemValue(item0, orderDef.dimIdx); + let val1 = source.retrieveItemValue(item1, orderDef.dimIdx); + if (orderDef.parser) { + val0 = orderDef.parser(val0); + val1 = orderDef.parser(val1); + } + if (val0 < val1) { + return orderDef.orderReturn; + } + else if (val0 > val1) { + return -orderDef.orderReturn; + } + } + return 0; + }); + + for (let i = 0; i < sourceHeaderCount; i++) { + resultData[i] = source.getRawHeaderItem(i); + } + + return { + data: resultData, + dimensions: source.dimensions, + sourceHeader: sourceHeaderCount + }; + } +}; + diff --git a/src/component/visualMap/PiecewiseModel.ts b/src/component/visualMap/PiecewiseModel.ts index 64e5f5e2d..d8d0785da 100644 --- a/src/component/visualMap/PiecewiseModel.ts +++ b/src/component/visualMap/PiecewiseModel.ts @@ -28,6 +28,7 @@ import ComponentModel from '../../model/Component'; import { inheritDefaultOption } from '../../util/component'; +// TODO: use `relationExpression.ts` instead interface VisualPiece extends VisualOptionPiecewise { min?: number max?: number diff --git a/src/data/List.ts b/src/data/List.ts index b40492ce0..871f5fd7e 100644 --- a/src/data/List.ts +++ b/src/data/List.ts @@ -36,13 +36,13 @@ import { DimensionIndex, DimensionName, DimensionLoose, OptionDataItem, ParsedValue, ParsedValueNumeric, OrdinalNumber, DimensionUserOuput, ModelOption, SeriesDataType } from '../util/types'; -import {parseDate} from '../util/number'; import {isDataItemOption} from '../util/model'; import { getECData } from '../util/ecData'; import { PathStyleProps } from 'zrender/src/graphic/Path'; import type Graph from './Graph'; import type Tree from './Tree'; import type { VisualMeta } from '../component/visualMap/VisualMapModel'; +import { parseDataValue } from './helper/parseDataValue'; const isObject = zrUtil.isObject; @@ -1916,7 +1916,7 @@ class List< objectRows: function ( this: List, dataItem: Dictionary, dimName: string, dataIndex: number, dimIndex: number ): ParsedValue { - return convertDataValue(dataItem[dimName], this._dimensionInfos[dimName]); + return parseDataValue(dataItem[dimName], this._dimensionInfos[dimName]); }, keyedColumns: getDimValueSimply, @@ -1934,7 +1934,7 @@ class List< if (!this._rawData.pure && isDataItemOption(dataItem)) { this.hasItemOption = true; } - return convertDataValue( + return parseDataValue( (value instanceof Array) ? value[dimIndex] // If value is a single number or something else not array. @@ -1954,43 +1954,8 @@ class List< function getDimValueSimply( this: List, dataItem: any, dimName: string, dataIndex: number, dimIndex: number ): ParsedValue { - return convertDataValue(dataItem[dimIndex], this._dimensionInfos[dimName]); - } - - /** - * Convert raw the value in to inner value in List. - * [Caution]: this is the key logic of user value parser. - * For backward compatibiliy, do not modify it until have to. - */ - function convertDataValue(value: any, dimInfo: DataDimensionInfo): ParsedValue { - // Performance sensitive. - const dimType = dimInfo && dimInfo.type; - if (dimType === 'ordinal') { - // If given value is a category string - const ordinalMeta = dimInfo && dimInfo.ordinalMeta; - return ordinalMeta - ? ordinalMeta.parseAndCollect(value) - : value; - } - - if (dimType === 'time' - // spead up when using timestamp - && typeof value !== 'number' - && value != null - && value !== '-' - ) { - value = +parseDate(value); - } - - // dimType defaults 'number'. - // If dimType is not ordinal and value is null or undefined or NaN or '-', - // parse to NaN. - return (value == null || value === '') - ? NaN - // If string (like '-'), using '+' parse to NaN - // If object, also parse to NaN - : +value; - }; + return parseDataValue(dataItem[dimIndex], this._dimensionInfos[dimName]); + } prepareInvertedIndex = function (list: List): void { const invertedIndicesMap = list._invertedIndicesMap; diff --git a/src/data/Source.ts b/src/data/Source.ts index 10d62afd6..bed37d657 100644 --- a/src/data/Source.ts +++ b/src/data/Source.ts @@ -67,11 +67,8 @@ import { class Source { - readonly fromDataset: boolean; - /** * Not null/undefined. - * @type {Array|Object} */ readonly data: OptionSourceData; @@ -98,7 +95,7 @@ class Source { * can be null/undefined. * Might be specified outside. */ - encodeDefine: HashMap; + readonly encodeDefine: HashMap; /** * Not null/undefined, uint. @@ -112,27 +109,33 @@ class Source { constructor(fields: { - fromDataset: boolean, - data?: OptionSourceData, - sourceFormat?: SourceFormat, // default: SOURCE_FORMAT_UNKNOWN + data: OptionSourceData, + sourceFormat: SourceFormat, // default: SOURCE_FORMAT_UNKNOWN + + // Visit config are optional: seriesLayoutBy?: SeriesLayoutBy, // default: 'column' dimensionsDefine?: DimensionDefinition[], - encodeDefine?: OptionEncode, startIndex?: number, // default: 0 - dimensionsDetectCount?: number + dimensionsDetectCount?: number, + + // [Caveat] + // This is the raw user defined `encode` in `series`. + // If user not defined, DO NOT make a empty object or hashMap here. + // An empty object or hashMap will prevent from auto generating encode. + encodeDefine?: HashMap }) { - this.fromDataset = fields.fromDataset; this.data = fields.data || ( fields.sourceFormat === SOURCE_FORMAT_KEYED_COLUMNS ? {} : [] ); this.sourceFormat = fields.sourceFormat || SOURCE_FORMAT_UNKNOWN; + + // Visit config this.seriesLayoutBy = fields.seriesLayoutBy || SERIES_LAYOUT_BY_COLUMN; - this.dimensionsDefine = fields.dimensionsDefine; - this.encodeDefine = fields.encodeDefine - && createHashMap(fields.encodeDefine); this.startIndex = fields.startIndex || 0; + this.dimensionsDefine = fields.dimensionsDefine; this.dimensionsDetectCount = fields.dimensionsDetectCount; + this.encodeDefine = fields.encodeDefine; } /** @@ -143,8 +146,7 @@ class Source { data: data, sourceFormat: isTypedArray(data) ? SOURCE_FORMAT_TYPED_ARRAY - : SOURCE_FORMAT_ORIGINAL, - fromDataset: false + : SOURCE_FORMAT_ORIGINAL }); }; } diff --git a/src/data/helper/dataProvider.ts b/src/data/helper/dataProvider.ts index 3363597a0..788610a0d 100644 --- a/src/data/helper/dataProvider.ts +++ b/src/data/helper/dataProvider.ts @@ -21,7 +21,7 @@ // ??? refactor? check the outer usage of data provider. // merge with defaultDimValueGetter? -import {isTypedArray, extend, assert, each, isObject} from 'zrender/src/core/util'; +import {isTypedArray, extend, assert, each, isObject, bind} from 'zrender/src/core/util'; import {getDataItemValue} from '../../util/model'; import Source from '../Source'; import {ArrayLike, Dictionary} from 'zrender/src/core/types'; @@ -34,7 +34,7 @@ import { SERIES_LAYOUT_BY_COLUMN, SERIES_LAYOUT_BY_ROW, DimensionName, DimensionIndex, OptionSourceData, - DimensionIndexLoose, OptionDataItem, OptionDataValue + DimensionIndexLoose, OptionDataItem, OptionDataValue, DimensionDefinition, SourceFormat, SeriesLayoutBy } from '../../util/types'; import List from '../List'; @@ -54,6 +54,7 @@ export interface DataProvider { let providerMethods: Dictionary; +let mountMethods: (provider: DefaultDataProvider, data: OptionSourceData, source: Source) => void; /** * If normal array used, mutable chunk size is supported. @@ -90,12 +91,10 @@ export class DefaultDataProvider implements DataProvider { // declare source is Source; this._source = source; - const data = this._data = source.data; - const sourceFormat = source.sourceFormat; // Typed array. TODO IE10+? - if (sourceFormat === SOURCE_FORMAT_TYPED_ARRAY) { + if (source.sourceFormat === SOURCE_FORMAT_TYPED_ARRAY) { if (__DEV__) { if (dimSize == null) { throw new Error('Typed array data must specify dimension size'); @@ -106,17 +105,7 @@ export class DefaultDataProvider implements DataProvider { this._data = data; } - const methods = providerMethods[ - sourceFormat === SOURCE_FORMAT_ARRAY_ROWS - ? sourceFormat + '_' + source.seriesLayoutBy - : sourceFormat - ]; - - if (__DEV__) { - assert(methods, 'Invalide sourceFormat: ' + sourceFormat); - } - - extend(this, methods); + mountMethods(this, data, source); } getSource(): Source { @@ -127,7 +116,7 @@ export class DefaultDataProvider implements DataProvider { return 0; } - getItem(idx: number): OptionDataItem { + getItem(idx: number, out?: ArrayLike): OptionDataItem { return; } @@ -139,35 +128,58 @@ export class DefaultDataProvider implements DataProvider { private static internalField = (function () { + mountMethods = function (provider, data, source) { + const sourceFormat = source.sourceFormat; + const seriesLayoutBy = source.seriesLayoutBy; + const startIndex = source.startIndex; + const dimsDef = source.dimensionsDefine; + + const methods = providerMethods[getMethodMapKey(sourceFormat, seriesLayoutBy)]; + if (__DEV__) { + assert(methods, 'Invalide sourceFormat: ' + sourceFormat); + } + + extend(provider, methods); + + if (sourceFormat === SOURCE_FORMAT_TYPED_ARRAY) { + provider.getItem = getItemForTypedArray; + provider.count = countForTypedArray; + } + else { + const rawItemGetter = getRawSourceItemGetter(sourceFormat, seriesLayoutBy); + provider.getItem = bind(rawItemGetter, null, data, startIndex, dimsDef); + const rawCounter = getRawSourceDataCounter(sourceFormat, seriesLayoutBy); + provider.count = bind(rawCounter, null, data, startIndex, dimsDef); + } + }; + + const getItemForTypedArray: DefaultDataProvider['getItem'] = function ( + this: DefaultDataProvider, idx: number, out: ArrayLike + ): ArrayLike { + idx = idx - this._offset; + out = out || []; + const offset = this._dimSize * idx; + for (let i = 0; i < this._dimSize; i++) { + out[i] = (this._data as ArrayLike)[offset + i]; + } + return out; + }; + + const countForTypedArray: DefaultDataProvider['count'] = function ( + this: DefaultDataProvider + ) { + return this._data ? ((this._data as ArrayLike).length / this._dimSize) : 0; + }; + providerMethods = { [SOURCE_FORMAT_ARRAY_ROWS + '_' + SERIES_LAYOUT_BY_COLUMN]: { pure: true, - count: function (this: DefaultDataProvider): number { - return Math.max(0, (this._data as OptionDataItem[][]).length - this._source.startIndex); - }, - getItem: function (this: DefaultDataProvider, idx: number): OptionDataValue[] { - return (this._data as OptionDataValue[][])[idx + this._source.startIndex]; - }, appendData: appendDataSimply }, [SOURCE_FORMAT_ARRAY_ROWS + '_' + SERIES_LAYOUT_BY_ROW]: { pure: true, - count: function (this: DefaultDataProvider): number { - const row = (this._data as OptionDataValue[][])[0]; - return row ? Math.max(0, row.length - this._source.startIndex) : 0; - }, - getItem: function (this: DefaultDataProvider, idx: number): OptionDataValue[] { - idx += this._source.startIndex; - const item = []; - const data = this._data as OptionDataValue[][]; - for (let i = 0; i < data.length; i++) { - const row = data[i]; - item.push(row ? row[idx] : null); - } - return item; - }, appendData: function () { throw new Error('Do not support appendData when set seriesLayoutBy: "row".'); } @@ -175,27 +187,11 @@ export class DefaultDataProvider implements DataProvider { [SOURCE_FORMAT_OBJECT_ROWS]: { pure: true, - count: countSimply, - getItem: getItemSimply, appendData: appendDataSimply }, [SOURCE_FORMAT_KEYED_COLUMNS]: { pure: true, - count: function (this: DefaultDataProvider): number { - const dimName = this._source.dimensionsDefine[0].name; - const col = (this._data as Dictionary)[dimName]; - return col ? col.length : 0; - }, - getItem: function (this: DefaultDataProvider, idx: number): OptionDataValue[] { - const item = []; - const dims = this._source.dimensionsDefine; - for (let i = 0; i < dims.length; i++) { - const col = (this._data as Dictionary)[dims[i].name]; - item.push(col ? col[idx] : null); - } - return item; - }, appendData: function (this: DefaultDataProvider, newData: Dictionary) { const data = this._data as Dictionary; each(newData, function (newCol, key) { @@ -208,26 +204,12 @@ export class DefaultDataProvider implements DataProvider { }, [SOURCE_FORMAT_ORIGINAL]: { - count: countSimply, - getItem: getItemSimply, appendData: appendDataSimply }, [SOURCE_FORMAT_TYPED_ARRAY]: { persistent: false, pure: true, - count: function (this: DefaultDataProvider): number { - return this._data ? ((this._data as ArrayLike).length / this._dimSize) : 0; - }, - getItem: function (this: DefaultDataProvider, idx: number, out: ArrayLike): ArrayLike { - idx = idx - this._offset; - out = out || []; - const offset = this._dimSize * idx; - for (let i = 0; i < this._dimSize; i++) { - out[i] = (this._data as ArrayLike)[offset + i]; - } - return out; - }, appendData: function (this: DefaultDataProvider, newData: ArrayLike): void { if (__DEV__) { assert( @@ -235,7 +217,6 @@ export class DefaultDataProvider implements DataProvider { 'Added data must be TypedArray if data in initialization is TypedArray' ); } - this._data = newData; }, @@ -248,12 +229,6 @@ export class DefaultDataProvider implements DataProvider { } }; - function countSimply(this: DefaultDataProvider): number { - return (this._data as []).length; - } - function getItemSimply(this: DefaultDataProvider, idx: number): OptionDataItem { - return (this._data as [])[idx]; - } function appendDataSimply(this: DefaultDataProvider, newData: ArrayLike): void { for (let i = 0; i < newData.length; i++) { (this._data as any[]).push(newData[i]); @@ -262,23 +237,136 @@ export class DefaultDataProvider implements DataProvider { })(); } + + + +type RawSourceItemGetter = ( + rawData: OptionSourceData, + startIndex: number, + dimsDef: DimensionDefinition[], + idx: number +) => OptionDataItem; + +const getItemSimply: RawSourceItemGetter = function ( + rawData, startIndex, dimsDef, idx +): OptionDataItem { + return (rawData as [])[idx]; +}; + +const rawSourceItemGetterMap: Dictionary = { + [SOURCE_FORMAT_ARRAY_ROWS + '_' + SERIES_LAYOUT_BY_COLUMN]: function ( + rawData, startIndex, dimsDef, idx + ): OptionDataValue[] { + return (rawData as OptionDataValue[][])[idx + startIndex]; + }, + [SOURCE_FORMAT_ARRAY_ROWS + '_' + SERIES_LAYOUT_BY_ROW]: function ( + rawData, startIndex, dimsDef, idx + ): OptionDataValue[] { + idx += startIndex; + const item = []; + const data = rawData as OptionDataValue[][]; + for (let i = 0; i < data.length; i++) { + const row = data[i]; + item.push(row ? row[idx] : null); + } + return item; + }, + [SOURCE_FORMAT_OBJECT_ROWS]: getItemSimply, + [SOURCE_FORMAT_KEYED_COLUMNS]: function ( + rawData, startIndex, dimsDef, idx + ): OptionDataValue[] { + const item = []; + for (let i = 0; i < dimsDef.length; i++) { + const col = (rawData as Dictionary)[dimsDef[i].name]; + item.push(col ? col[idx] : null); + } + return item; + }, + [SOURCE_FORMAT_ORIGINAL]: getItemSimply +}; + +export function getRawSourceItemGetter( + sourceFormat: SourceFormat, seriesLayoutBy: SeriesLayoutBy +): RawSourceItemGetter { + const method = rawSourceItemGetterMap[getMethodMapKey(sourceFormat, seriesLayoutBy)]; + if (__DEV__) { + assert(method, 'Do not suppport get item on "' + sourceFormat + '", "' + seriesLayoutBy + '".'); + } + return method; +} + + + + +type RawSourceDataCounter = ( + rawData: OptionSourceData, + startIndex: number, + dimsDef: DimensionDefinition[] +) => number; + +const countSimply: RawSourceDataCounter = function ( + rawData, startIndex, dimsDef +) { + return (rawData as []).length; +}; + +const rawSourceDataCounterMap: Dictionary = { + [SOURCE_FORMAT_ARRAY_ROWS + '_' + SERIES_LAYOUT_BY_COLUMN]: function ( + rawData, startIndex, dimsDef + ) { + return Math.max(0, (rawData as OptionDataItem[][]).length - startIndex); + }, + [SOURCE_FORMAT_ARRAY_ROWS + '_' + SERIES_LAYOUT_BY_ROW]: function ( + rawData, startIndex, dimsDef + ) { + const row = (rawData as OptionDataValue[][])[0]; + return row ? Math.max(0, row.length - startIndex) : 0; + }, + [SOURCE_FORMAT_OBJECT_ROWS]: countSimply, + [SOURCE_FORMAT_KEYED_COLUMNS]: function ( + rawData, startIndex, dimsDef + ) { + const dimName = dimsDef[0].name; + const col = (rawData as Dictionary)[dimName]; + return col ? col.length : 0; + }, + [SOURCE_FORMAT_ORIGINAL]: countSimply +}; + +export function getRawSourceDataCounter( + sourceFormat: SourceFormat, seriesLayoutBy: SeriesLayoutBy +): RawSourceDataCounter { + const method = rawSourceDataCounterMap[getMethodMapKey(sourceFormat, seriesLayoutBy)]; + if (__DEV__) { + assert(method, 'Do not suppport count on "' + sourceFormat + '", "' + seriesLayoutBy + '".'); + } + return method; +} + + + // TODO // merge it to dataProvider? -type RawValueGetter = ( +type RawSourceValueGetter = ( dataItem: OptionDataItem, - dataIndex: number, dimIndex: DimensionIndex, dimName: DimensionName // If dimIndex not provided, return OptionDataItem. // If dimIndex provided, return OptionDataPrimitive. ) => OptionDataValue | OptionDataItem; -const rawValueGetters: {[sourceFormat: string]: RawValueGetter} = { +const getRawValueSimply = function ( + dataItem: ArrayLike, dimIndex: number, dimName: string +): OptionDataValue | ArrayLike { + return dimIndex != null ? dataItem[dimIndex] : dataItem; +}; + +const rawSourceValueGetterMap: {[sourceFormat: string]: RawSourceValueGetter} = { [SOURCE_FORMAT_ARRAY_ROWS]: getRawValueSimply, [SOURCE_FORMAT_OBJECT_ROWS]: function ( - dataItem: Dictionary, dataIndex: number, dimIndex: number, dimName: string + dataItem: Dictionary, dimIndex: number, dimName: string ): OptionDataValue | Dictionary { return dimIndex != null ? dataItem[dimName] : dataItem; }, @@ -286,7 +374,7 @@ const rawValueGetters: {[sourceFormat: string]: RawValueGetter} = { [SOURCE_FORMAT_KEYED_COLUMNS]: getRawValueSimply, [SOURCE_FORMAT_ORIGINAL]: function ( - dataItem: OptionDataItem, dataIndex: number, dimIndex: number, dimName: string + dataItem: OptionDataItem, dimIndex: number, dimName: string ): OptionDataValue | OptionDataItem { // FIXME: In some case (markpoint in geo (geo-map.html)), // dataItem is {coord: [...]} @@ -299,12 +387,22 @@ const rawValueGetters: {[sourceFormat: string]: RawValueGetter} = { [SOURCE_FORMAT_TYPED_ARRAY]: getRawValueSimply }; -function getRawValueSimply( - dataItem: ArrayLike, dataIndex: number, dimIndex: number, dimName: string -): OptionDataValue | ArrayLike { - return dimIndex != null ? dataItem[dimIndex] : dataItem; +export function getRawSourceValueGetter(sourceFormat: SourceFormat): RawSourceValueGetter { + const method = rawSourceValueGetterMap[sourceFormat]; + if (__DEV__) { + assert(method, 'Do not suppport get value on "' + sourceFormat + '".'); + } + return method; } + +function getMethodMapKey(sourceFormat: SourceFormat, seriesLayoutBy: SeriesLayoutBy): string { + return sourceFormat === SOURCE_FORMAT_ARRAY_ROWS + ? sourceFormat + '_' + seriesLayoutBy + : sourceFormat; +} + + // ??? FIXME can these logic be more neat: getRawValue, getRawDataItem, // Consider persistent. // Caution: why use raw value to display on label or tooltip? @@ -336,9 +434,10 @@ export function retrieveRawValue( dimIndex = dimInfo.index; } - return rawValueGetters[sourceFormat](dataItem, dataIndex, dimIndex, dimName); + return getRawSourceValueGetter(sourceFormat)(dataItem, dimIndex, dimName); } + /** * Compatible with some cases (in pie, map) like: * data: [{name: 'xx', value: 5, selected: true}, ...] diff --git a/src/data/helper/parseDataValue.ts b/src/data/helper/parseDataValue.ts new file mode 100644 index 000000000..699437820 --- /dev/null +++ b/src/data/helper/parseDataValue.ts @@ -0,0 +1,73 @@ +/* +* Licensed to the Apache Software Foundation (ASF) under one +* or more contributor license agreements. See the NOTICE file +* distributed with this work for additional information +* regarding copyright ownership. The ASF licenses this file +* to you under the Apache License, Version 2.0 (the +* "License"); you may not use this file except in compliance +* with the License. You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an +* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +* KIND, either express or implied. See the License for the +* specific language governing permissions and limitations +* under the License. +*/ + +import { ParsedValue, DimensionType } from '../../util/types'; +import OrdinalMeta from '../OrdinalMeta'; +import { parseDate } from '../../util/number'; + + +/** + * Convert raw the value in to inner value in List. + * + * [Performance sensitive] + * + * [Caution]: this is the key logic of user value parser. + * For backward compatibiliy, do not modify it until have to ! + */ +export function parseDataValue( + value: any, + // For high performance, do not omit the second param. + opt: { + // Default type: 'number'. There is no 'unknown' type. That is, a string + // will be parsed to NaN if do not set `type` as 'ordinal'. It has been + // the logic in `List.ts` for long time. Follow the same way if you need + // to get same result as List did from a raw value. + type?: DimensionType, + ordinalMeta?: OrdinalMeta + } +): ParsedValue { + // Performance sensitive. + const dimType = opt && opt.type; + if (dimType === 'ordinal') { + // If given value is a category string + const ordinalMeta = opt && opt.ordinalMeta; + return ordinalMeta + ? ordinalMeta.parseAndCollect(value) + : value; + } + + if (dimType === 'time' + // spead up when using timestamp + && typeof value !== 'number' + && value != null + && value !== '-' + ) { + value = +parseDate(value); + } + + // dimType defaults 'number'. + // If dimType is not ordinal and value is null or undefined or NaN or '-', + // parse to NaN. + return (value == null || value === '') + ? NaN + // If string (like '-'), using '+' parse to NaN + // If object, also parse to NaN + : +value; +}; + diff --git a/src/data/helper/sourceHelper.ts b/src/data/helper/sourceHelper.ts index 8f9d7cba1..9ae015112 100644 --- a/src/data/helper/sourceHelper.ts +++ b/src/data/helper/sourceHelper.ts @@ -18,7 +18,7 @@ */ -import {makeInner, getDataItemValue} from '../../util/model'; +import {makeInner, getDataItemValue, queryReferringComponents, SINGLE_REFERRING} from '../../util/model'; import { createHashMap, each, @@ -31,7 +31,9 @@ import { extend, assert, hasOwn, - HashMap + HashMap, + isNumber, + clone } from 'zrender/src/core/util'; import Source from '../Source'; @@ -45,8 +47,6 @@ import { SOURCE_FORMAT_UNKNOWN, SourceFormat, Dictionary, - SeriesEncodeOptionMixin, - SeriesOption, OptionSourceData, SeriesLayoutBy, OptionSourceHeader, @@ -59,9 +59,11 @@ import { OptionSourceDataOriginal, OptionSourceDataObjectRows, OptionEncode, - DimensionIndex + DimensionIndex, + SeriesEncodableModel, + OptionEncodeValue } from '../../util/types'; -import { DatasetModel } from '../../component/dataset'; +import { DatasetModel, DatasetOption } from '../../component/dataset'; import SeriesModel from '../../model/Series'; import GlobalModel from '../../model/Global'; import { CoordDimensionDefinition } from './createDimensions'; @@ -74,16 +76,11 @@ export const BE_ORDINAL = { }; type BeOrdinalValue = (typeof BE_ORDINAL)[keyof typeof BE_ORDINAL]; -const innerDatasetModel = makeInner<{ - sourceFormat: SourceFormat; -}, DatasetModel>(); -const innerSeriesModel = makeInner<{ - source: Source; -}, SeriesModel>(); const innerGlobalModel = makeInner<{ datasetMap: HashMap }, GlobalModel>(); + interface DatasetRecord { categoryWayDim: number; valueWayDim: number; @@ -93,11 +90,13 @@ type SeriesEncodeInternal = { [key in keyof OptionEncode]: DimensionIndex[]; }; -type SeriesEncodableModel = SeriesModel; - +export interface SourceMetaRawOption { + seriesLayoutBy: SeriesLayoutBy; + sourceHeader: OptionSourceHeader; + dimensions: DimensionDefinitionLoose[]; +} -export function detectSourceFormat(datasetModel: DatasetModel): void { - const data = datasetModel.option.source; +export function detectSourceFormat(data: DatasetOption['source']): SourceFormat { let sourceFormat: SourceFormat = SOURCE_FORMAT_UNKNOWN; if (isTypedArray(data)) { @@ -137,33 +136,7 @@ export function detectSourceFormat(datasetModel: DatasetModel): void { throw new Error('Invalid data'); } - innerDatasetModel(datasetModel).sourceFormat = sourceFormat; -} - -/** - * [Scenarios]: - * (1) Provide source data directly: - * series: { - * encode: {...}, - * dimensions: [...] - * seriesLayoutBy: 'row', - * data: [[...]] - * } - * (2) Refer to datasetModel. - * series: [{ - * encode: {...} - * // Ignore datasetIndex means `datasetIndex: 0` - * // and the dimensions defination in dataset is used - * }, { - * encode: {...}, - * seriesLayoutBy: 'column', - * datasetIndex: 1 - * }] - * - * Get data from series itself or datset. - */ -export function getSource(seriesModel: SeriesModel): Source { - return innerSeriesModel(seriesModel).source; + return sourceFormat; } /** @@ -174,65 +147,63 @@ export function resetSourceDefaulter(ecModel: GlobalModel): void { innerGlobalModel(ecModel).datasetMap = createHashMap(); } -/** - * [Caution]: - * MUST be called after series option merged and - * before "series.getInitailData()" called. - * - * [The rule of making default encode]: - * Category axis (if exists) alway map to the first dimension. - * Each other axis occupies a subsequent dimension. - * - * [Why make default encode]: - * Simplify the typing of encode in option, avoiding the case like that: - * series: [{encode: {x: 0, y: 1}}, {encode: {x: 0, y: 2}}, {encode: {x: 0, y: 3}}], - * where the "y" have to be manually typed as "1, 2, 3, ...". - */ -export function prepareSource(seriesModel: SeriesEncodableModel): void { - const seriesOption = seriesModel.option; - - let data = seriesOption.data as OptionSourceData; - let sourceFormat: SourceFormat = isTypedArray(data) - ? SOURCE_FORMAT_TYPED_ARRAY : SOURCE_FORMAT_ORIGINAL; - let fromDataset = false; - - let seriesLayoutBy = seriesOption.seriesLayoutBy; - let sourceHeader = seriesOption.sourceHeader; - let dimensionsDefine = seriesOption.dimensions; - - const datasetModel = getDatasetModel(seriesModel); - if (datasetModel) { - const datasetOption = datasetModel.option; - - data = datasetOption.source; - sourceFormat = innerDatasetModel(datasetModel).sourceFormat; - fromDataset = true; - - // These settings from series has higher priority. - seriesLayoutBy = seriesLayoutBy || datasetOption.seriesLayoutBy; - sourceHeader == null && (sourceHeader = datasetOption.sourceHeader); - dimensionsDefine = dimensionsDefine || datasetOption.dimensions; - } - - const completeResult = completeBySourceData( - data, sourceFormat, seriesLayoutBy, sourceHeader, dimensionsDefine +export function createSource( + sourceData: OptionSourceData, + thisMetaRawOption: SourceMetaRawOption, + // can be null. If not provided, auto detect it from `sourceData`. + sourceFormat: SourceFormat, + encodeDefine: OptionEncode // can be null +): Source { + sourceFormat = sourceFormat || detectSourceFormat(sourceData); + const dimInfo = determineSourceDimensions( + sourceData, + sourceFormat, + thisMetaRawOption.seriesLayoutBy, + thisMetaRawOption.sourceHeader, + thisMetaRawOption.dimensions ); - - innerSeriesModel(seriesModel).source = new Source({ - data: data, - fromDataset: fromDataset, - seriesLayoutBy: seriesLayoutBy, + const source = new Source({ + data: sourceData, sourceFormat: sourceFormat, - dimensionsDefine: completeResult.dimensionsDefine, - startIndex: completeResult.startIndex, - dimensionsDetectCount: completeResult.dimensionsDetectCount, - // Note: dataset option does not have `encode`. - encodeDefine: seriesOption.encode + + seriesLayoutBy: thisMetaRawOption.seriesLayoutBy, + dimensionsDefine: dimInfo.dimensionsDefine, + startIndex: dimInfo.startIndex, + dimensionsDetectCount: dimInfo.dimensionsDetectCount, + encodeDefine: makeEncodeDefine(encodeDefine) + }); + + return source; +} + +/** + * Clone except source data. + */ +export function cloneSourceShallow(source: Source) { + return new Source({ + data: source.data, + sourceFormat: source.sourceFormat, + + seriesLayoutBy: source.seriesLayoutBy, + dimensionsDefine: clone(source.dimensionsDefine), + startIndex: source.startIndex, + dimensionsDetectCount: source.dimensionsDetectCount, + encodeDefine: makeEncodeDefine(source.encodeDefine) }); } +function makeEncodeDefine( + encodeDefine: OptionEncode | HashMap +): HashMap { + // null means user not specify `series.encode`. + return encodeDefine + ? createHashMap(encodeDefine) + : null; +} + + // return {startIndex, dimensionsDefine, dimensionsCount} -function completeBySourceData( +export function determineSourceDimensions( data: OptionSourceData, sourceFormat: SourceFormat, seriesLayoutBy: SeriesLayoutBy, @@ -248,7 +219,7 @@ function completeBySourceData( if (!data) { return { - dimensionsDefine: normalizeDimensionsDefine(dimensionsDefine), + dimensionsDefine: normalizeDimensionsOption(dimensionsDefine), startIndex, dimensionsDetectCount }; @@ -275,7 +246,7 @@ function completeBySourceData( }, seriesLayoutBy, dataArrayRows, 10); } else { - startIndex = sourceHeader ? 1 : 0; + startIndex = isNumber(sourceHeader) ? sourceHeader : sourceHeader ? 1 : 0; } if (!dimensionsDefine && startIndex === 1) { @@ -318,7 +289,7 @@ function completeBySourceData( return { startIndex: startIndex, - dimensionsDefine: normalizeDimensionsDefine(dimensionsDefine), + dimensionsDefine: normalizeDimensionsOption(dimensionsDefine), dimensionsDetectCount: dimensionsDetectCount }; } @@ -326,7 +297,7 @@ function completeBySourceData( // Consider dimensions defined like ['A', 'price', 'B', 'price', 'C', 'price'], // which is reasonable. But dimension name is duplicated. // Returns undefined or an array contains only object without null/undefiend or string. -function normalizeDimensionsDefine(dimensionsDefine: DimensionDefinitionLoose[]): DimensionDefinition[] { +function normalizeDimensionsOption(dimensionsDefine: DimensionDefinitionLoose[]): DimensionDefinition[] { if (!dimensionsDefine) { // The meaning of null/undefined is different from empty array. return; @@ -419,7 +390,7 @@ export function makeSeriesEncodeForAxisCoordSys( ): SeriesEncodeInternal { const encode: SeriesEncodeInternal = {}; - const datasetModel = getDatasetModel(seriesModel); + const datasetModel = querySeriesUpstreamDatasetModel(seriesModel); // Currently only make default when using dataset, util more reqirements occur. if (!datasetModel || !coordDimensions) { return encode; @@ -512,7 +483,7 @@ export function makeSeriesEncodeForNameBased( ): SeriesEncodeInternal { const encode: SeriesEncodeInternal = {}; - const datasetModel = getDatasetModel(seriesModel); + const datasetModel = querySeriesUpstreamDatasetModel(seriesModel); // Currently only make default when using dataset, util more reqirements occur. if (!datasetModel) { return encode; @@ -600,19 +571,53 @@ export function makeSeriesEncodeForNameBased( } /** - * If return null/undefined, indicate that should not use datasetModel. + * @return If return null/undefined, indicate that should not use datasetModel. */ -function getDatasetModel(seriesModel: SeriesEncodableModel): DatasetModel { - const option = seriesModel.option; +export function querySeriesUpstreamDatasetModel( + seriesModel: SeriesEncodableModel +): DatasetModel { // Caution: consider the scenario: // A dataset is declared and a series is not expected to use the dataset, // and at the beginning `setOption({series: { noData })` (just prepare other // option but no data), then `setOption({series: {data: [...]}); In this case, // the user should set an empty array to avoid that dataset is used by default. - const thisData = option.data; + const thisData = seriesModel.get('data', true); if (!thisData) { - return seriesModel.ecModel.getComponent('dataset', option.datasetIndex || 0) as DatasetModel; + return queryReferringComponents( + seriesModel.ecModel, + 'dataset', + { + index: seriesModel.get('datasetIndex', true), + id: seriesModel.get('datasetId', true) + }, + SINGLE_REFERRING + ).models[0] as DatasetModel; + } +} + +/** + * @return Always return an array event empty. + */ +export function queryDatasetUpstreamDatasetModels( + datasetModel: DatasetModel +): DatasetModel[] { + // Only these attributes declared, we by defualt reference to `datasetIndex: 0`. + // Otherwise, no reference. + if (!datasetModel.get('transform', true) + && !datasetModel.get('fromTransformResult', true) + ) { + return []; } + + return queryReferringComponents( + datasetModel.ecModel, + 'dataset', + { + index: datasetModel.get('fromDatasetIndex', true), + id: datasetModel.get('fromDatasetId', true) + }, + SINGLE_REFERRING + ).models as DatasetModel[]; } /** diff --git a/src/data/helper/sourceManager.ts b/src/data/helper/sourceManager.ts new file mode 100644 index 000000000..74ed26472 --- /dev/null +++ b/src/data/helper/sourceManager.ts @@ -0,0 +1,368 @@ +/* +* Licensed to the Apache Software Foundation (ASF) under one +* or more contributor license agreements. See the NOTICE file +* distributed with this work for additional information +* regarding copyright ownership. The ASF licenses this file +* to you under the Apache License, Version 2.0 (the +* "License"); you may not use this file except in compliance +* with the License. You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an +* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +* KIND, either express or implied. See the License for the +* specific language governing permissions and limitations +* under the License. +*/ + +import { DatasetModel } from '../../component/dataset'; +import SeriesModel from '../../model/Series'; +import { setAsPrimitive, map, isTypedArray, defaults, assert, each } from 'zrender/src/core/util'; +import Source from '../Source'; +import { + SeriesEncodableModel, OptionSourceData, + SOURCE_FORMAT_TYPED_ARRAY, SOURCE_FORMAT_ORIGINAL, + SourceFormat, SeriesLayoutBy, OptionSourceHeader, DimensionDefinitionLoose +} from '../../util/types'; +import { + querySeriesUpstreamDatasetModel, queryDatasetUpstreamDatasetModels, + createSource, SourceMetaRawOption, cloneSourceShallow +} from './sourceHelper'; +import { applyDataTransform } from './transform'; + + +/** + * [REQUIREMENT_MEMO]: + * (0) `metaRawOption` means `dimensions`/`sourceHeader`/`seriesLayoutBy` in raw option. + * (1) Keep support the feature: `metaRawOption` can be specified both on `series` and + * `root-dataset`. Them on `series` has higher priority. + * (2) Do not support to set `metaRawOption` on a `non-root-dataset`, because it might + * confuse users: whether those props indicate how to visit the upstream source or visit + * the transform result source, and some transforms has nothing to do with these props, + * and some transforms might have multiple upstream. + * (3) Transforms should specify `metaRawOption` in each output, just like they can be + * declared in `root-dataset`. + * (4) At present only support visit source in `SERIES_LAYOUT_BY_COLUMN` in transforms. + * That is for reducing complexity in transfroms. + * PENDING: Whether to provide transposition transform? + * + * [IMPLEMENTAION_MEMO]: + * "sourceVisitConfig" are calculated from `metaRawOption` and `data`. + * They will not be calculated until `source` is about to be visited (to prevent from + * duplicate calcuation). `source` is visited only in series and input to transforms. + * + * [SCENARIO]: + * (1) Provide source data directly: + * ```js + * series: { + * encode: {...}, + * dimensions: [...] + * seriesLayoutBy: 'row', + * data: [[...]] + * } + * ``` + * (2) Series refer to dataset. + * ```js + * series: [{ + * encode: {...} + * // Ignore datasetIndex means `datasetIndex: 0` + * // and the dimensions defination in dataset is used + * }, { + * encode: {...}, + * seriesLayoutBy: 'column', + * datasetIndex: 1 + * }] + * ``` + * (3) dataset transform + * ```js + * dataset: [{ + * source: [...] + * }, { + * source: [...] + * }, { + * // By default from 0. + * transform: { type: 'filter', config: {...} } + * }, { + * // Piped. + * transform: [ + * { type: 'filter', config: {...} }, + * { type: 'sort', config: {...} } + * ] + * }, { + * id: 'regressionData', + * fromDatasetIndex: 1, + * // Third-party transform + * transform: { type: 'ecStat:regression', config: {...} } + * }, { + * // retrieve the extra result. + * id: 'regressionFormula', + * fromDatasetId: 'regressionData', + * fromTransformResult: 1 + * }] + * ``` + */ + +export class SourceManager { + + // Currently only datasetModel can host `transform` + private _sourceHost: DatasetModel | SeriesModel; + + // Cached source. Do not repeat calculating if not dirty. + private _sourceList: Source[] = []; + + // version sign of each upstream source manager. + private _upstreamSignList: string[] = []; + + private _versionSignBase = 0; + + constructor(sourceHost: DatasetModel | SeriesModel) { + this._sourceHost = sourceHost; + } + + /** + * Mark dirty. + */ + dirty() { + this._setLocalSource([], []); + } + + private _setLocalSource( + sourceList: Source[], + upstreamSignList: string[] + ): void { + this._sourceList = sourceList; + this._upstreamSignList = upstreamSignList; + this._versionSignBase++; + if (this._versionSignBase > 9e10) { + this._versionSignBase = 0; + } + } + + /** + * For detecting whether the upstream source is dirty, so that + * the local cached source (in `_sourceList`) should be discarded. + */ + private _getVersionSign(): string { + return this._sourceHost.uid + '_' + this._versionSignBase; + } + + /** + * Always return a source instance. Otherwise throw error. + */ + prepareSource(): void { + // For the case that call `setOption` multiple time but no data changed, + // cache the result source to prevent from repeating transform. + if (this._isDirty()) { + this._createSource(); + } + } + + private _createSource(): void { + this._setLocalSource([], []); + const sourceHost = this._sourceHost; + + const upSourceMgrList = this._getUpstreamSourceManagers(); + const hasUpstream = !!upSourceMgrList.length; + let resultSourceList: Source[]; + let upstreamSignList: string[]; + + if (isSeries(sourceHost)) { + const seriesModel = sourceHost as SeriesEncodableModel; + let data; + let sourceFormat: SourceFormat; + let upMetaRawOption; + + // Has upstream dataset + if (hasUpstream) { + const upSourceMgr = upSourceMgrList[0]; + upSourceMgr.prepareSource(); + const upSource = upSourceMgr.getSource(); + data = upSource.data; + sourceFormat = upSource.sourceFormat; + upMetaRawOption = upSourceMgr._getSourceMetaRawOption(); + upstreamSignList = [upSourceMgr._getVersionSign()]; + } + // Series data is from own. + else { + data = seriesModel.get('data', true) as OptionSourceData; + sourceFormat = isTypedArray(data) + ? SOURCE_FORMAT_TYPED_ARRAY : SOURCE_FORMAT_ORIGINAL; + upstreamSignList = []; + } + + const thisMetaRawOption = defaults( + this._getSourceMetaRawOption(), + // See [REQUIREMENT MEMO], merge settings on series and parent dataset if it is root. + upMetaRawOption + ); + resultSourceList = [createSource( + data, + thisMetaRawOption, + sourceFormat, + seriesModel.get('encode', true) + )]; + } + else { + const datasetModel = sourceHost as DatasetModel; + + // Has upstream dataset. + if (hasUpstream) { + const result = this._applyTransform(upSourceMgrList); + resultSourceList = result.sourceList; + upstreamSignList = result.upstreamSignList; + } + // Is root dataset. + else { + const sourceData = datasetModel.get('source', true); + resultSourceList = [createSource( + sourceData, + this._getSourceMetaRawOption(), + null, + // Note: dataset option does not have `encode`. + null + )]; + upstreamSignList = []; + } + } + + if (__DEV__) { + assert(resultSourceList && upstreamSignList); + } + + this._setLocalSource(resultSourceList, upstreamSignList); + } + + private _applyTransform( + upMgrList: SourceManager[] + ): { + sourceList: Source[], + upstreamSignList: string[] + } { + const datasetModel = this._sourceHost as DatasetModel; + const transformOption = datasetModel.get('transform', true); + const fromTransformResult = datasetModel.get('fromTransformResult', true); + let sourceList: Source[]; + let upstreamSignList: string[]; + + if (transformOption) { + const upSourceList: Source[] = []; + upstreamSignList = []; + each(upMgrList, upMgr => { + upMgr.prepareSource(); + upSourceList.push(upMgr.getSource()); + upstreamSignList.push(upMgr._getVersionSign()); + }); + sourceList = applyDataTransform( + transformOption, + upSourceList, + { datasetIndex: datasetModel.componentIndex } + ); + } + else if (fromTransformResult != null) { + if (upMgrList.length !== 1) { + let errMsg = ''; + if (__DEV__) { + errMsg = 'When using `fromTransformResult`, there should be only one upstream dataset'; + } + doThrow(errMsg); + } + const upMgr = upMgrList[0]; + upMgr.prepareSource(); + const upSource = upMgr.getSource(fromTransformResult); + upstreamSignList = [upMgr._getVersionSign()]; + sourceList = [cloneSourceShallow(upSource)]; + } + + return { sourceList, upstreamSignList }; + } + + private _isDirty(): boolean { + const sourceList = this._sourceList; + if (!sourceList.length) { + return true; + } + + // All sourceList is from the some upsteam. + const upSourceMgrList = this._getUpstreamSourceManagers(); + for (let i = 0; i < upSourceMgrList.length; i++) { + const upSrcMgr = upSourceMgrList[i]; + if ( + // Consider the case that there is ancestor diry, call it recursively. + // The performance is probably not an issue because usually the chain is not long. + upSrcMgr._isDirty() + || this._upstreamSignList[i] !== upSrcMgr._getVersionSign() + ) { + return true; + } + } + } + + /** + * @param sourceIndex By defualt 0, means "main source". + * Most cases there is only one source. + */ + getSource(sourceIndex?: number) { + return this._sourceList[sourceIndex || 0]; + } + + /** + * PEDING: Is it fast enough? + * If no upstream, return empty array. + */ + private _getUpstreamSourceManagers(): SourceManager[] { + // Always get the relationship from the raw option. + // Do not cache the link of the dependency graph, so that + // no need to update them when change happen. + const sourceHost = this._sourceHost; + + if (isSeries(sourceHost)) { + const datasetModel = querySeriesUpstreamDatasetModel(sourceHost); + return !datasetModel ? [] : [datasetModel.getSourceManager()]; + } + else { + return map( + queryDatasetUpstreamDatasetModels(sourceHost as DatasetModel), + datasetModel => datasetModel.getSourceManager() + ); + } + } + + private _getSourceMetaRawOption(): SourceMetaRawOption { + const sourceHost = this._sourceHost; + let seriesLayoutBy: SeriesLayoutBy; + let sourceHeader: OptionSourceHeader; + let dimensions: DimensionDefinitionLoose[]; + if (isSeries(sourceHost)) { + seriesLayoutBy = sourceHost.get('seriesLayoutBy', true); + sourceHeader = sourceHost.get('sourceHeader', true); + dimensions = sourceHost.get('dimensions', true); + } + // See [REQUIREMENT MEMO], `non-root-dataset` do not support them. + else if (!this._getUpstreamSourceManagers().length) { + const model = sourceHost as DatasetModel; + seriesLayoutBy = model.get('seriesLayoutBy', true); + sourceHeader = model.get('sourceHeader', true); + dimensions = model.get('dimensions', true); + } + return { seriesLayoutBy, sourceHeader, dimensions }; + } + +} + +// Call this method after `super.init` and `super.mergeOption` to +// disable the transform merge, but do not disable transfrom clone from rawOption. +export function disableTransformOptionMerge(datasetModel: DatasetModel): void { + const transformOption = datasetModel.option.transform; + transformOption && setAsPrimitive(datasetModel.option.transform); +} + +function isSeries(sourceHost: SourceManager['_sourceHost']): sourceHost is SeriesEncodableModel { + // Avoid circular dependency with Series.ts + return (sourceHost as SeriesModel).mainType === 'series'; +} + +function doThrow(errMsg: string): void { + throw new Error(errMsg); +} diff --git a/src/data/helper/transform.ts b/src/data/helper/transform.ts new file mode 100644 index 000000000..e6150b1d5 --- /dev/null +++ b/src/data/helper/transform.ts @@ -0,0 +1,317 @@ +/* +* Licensed to the Apache Software Foundation (ASF) under one +* or more contributor license agreements. See the NOTICE file +* distributed with this work for additional information +* regarding copyright ownership. The ASF licenses this file +* to you under the Apache License, Version 2.0 (the +* "License"); you may not use this file except in compliance +* with the License. You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an +* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +* KIND, either express or implied. See the License for the +* specific language governing permissions and limitations +* under the License. +*/ + +import { + Dictionary, OptionSourceData, DimensionDefinitionLoose, OptionSourceHeader, + SourceFormat, DimensionDefinition, OptionDataItem, DimensionIndex, + OptionDataValue, DimensionLoose, DimensionName, ParsedValue, SERIES_LAYOUT_BY_COLUMN +} from '../../util/types'; +import Source from '../Source'; +import { normalizeToArray } from '../../util/model'; +import { + assert, createHashMap, bind, each, hasOwn, map, clone, isObject, + isArrayLike +} from 'zrender/src/core/util'; +import { + getRawSourceItemGetter, getRawSourceDataCounter, getRawSourceValueGetter +} from './dataProvider'; +import { parseDataValue } from './parseDataValue'; +import { createSource } from './sourceHelper'; +import { consoleLog, makePrintable } from '../../util/log'; + + +export type PipedDataTransformOption = DataTransformOption[]; +export type DataTransformType = string; +export type DataTransformConfig = unknown; + +export interface DataTransformOption { + type: DataTransformType; + config: DataTransformConfig; + // Print the result via `console.log` when transform performed. Only work in dev mode for debug. + print?: boolean; +} + +export interface DataTransformResult { + source: Source; +} + +export interface DataTransform { + (sourceList: Source[], config: DataTransformConfig): { + } +} + +export interface ExternalDataTransform { + // Must include namespace like: 'ecStat:regression' + type: string, + transform?: ( + param: ExternalDataTransformParam + ) => ExternalDataTransformResultItem | ExternalDataTransformResultItem[] +} + +interface ExternalDataTransformParam { + // This is the first source in sourceList. In most cases, + // there is only one upstream source. + source: ExternalSource; + sourceList: ExternalSource[]; + config: TO['config']; +} +export interface ExternalDataTransformResultItem { + data: OptionSourceData; + dimensions?: DimensionDefinitionLoose[]; + sourceHeader?: OptionSourceHeader; +} +export interface ExternalDimensionDefinition extends DimensionDefinition { + // Mandatory + index: DimensionIndex; +} + +/** + * TODO: disable writable. + * This structure will be exposed to users. + */ +class ExternalSource { + /** + * [Caveat] + * This instance is to be exposed to users. + * DO NOT mount private members on this instance directly. + * If we have to use private members, we can make them in closure or use `makeInner`. + */ + + data: OptionSourceData; + sourceFormat: SourceFormat; + dimensions: ExternalDimensionDefinition[]; + sourceHeaderCount: number; + + getDimensionInfo(dim: DimensionLoose): ExternalDimensionDefinition { + return; + } + + getRawDataItem(dataIndex: number): OptionDataItem { + return; + } + + getRawHeaderItem(dataIndex: number): OptionDataItem { + return; + } + + count(): number { + return; + } + + /** + * Only support by dimension index. + * No need to support by dimension name in transform function, + * becuase transform function is not case-specific, no need to use name literally. + */ + retrieveItemValue(rawItem: OptionDataItem, dimIndex: DimensionIndex): OptionDataValue { + return; + } + + convertDataValue(rawVal: unknown, dimInfo: ExternalDimensionDefinition): ParsedValue { + return parseDataValue(rawVal, dimInfo); + } +} + +function createExternalSource( + data: OptionSourceData, + sourceFormat: SourceFormat, + dimsDef: DimensionDefinition[], + sourceHeaderCount: number +): ExternalSource { + const extSource = new ExternalSource(); + + extSource.data = data; + extSource.sourceFormat = sourceFormat; + extSource.sourceHeaderCount = sourceHeaderCount; + + // Create a new dimensions structure for exposing. + const dimensions = extSource.dimensions = [] as ExternalDimensionDefinition[]; + const dimsByName = {} as Dictionary; + each(dimsDef, function (dimDef, idx) { + const name = dimDef.name; + const dimDefExt = { + index: idx, + name: name, + displayName: dimDef.displayName + }; + dimensions.push(dimDefExt); + // Users probably not sepcify dimension name. For simplicity, data transform + // do not generate dimension name. + if (name != null) { + // Dimension name should not be duplicated. + // For simplicity, data transform forbid name duplication, do not generate + // new name like module `completeDimensions.ts` did, but just tell users. + assert(!hasOwn(dimsByName, name), 'dimension name "' + name + '" duplicated.'); + dimsByName[name] = dimDefExt; + } + }); + + // Implement public methods: + const rawItemGetter = getRawSourceItemGetter(sourceFormat, SERIES_LAYOUT_BY_COLUMN); + extSource.getRawDataItem = bind(rawItemGetter, null, data, sourceHeaderCount, dimensions); + extSource.getRawHeaderItem = function (dataIndex: number) { + if (dataIndex < sourceHeaderCount) { + return rawItemGetter(data, 0, dimensions, dataIndex); + } + }; + + const rawCounter = getRawSourceDataCounter(sourceFormat, SERIES_LAYOUT_BY_COLUMN); + extSource.count = bind(rawCounter, null, data, sourceHeaderCount, dimensions); + + const rawValueGetter = getRawSourceValueGetter(sourceFormat); + extSource.retrieveItemValue = function (rawItem, dimIndex) { + if (rawItem == null) { + return; + } + const dimDef = extSource.dimensions[dimIndex]; + // When `dimIndex` is `null`, `rawValueGetter` return the whole item. + if (dimDef) { + return rawValueGetter(rawItem, dimIndex, dimDef.name) as OptionDataValue; + } + }; + + extSource.getDimensionInfo = bind(getDimensionInfo, null, dimensions, dimsByName); + + return extSource; +} + + +function getDimensionInfo( + dimensions: ExternalDimensionDefinition[], + dimsByName: Dictionary, + dim: DimensionLoose +): ExternalDimensionDefinition { + if (dim == null) { + return; + } + // Keep the same logic as `List::getDimension` did. + if (typeof dim === 'number' + // If being a number-like string but not being defined a dimension name. + || (!isNaN(dim as any) && !hasOwn(dimsByName, dim)) + ) { + return dimensions[dim as DimensionIndex]; + } + else if (hasOwn(dimsByName, dim)) { + return dimsByName[dim as DimensionName]; + } +} + + + +const externalTransformMap = createHashMap(); + +export function registerExternalTransform( + externalTransform: ExternalDataTransform +): void { + externalTransform = clone(externalTransform); + let type = externalTransform.type; + assert(type, 'Must have a `type` when `registerTransform`.'); + const typeParsed = type.split(':'); + assert(typeParsed.length === 2, 'Name must include namespace like "ns:regression".'); + // Namespace 'echarts:xxx' is official namespace, where the transforms should + // be called directly via 'xxx' rather than 'echarts:xxx'. + if (typeParsed[0] === 'echarts') { + type = typeParsed[1]; + } + externalTransformMap.set(type, externalTransform); +} + +export function applyDataTransform( + rawTransOption: DataTransformOption | PipedDataTransformOption, + sourceList: Source[], + infoForPrint: { datasetIndex: number } +): Source[] { + const pipedTransOption: PipedDataTransformOption = normalizeToArray(rawTransOption); + + for (let i = 0, len = pipedTransOption.length; i < len; i++) { + const transOption = pipedTransOption[i]; + sourceList = applySingleDataTransform(transOption, sourceList); + // piped transform only support single input, except the fist one. + // piped transform only support single output, except the last one. + if (i < len - 1) { + sourceList.length = Math.max(sourceList.length, 1); + } + + if (__DEV__) { + if (transOption.print) { + const printStrArr = map(sourceList, source => { + return '--- datasetIndex: ' + infoForPrint.datasetIndex + ', transform result: ---\n' + + makePrintable(source.data); + }).join('\n'); + consoleLog(printStrArr); + } + } + } + + return sourceList; +} + +function applySingleDataTransform( + rawTransOption: DataTransformOption, + upSourceList: Source[] +): Source[] { + assert(upSourceList.length, 'Must have at least one upstream dataset.'); + + const transOption = rawTransOption; + const transType = transOption.type; + const externalTransform = externalTransformMap.get(transType); + + assert(externalTransform, 'Can not find transform on type "' + transType + '".'); + + // Prepare source + const sourceList = map(upSourceList, function (source) { + return createExternalSource( + source.data, + source.sourceFormat, + source.dimensionsDefine, + source.startIndex + ); + }); + + const resultList = normalizeToArray( + externalTransform.transform({ + source: sourceList[0], + sourceList: sourceList, + config: clone(transOption.config) + }) + ); + + return map(resultList, function (result) { + assert( + isObject(result), + 'A transform should not return some empty results.' + ); + assert( + isObject(result.data) || isArrayLike(result.data), + 'Result data should be object or array in data transform.' + ); + + return createSource( + result.data, + { + seriesLayoutBy: SERIES_LAYOUT_BY_COLUMN, + sourceHeader: result.sourceHeader, + dimensions: result.dimensions + }, + null, + null + ); + }); +} + diff --git a/src/echarts.ts b/src/echarts.ts index 76334a7b0..fa991fdfc 100644 --- a/src/echarts.ts +++ b/src/echarts.ts @@ -100,6 +100,7 @@ import { handleLegacySelectEvents } from './legacy/dataSelectAction'; // At least canvas renderer. import 'zrender/src/canvas/canvas'; +import { registerExternalTransform } from './data/helper/transform'; declare let global: any; type ModelFinder = modelUtil.ModelFinder; @@ -2716,6 +2717,8 @@ export function getMap(mapName: string) { }; } +export const registerTransform = registerExternalTransform; + /** * Globa dispatchAction to a specified chart instance. */ diff --git a/src/model/Component.ts b/src/model/Component.ts index 4fc656a98..373ea6830 100644 --- a/src/model/Component.ts +++ b/src/model/Component.ts @@ -29,7 +29,9 @@ import { ClassManager, mountExtend } from '../util/clazz'; -import {makeInner, ModelFinderIndexQuery, queryReferringComponents, ModelFinderIdQuery, QueryReferringOpt} from '../util/model'; +import { + makeInner, ModelFinderIndexQuery, queryReferringComponents, ModelFinderIdQuery, QueryReferringOpt +} from '../util/model'; import * as layout from '../util/layout'; import GlobalModel from './Global'; import { diff --git a/src/model/OptionManager.ts b/src/model/OptionManager.ts index 33f074264..b92d5b40e 100644 --- a/src/model/OptionManager.ts +++ b/src/model/OptionManager.ts @@ -36,6 +36,7 @@ import { each, clone, map, isTypedArray, setAsPrimitive // , HashMap , createHashMap, extend, merge, } from 'zrender/src/core/util'; +import { DatasetOption } from '../component/dataset'; const QUERY_REG = /^(min|max)?(.+)$/; @@ -98,6 +99,9 @@ class OptionManager { each(normalizeToArray((rawOption as ECUnitOption).series), function (series: SeriesOption) { series && series.data && isTypedArray(series.data) && setAsPrimitive(series.data); }); + each(normalizeToArray((rawOption as ECUnitOption).dataset), function (dataset: DatasetOption) { + dataset && dataset.source && isTypedArray(dataset.source) && setAsPrimitive(dataset.source); + }); } // Caution: some series modify option data, if do not clone, diff --git a/src/model/Series.ts b/src/model/Series.ts index 6fdd6915a..b697c5622 100644 --- a/src/model/Series.ts +++ b/src/model/Series.ts @@ -41,10 +41,6 @@ import { fetchLayoutMode } from '../util/layout'; import {createTask} from '../stream/task'; -import { - prepareSource, - getSource -} from '../data/helper/sourceHelper'; import {retrieveRawValue} from '../data/helper/dataProvider'; import GlobalModel from './Global'; import { CoordinateSystem } from '../coord/CoordinateSystem'; @@ -57,10 +53,12 @@ import Axis from '../coord/Axis'; import { GradientObject } from 'zrender/src/graphic/Gradient'; import type { BrushCommonSelectorsForSeries, BrushSelectableArea } from '../component/brush/selector'; import makeStyleMapper from './mixin/makeStyleMapper'; +import { SourceManager } from '../data/helper/sourceManager'; const inner = modelUtil.makeInner<{ data: List dataBeforeProcessed: List + sourceManager: SourceManager }, SeriesModel>(); function getSelectionKey(data: List, dataIndex: number): string { @@ -139,7 +137,6 @@ class SeriesModel extends ComponentMode // Injected outside pipelineContext: PipelineContext; - // --------------------------------------- // Props to tell visual/style.ts about how to do visual encoding. // --------------------------------------- @@ -197,7 +194,8 @@ class SeriesModel extends ComponentMode this.mergeDefaultAndTheme(option, ecModel); - prepareSource(this); + const sourceManager = inner(this).sourceManager = new SourceManager(this); + sourceManager.prepareSource(); const data = this.getInitialData(option, ecModel); wrapData(data, this); @@ -273,7 +271,9 @@ class SeriesModel extends ComponentMode ); } - prepareSource(this); + const sourceManager = inner(this).sourceManager; + sourceManager.dirty(); + sourceManager.prepareSource(); const data = this.getInitialData(newSeriesOption, ecModel); wrapData(data, this); @@ -377,7 +377,7 @@ class SeriesModel extends ComponentMode } getSource(): Source { - return getSource(this); + return inner(this).sourceManager.getSource(); } /** diff --git a/src/util/conditionalExpression.ts b/src/util/conditionalExpression.ts new file mode 100644 index 000000000..b48a896d3 --- /dev/null +++ b/src/util/conditionalExpression.ts @@ -0,0 +1,536 @@ +/* +* Licensed to the Apache Software Foundation (ASF) under one +* or more contributor license agreements. See the NOTICE file +* distributed with this work for additional information +* regarding copyright ownership. The ASF licenses this file +* to you under the Apache License, Version 2.0 (the +* "License"); you may not use this file except in compliance +* with the License. You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an +* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +* KIND, either express or implied. See the License for the +* specific language governing permissions and limitations +* under the License. +*/ + +import { OptionDataValue, DimensionLoose, Dictionary } from './types'; +import { + createHashMap, keys, isArray, map, isObject, isString, trim, HashMap, isRegExp, isArrayLike +} from 'zrender/src/core/util'; +import { throwError, makePrintable } from './log'; +import { parseDate } from './number'; + + +// PENDING: +// (1) Support more parser like: `parse: 'trim'`, `parse: 'lowerCase'`, `parse: 'year'`, `parse: 'dayOfWeek'`? +// (2) Support piped parser ? +// (3) Support callback parser or callback condition? +// (4) At present do not support string expression yet but only stuctured expression. + + +/** + * The structured expression considered: + * (1) Literal simplicity + * (2) Sementic displayed clearly + * + * Sementic supports: + * (1) relational expression + * (2) logical expression + * + * For example: + * ```js + * { + * and: [{ + * or: [{ + * dimension: 'Year', gt: 2012, lt: 2019 + * }, { + * dimension: 'Year', '>': 2002, '<=': 2009 + * }] + * }, { + * dimension: 'Product', eq: 'Tofu' + * }] + * } + * + * { dimension: 'Product', eq: 'Tofu' } + * + * { + * or: [ + * { dimension: 'Product', value: 'Tofu' }, + * { dimension: 'Product', value: 'Biscuit' } + * ] + * } + * + * { + * and: [true] + * } + * ``` + * + * [PARSER] + * In an relation expression object, we can specify some built-in parsers: + * ```js + * // Trim if string + * { + * parse: 'trim', + * eq: 'Flowers' + * } + * // Parse as time and enable arithmetic relation comparison. + * { + * parse: 'time', + * lt: '2012-12-12' + * } + * // RegExp, include the feature in SQL: `like '%xxx%'`. + * { + * reg: /^asdf$/ + * } + * { + * reg: '^asdf$' // Serializable reg exp, will be `new RegExp(...)` + * } + * ``` + * + * + * [EMPTY_RULE] + * (1) If a relational expression set value as `null`/`undefined` like: + * `{ dimension: 'Product', lt: undefined }`, + * The result will be `false` rather than `true`. + * Consider the case like "filter condition", return all result when null/undefined + * is probably not expected and even dangours. + * (2) If a relational expression has no operator like: + * `{ dimension: 'Product' }`, + * An error will be thrown. Because it is probably a mistake. + * (3) If a logical expression has no children like + * `{ and: undefined }` or `{ and: [] }`, + * An error will be thrown. Because it is probably an mistake. + * (4) If intending have a condition that always `true` or always `false`, + * Use `true` or `flase`. + * The entire condition can be `true`/`false`, + * or also can be `{ and: [true] }`, `{ or: [false] }` + */ + + +// -------------------------------------------------- +// --- Relational Expression -------------------------- +// -------------------------------------------------- + +/** + * Date string and ordinal string can be accepted. + */ +interface RelationalExpressionOptionByOp { + lt?: OptionDataValue; // less than + lte?: OptionDataValue; // less than or equal + gt?: OptionDataValue; // greater than + gte?: OptionDataValue; // greater than or equal + eq?: OptionDataValue; // equal + ne?: OptionDataValue; // not equal + reg?: RegExp | string; // RegExp +}; +interface RelationalExpressionOptionByOpAlias { + value?: RelationalExpressionOptionByOp['eq']; + + '<'?: OptionDataValue; // lt + '<='?: OptionDataValue; // lte + '>'?: OptionDataValue; // gt + '>='?: OptionDataValue; // gte + '='?: OptionDataValue; // eq + '!='?: OptionDataValue; // ne + '<>'?: OptionDataValue; // ne (SQL style) + + // '=='?: OptionDataValue; // eq + // '==='?: OptionDataValue; // eq + // '!=='?: OptionDataValue; // eq + + // ge: RelationalExpressionOptionByOp['gte']; + // le: RelationalExpressionOptionByOp['lte']; + // neq: RelationalExpressionOptionByOp['ne']; +}; +const aliasToOpMap = createHashMap({ + value: 'eq', + + // PENDING: not good for literal semantic? + '<': 'lt', + '<=': 'lte', + '>': 'gt', + '>=': 'gte', + '=': 'eq', + '!=': 'ne', + '<>': 'ne' + + // Might mileading for sake of the different between '==' and '===', + // So dont support them. + // '==': 'eq', + // '===': 'seq', + // '!==': 'sne' + + // PENDING: Whether support some common alias "ge", "le", "neq"? + // ge: 'gte', + // le: 'lte', + // neq: 'ne', +}); + +type RelationalExpressionOp = keyof RelationalExpressionOptionByOp; +type RelationalExpressionOpAlias = keyof RelationalExpressionOptionByOpAlias; + +interface RelationalExpressionOption extends + RelationalExpressionOptionByOp, RelationalExpressionOptionByOpAlias { + dimension?: DimensionLoose; + parse?: RelationalExpressionValueParserType; +} + +type RelationalExpressionOpEvaluate = (tarVal: unknown, condVal: unknown) => boolean; + +const relationalOpEvaluateMap = createHashMap({ + // PENDING: should keep supporting string compare? + lt: function (tarVal, condVal) { + return tarVal < condVal; + }, + lte: function (tarVal, condVal) { + return tarVal <= condVal; + }, + gt: function (tarVal, condVal) { + return tarVal > condVal; + }, + gte: function (tarVal, condVal) { + return tarVal >= condVal; + }, + eq: function (tarVal, condVal) { + // eq is probably most used, DO NOT use JS ==, + // the rule is too complicated. + return tarVal === condVal; + }, + ne: function (tarVal, condVal) { + return tarVal !== condVal; + }, + reg: function (tarVal, condVal: RegExp) { + const type = typeof tarVal; + return type === 'string' ? condVal.test(tarVal as string) + : type === 'number' ? condVal.test(tarVal + '') + : false; + } +}); + +function parseRegCond(condVal: unknown): RegExp { + // Support condVal: RegExp | string + return isString(condVal) ? new RegExp(condVal) + : isRegExp(condVal) ? condVal as RegExp + : null; +} + +type RelationalExpressionValueParserType = 'time' | 'trim'; +type RelationalExpressionValueParser = (val: unknown) => unknown; +const valueParserMap = createHashMap({ + time: function (val): number { + // return timestamp. + return +parseDate(val); + }, + trim: function (val) { + return typeof val === 'string' ? trim(val) : val; + } +}); + + +// -------------------------------------------------- +// --- Logical Expression --------------------------- +// -------------------------------------------------- + + +interface LogicalExpressionOption { + and?: LogicalExpressionSubOption[]; + or?: LogicalExpressionSubOption[]; + not?: LogicalExpressionSubOption; +} +type LogicalExpressionSubOption = + LogicalExpressionOption | RelationalExpressionOption | TrueFalseExpressionOption; + + + +// ----------------------------------------------------- +// --- Conditional Expression -------------------------- +// ----------------------------------------------------- + + +export type TrueExpressionOption = true; +export type FalseExpressionOption = false; +export type TrueFalseExpressionOption = TrueExpressionOption | FalseExpressionOption; + +export type ConditionalExpressionOption = + LogicalExpressionOption + | RelationalExpressionOption + | TrueFalseExpressionOption; + +type ValueGetterParam = Dictionary; +export interface ConditionalExpressionValueGetterParamGetter { + (relExpOption: RelationalExpressionOption): VGP +} +export interface ConditionalExpressionValueGetter { + (param: VGP): OptionDataValue +} + +interface ParsedConditionInternal { + evaluate(): boolean; +} +class ConstConditionInternal implements ParsedConditionInternal { + value: boolean; + evaluate(): boolean { + return this.value; + } +} +class AndConditionInternal implements ParsedConditionInternal { + children: ParsedConditionInternal[]; + evaluate() { + const children = this.children; + for (let i = 0; i < children.length; i++) { + if (!children[i].evaluate()) { + return false; + } + } + return true; + } +} +class OrConditionInternal implements ParsedConditionInternal { + children: ParsedConditionInternal[]; + evaluate() { + const children = this.children; + for (let i = 0; i < children.length; i++) { + if (children[i].evaluate()) { + return true; + } + } + return false; + } +} +class NotConditionInternal implements ParsedConditionInternal { + child: ParsedConditionInternal; + evaluate() { + return !this.child.evaluate(); + } +} +class RelationalConditionInternal implements ParsedConditionInternal { + valueGetterParam: ValueGetterParam; + valueParser: RelationalExpressionValueParser; + // If no parser, be null/undefined. + getValue: ConditionalExpressionValueGetter; + subCondList: { + condValue: unknown; + evaluate: RelationalExpressionOpEvaluate; + }[]; + + evaluate() { + const getValue = this.getValue; + const needParse = !!this.valueParser; + // Call getValue with no `this`. + const tarValRaw = getValue(this.valueGetterParam); + const tarValParsed = needParse ? this.valueParser(tarValRaw) : null; + + // Relational cond follow "and" logic internally. + for (let i = 0; i < this.subCondList.length; i++) { + const subCond = this.subCondList[i]; + if ( + !subCond.evaluate( + needParse ? tarValParsed : tarValRaw, + subCond.condValue + ) + ) { + return false; + } + } + return true; + } +} + +function parseOption( + exprOption: ConditionalExpressionOption, + getters: ConditionalGetters +): ParsedConditionInternal { + if (exprOption === true || exprOption === false) { + const cond = new ConstConditionInternal(); + cond.value = exprOption as boolean; + return cond; + } + + let errMsg = ''; + if (!isObjectNotArray(exprOption)) { + if (__DEV__) { + errMsg = makePrintable( + 'Illegal config. Expect a plain object but actually', exprOption + ); + } + throwError(errMsg); + } + + if ((exprOption as LogicalExpressionOption).and) { + return parseAndOrOption('and', exprOption as LogicalExpressionOption, getters); + } + else if ((exprOption as LogicalExpressionOption).or) { + return parseAndOrOption('or', exprOption as LogicalExpressionOption, getters); + } + else if ((exprOption as LogicalExpressionOption).not) { + return parseNotOption(exprOption as LogicalExpressionOption, getters); + } + + return parseRelationalOption(exprOption as RelationalExpressionOption, getters); +} + +function parseAndOrOption( + op: 'and' | 'or', + exprOption: LogicalExpressionOption, + getters: ConditionalGetters +): ParsedConditionInternal { + const subOptionArr = exprOption[op] as ConditionalExpressionOption[]; + let errMsg = ''; + if (__DEV__) { + errMsg = makePrintable( + '"and"/"or" condition should only be `' + op + ': [...]` and must not be empty array.', + 'Illegal condition:', exprOption + ); + } + if (!isArray(subOptionArr)) { + throwError(errMsg); + } + if (!(subOptionArr as []).length) { + throwError(errMsg); + } + const cond = op === 'and' ? new AndConditionInternal() : new OrConditionInternal(); + cond.children = map(subOptionArr, subOption => parseOption(subOption, getters)); + if (!cond.children.length) { + throwError(errMsg); + } + return cond; +} + +function parseNotOption( + exprOption: LogicalExpressionOption, + getters: ConditionalGetters +): ParsedConditionInternal { + const subOption = exprOption.not as ConditionalExpressionOption; + let errMsg = ''; + if (__DEV__) { + errMsg = makePrintable( + '"not" condition should only be `not: {}`.', + 'Illegal condition:', exprOption + ); + } + if (!isObjectNotArray(subOption)) { + throwError(errMsg); + } + const cond = new NotConditionInternal(); + cond.child = parseOption(subOption, getters); + if (!cond.child) { + throwError(errMsg); + } + return cond; +} + +function parseRelationalOption( + exprOption: RelationalExpressionOption, + getters: ConditionalGetters +): ParsedConditionInternal { + let errMsg = ''; + + const valueGetterParam = getters.prepareGetValue(exprOption); + + const subCondList = [] as RelationalConditionInternal['subCondList']; + const exprKeys = keys(exprOption); + + const parserName = exprOption.parse; + const valueParser = parserName ? valueParserMap.get(parserName) : null; + + for (let i = 0; i < exprKeys.length; i++) { + const keyRaw = exprKeys[i]; + if (keyRaw === 'parse' || getters.valueGetterAttrMap.get(keyRaw)) { + continue; + } + + const op: RelationalExpressionOp = aliasToOpMap.get(keyRaw as RelationalExpressionOpAlias) + || (keyRaw as RelationalExpressionOp); + const evaluateHandler = relationalOpEvaluateMap.get(op); + + if (!evaluateHandler) { + if (__DEV__) { + errMsg = makePrintable( + 'Illegal relational operation: "' + keyRaw + '" in condition:', exprOption + ); + } + throwError(errMsg); + } + + const condValueRaw = exprOption[keyRaw]; + let condValue; + if (keyRaw === 'reg') { + condValue = parseRegCond(condValueRaw); + if (condValue == null) { + let errMsg = ''; + if (__DEV__) { + errMsg = makePrintable('Illegal regexp', condValueRaw, 'in', exprOption); + } + throwError(errMsg); + } + } + else { + // At present, all other operators are applicable `RelationalExpressionValueParserType`. + // But if adding new parser, we should check it again. + condValue = valueParser ? valueParser(condValueRaw) : condValueRaw; + } + + subCondList.push({ + condValue: condValue, + evaluate: evaluateHandler + }); + } + + if (!subCondList.length) { + if (__DEV__) { + errMsg = makePrintable( + 'Relational condition must have at least one operator.', + 'Illegal condition:', exprOption + ); + } + // No relational operator always disabled in case of dangers result. + throwError(errMsg); + } + + const cond = new RelationalConditionInternal(); + cond.valueGetterParam = valueGetterParam; + cond.valueParser = valueParser; + cond.getValue = getters.getValue; + cond.subCondList = subCondList; + + return cond; +} + +function isObjectNotArray(val: unknown): boolean { + return isObject(val) && !isArrayLike(val); +} + + +class ConditionalExpressionParsed { + + private _cond: ParsedConditionInternal; + + constructor( + exprOption: ConditionalExpressionOption, + getters: ConditionalGetters + ) { + this._cond = parseOption(exprOption, getters); + } + + evaluate(): boolean { + return this._cond.evaluate(); + } +}; + +interface ConditionalGetters { + prepareGetValue: ConditionalExpressionValueGetterParamGetter; + getValue: ConditionalExpressionValueGetter; + valueGetterAttrMap: HashMap; +} + +export function parseConditionalExpression( + exprOption: ConditionalExpressionOption, + getters: ConditionalGetters +): ConditionalExpressionParsed { + return new ConditionalExpressionParsed(exprOption, getters); +} + diff --git a/src/util/ecData.ts b/src/util/ecData.ts index 2dc31fcf9..fcd176d5c 100644 --- a/src/util/ecData.ts +++ b/src/util/ecData.ts @@ -18,7 +18,7 @@ */ import Element from 'zrender/src/Element'; -import { DataModel, ECEventData, BlurScope, InnerFocus } from './types'; +import { DataModel, ECEventData, BlurScope, InnerFocus, SeriesDataType } from './types'; import { makeInner } from './model'; /** * ECData stored on graphic element @@ -28,7 +28,7 @@ export interface ECData { dataModel?: DataModel; eventData?: ECEventData; seriesIndex?: number; - dataType?: string; + dataType?: SeriesDataType; focus?: InnerFocus; blurScope?: BlurScope; } diff --git a/src/util/log.ts b/src/util/log.ts index c841bacca..3aaab2d89 100644 --- a/src/util/log.ts +++ b/src/util/log.ts @@ -18,6 +18,7 @@ */ import { Dictionary } from './types'; +import { map, isString, isFunction, eqNaN, isRegExp } from 'zrender/src/core/util'; const storedLogs: Dictionary = {}; @@ -37,4 +38,68 @@ export function deprecateReplaceLog(oldOpt: string, newOpt: string, scope?: stri if (__DEV__) { deprecateLog((scope ? `[${scope}]` : '') + `${oldOpt} is deprecated, use ${newOpt} instead.`); } -} \ No newline at end of file +} + +export function consoleLog(...args: unknown[]) { + if (__DEV__) { + /* eslint-disable no-console */ + if (typeof console !== 'undefined' && console.log) { + console.log.apply(console, args); + } + /* eslint-enable no-console */ + } +} + +/** + * If in __DEV__ environment, get console printable message for users hint. + * Parameters are separated by ' '. + * @usuage + * makePrintable('This is an error on', someVar, someObj); + * + * @param hintInfo anything about the current execution context to hint users. + * @throws Error + */ +export function makePrintable(...hintInfo: unknown[]) { + let msg = ''; + + if (__DEV__) { + // Fuzzy stringify for print. + // This code only exist in dev environment. + msg = map(hintInfo, arg => { + if (isString(arg)) { + // Print without quotation mark for some statement. + return arg; + } + else if (typeof JSON !== 'undefined' && JSON.stringify) { + try { + return JSON.stringify(arg, function (n, val) { + return val === void 0 ? 'undefined' + : val === Infinity ? 'Infinity' + : val === -Infinity ? '-Infinity' + : eqNaN(val) ? 'NaN' + : val instanceof Date ? 'Date(' + val.toISOString() + ')' + : isFunction(val) ? 'function () { ... }' + : isRegExp(val) ? val + '' + : val; + }); + // In most cases the info object is small, so do not line break. + } + catch (err) { + return '?'; + } + } + else { + return '?'; + } + }).join(' '); + } + + return msg; +} + +/** + * @throws Error + */ +export function throwError(msg?: string) { + throw new Error(msg); +} diff --git a/src/util/model.ts b/src/util/model.ts index ee975c5f6..699b297de 100644 --- a/src/util/model.ts +++ b/src/util/model.ts @@ -791,7 +791,7 @@ export function parseFinder( } const defaultMainType = opt ? opt.defaultMainType : null; - const queryOptionMap = createHashMap(); + const queryOptionMap = createHashMap(); const result = {} as ParsedModelFinder; each(finder, function (value, key) { @@ -803,7 +803,7 @@ export function parseFinder( const parsedKey = key.match(/^(\w+)(Index|Id|Name)$/) || []; const mainType = parsedKey[1]; - const queryType = (parsedKey[2] || '').toLowerCase() as keyof QueryReferringOption; + const queryType = (parsedKey[2] || '').toLowerCase() as keyof QueryReferringUserOption; if ( !mainType @@ -836,7 +836,7 @@ export function parseFinder( return result; } -type QueryReferringOption = { +export type QueryReferringUserOption = { index?: ModelFinderIndexQuery, id?: ModelFinderIdQuery, name?: ModelFinderNameQuery, @@ -857,7 +857,7 @@ export type QueryReferringOpt = { export function queryReferringComponents( ecModel: GlobalModel, mainType: ComponentMainType, - userOption: QueryReferringOption, + userOption: QueryReferringUserOption, opt: QueryReferringOpt ): { // Always be array rather than null/undefined, which is convenient to use. diff --git a/src/util/number.ts b/src/util/number.ts index d30353497..68cad8b89 100644 --- a/src/util/number.ts +++ b/src/util/number.ts @@ -286,7 +286,8 @@ export function isRadianAroundZero(val: number): boolean { const TIME_REG = /^(?:(\d{4})(?:[-\/](\d{1,2})(?:[-\/](\d{1,2})(?:[T ](\d{1,2})(?::(\d{1,2})(?::(\d{1,2})(?:[.,](\d+))?)?)?(Z|[\+\-]\d\d:?\d\d)?)?)?)?)?$/; // jshint ignore:line /** - * @param value These values can be accepted: + * @param value valid type: number | string | Date, otherwise return `new Date(NaN)` + * These values can be accepted: * + An instance of Date, represent a time in its own time zone. * + Or string in a subset of ISO 8601, only including: * + only year, month, date: '2012-03', '2012-03-01', '2012-03-01 05', '2012-03-01 05:06', @@ -298,9 +299,9 @@ const TIME_REG = /^(?:(\d{4})(?:[-\/](\d{1,2})(?:[-\/](\d{1,2})(?:[T ](\d{1,2})( * '2012', '2012-3-1', '2012/3/1', '2012/03/01', * '2009/6/12 2:00', '2009/6/12 2:05:08', '2009/6/12 2:05:08.123' * + a timestamp, which represent a time in UTC. - * @return date + * @return date Never be null/undefined. If invalid, return `new Date(NaN)`. */ -export function parseDate(value: number | string | Date): Date { +export function parseDate(value: unknown): Date { if (value instanceof Date) { return value; } @@ -358,7 +359,7 @@ export function parseDate(value: number | string | Date): Date { return new Date(NaN); } - return new Date(Math.round(value)); + return new Date(Math.round(value as number)); } /** diff --git a/src/util/types.ts b/src/util/types.ts index c40b28106..f475a7cc4 100644 --- a/src/util/types.ts +++ b/src/util/types.ts @@ -372,7 +372,9 @@ export const SERIES_LAYOUT_BY_ROW = 'row' as const; export type SeriesLayoutBy = typeof SERIES_LAYOUT_BY_COLUMN | typeof SERIES_LAYOUT_BY_ROW; // null/undefined/'auto': auto detect header, see "src/data/helper/sourceHelper". -export type OptionSourceHeader = boolean | 'auto'; +// If number, means header lines count, or say, `startIndex`. +// Like `sourceHeader: 2`, means line 0 and line 1 are header, data start from line 2. +export type OptionSourceHeader = boolean | 'auto' | number; export type SeriesDataType = 'main' | 'node' | 'edge'; @@ -1395,8 +1397,11 @@ export interface SeriesSamplingOptionMixin { export interface SeriesEncodeOptionMixin { datasetIndex?: number; + datasetId?: string | number; seriesLayoutBy?: SeriesLayoutBy; sourceHeader?: OptionSourceHeader; dimensions?: DimensionDefinitionLoose[]; encode?: OptionEncode } + +export type SeriesEncodableModel = SeriesModel; diff --git a/test/data-transform.html b/test/data-transform.html new file mode 100644 index 000000000..05ada217b --- /dev/null +++ b/test/data-transform.html @@ -0,0 +1,661 @@ + + + + + + + + + + + + + + + + + + + + + + +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + -- GitLab