Skip to content

TDengine
TDengine

taosdata / TDengine
1 年多 前同步成功

通知 1185
Star 22016
Fork 4786
TDengine
TDengine
未验证 提交 f0b77fa3 编写于 作者: H Huo Linhe 提交者： GitHub
浏览文件
操作

[TD-4258]<fix>: speedup nodejs fetchall() with taos_fetch_block (#6165)

上级 4697cdfd
展开全部 隐藏空白更改
内联 并排
Showing with 189 addition and 475 deletion
src/connector/nodejs/nodetaos/cursor.js
浏览文件 @ f0b77fa3
const ref = require('ref-napi');
require('./globalfunc.js')
const CTaosInterface = require('./cinterface')
const errors = require ('./error')
const errors = require('./error')
const TaosQuery = require('./taosquery')
const { PerformanceObserver, performance } = require('perf_hooks');
module.exports = TDengineCursor;
......@@ -22,7 +22,7 @@ module.exports = TDengineCursor;
* @property {fields} - Array of the field objects in order from left to right of the latest data retrieved
* @since 1.0.0
*/
function TDengineCursor(connection=null) {
function TDengineCursor(connection = null) {
//All parameters are store for sync queries only.
this._rowcount = -1;
this._connection = null;
......@@ -91,7 +91,7 @@ TDengineCursor.prototype.execute = function execute(operation, options, callback
return null;
}
if (typeof options == 'function') {
if (typeof options == 'function') {
callback = options;
}
if (typeof options != 'object') options = {}
......@@ -144,10 +144,10 @@ TDengineCursor.prototype.execute = function execute(operation, options, callback
}
TDengineCursor.prototype._createAffectedResponse = function (num, time) {
return "Query OK, " + num + " row(s) affected (" + (time * 0.001).toFixed(8) + "s)";
return "Query OK, " + num + " row(s) affected (" + (time * 0.001).toFixed(8) + "s)";
}
TDengineCursor.prototype._createSetResponse = function (num, time) {
return "Query OK, " + num + " row(s) in set (" + (time * 0.001).toFixed(8) + "s)";
return "Query OK, " + num + " row(s) in set (" + (time * 0.001).toFixed(8) + "s)";
}
TDengineCursor.prototype.executemany = function executemany() {
......@@ -176,27 +176,22 @@ TDengineCursor.prototype.fetchall = function fetchall(options, callback) {
throw new errors.OperationalError("Invalid use of fetchall, either result or fields from query are null. First execute a query first");
}
let data = [];
let num_of_rows = this._chandle.affectedRows(this._result);
let data = new Array(num_of_rows);
this._rowcount = 0;
//let nodetime = 0;
let time = 0;
const obs = new PerformanceObserver((items) => {
time += items.getEntries()[0].duration;
performance.clearMarks();
});
/*
const obs2 = new PerformanceObserver((items) => {
nodetime += items.getEntries()[0].duration;
performance.clearMarks();
});
obs2.observe({ entryTypes: ['measure'] });
performance.mark('nodea');
*/
obs.observe({ entryTypes: ['measure'] });
performance.mark('A');
while(true) {
while (true) {
let blockAndRows = this._chandle.fetchBlock(this._result, this._fields);
// console.log(blockAndRows);
// break;
let block = blockAndRows.blocks;
let num_of_rows = blockAndRows.num_of_rows;
if (num_of_rows == 0) {
......@@ -205,22 +200,24 @@ TDengineCursor.prototype.fetchall = function fetchall(options, callback) {
this._rowcount += num_of_rows;
let numoffields = this._fields.length;
for (let i = 0; i < num_of_rows; i++) {
data.push([]);
// data.push([]);
let rowBlock = new Array(numoffields);
for (let j = 0; j < numoffields; j++) {
rowBlock[j] = block[j][i];
}
data[data.length-1] = (rowBlock);
data[this._rowcount - num_of_rows + i] = (rowBlock);
// data.push(rowBlock);
}
}
performance.mark('B');
performance.measure('query', 'A', 'B');
let response = this._createSetResponse(this._rowcount, time)
console.log(response);
// this._connection._clearResultSet();
// this._connection._clearResultSet();
let fields = this.fields;
this._reset_result();
this.data = data;
......@@ -239,12 +236,12 @@ TDengineCursor.prototype.fetchall = function fetchall(options, callback) {
* @return {number | Buffer} Number of affected rows or a Buffer that points to the results of the query
* @since 1.0.0
*/
TDengineCursor.prototype.execute_a = function execute_a (operation, options, callback, param) {
TDengineCursor.prototype.execute_a = function execute_a(operation, options, callback, param) {
if (operation == undefined) {
throw new errors.ProgrammingError('No operation passed as argument');
return null;
}
if (typeof options == 'function') {
if (typeof options == 'function') {
//we expect the parameter after callback to be param
param = callback;
callback = options;
......@@ -265,14 +262,14 @@ TDengineCursor.prototype.execute_a = function execute_a (operation, options, cal
}
if (resCode >= 0) {
// let fieldCount = cr._chandle.numFields(res2);
// if (fieldCount == 0) {
// //cr._chandle.freeResult(res2);
// return res2;
// }
// else {
// return res2;
// }
// let fieldCount = cr._chandle.numFields(res2);
// if (fieldCount == 0) {
// //cr._chandle.freeResult(res2);
// return res2;
// }
// else {
// return res2;
// }
return res2;
}
......@@ -317,7 +314,7 @@ TDengineCursor.prototype.execute_a = function execute_a (operation, options, cal
* })
*/
TDengineCursor.prototype.fetchall_a = function fetchall_a(result, options, callback, param = {}) {
if (typeof options == 'function') {
if (typeof options == 'function') {
//we expect the parameter after callback to be param
param = callback;
callback = options;
......@@ -360,17 +357,17 @@ TDengineCursor.prototype.fetchall_a = function fetchall_a(result, options, callb
for (let k = 0; k < fields.length; k++) {
rowBlock[k] = block[k][j];
}
data[data.length-1] = rowBlock;
data[data.length - 1] = rowBlock;
}
}
cr._chandle.freeResult(result2); // free result, avoid seg faults and mem leaks!
callback(param2, result2, numOfRows2, {data:data,fields:fields});
callback(param2, result2, numOfRows2, { data: data, fields: fields });
}
}
ref.writeObject(buf, 0, param);
param = this._chandle.fetch_rows_a(result, asyncCallbackWrapper, buf); //returned param
return {param:param,result:result};
return { param: param, result: result };
}
/**
* Stop a query given the result handle.
......@@ -428,7 +425,7 @@ TDengineCursor.prototype.subscribe = function subscribe(config) {
*/
TDengineCursor.prototype.consumeData = async function consumeData(subscription, callback) {
while (true) {
let { data, fields, result} = this._chandle.consume(subscription);
let { data, fields, result } = this._chandle.consume(subscription);
callback(data, fields, result);
}
}
......@@ -450,30 +447,30 @@ TDengineCursor.prototype.unsubscribe = function unsubscribe(subscription) {
* @return {Buffer} A buffer pointing to the stream handle
* @since 1.3.0
*/
TDengineCursor.prototype.openStream = function openStream(sql, callback, stime = 0, stoppingCallback, param = {}) {
let buf = ref.alloc('Object');
ref.writeObject(buf, 0, param);
TDengineCursor.prototype.openStream = function openStream(sql, callback, stime = 0, stoppingCallback, param = {}) {
let buf = ref.alloc('Object');
ref.writeObject(buf, 0, param);
let asyncCallbackWrapper = function (param2, result2, blocks, fields) {
let data = [];
let num_of_rows = blocks[0].length;
for (let j = 0; j < num_of_rows; j++) {
data.push([]);
let rowBlock = new Array(fields.length);
for (let k = 0; k < fields.length; k++) {
rowBlock[k] = blocks[k][j];
}
data[data.length-1] = rowBlock;
}
callback(param2, result2, blocks, fields);
}
return this._chandle.openStream(this._connection._conn, sql, asyncCallbackWrapper, stime, stoppingCallback, buf);
}
/**
* Close a stream
* @param {Buffer} - A buffer pointing to the handle of the stream to be closed
* @since 1.3.0
*/
TDengineCursor.prototype.closeStream = function closeStream(stream) {
this._chandle.closeStream(stream);
}
let asyncCallbackWrapper = function (param2, result2, blocks, fields) {
let data = [];
let num_of_rows = blocks[0].length;
for (let j = 0; j < num_of_rows; j++) {
data.push([]);
let rowBlock = new Array(fields.length);
for (let k = 0; k < fields.length; k++) {
rowBlock[k] = blocks[k][j];
}
data[data.length - 1] = rowBlock;
}
callback(param2, result2, blocks, fields);
}
return this._chandle.openStream(this._connection._conn, sql, asyncCallbackWrapper, stime, stoppingCallback, buf);
}
/**
* Close a stream
* @param {Buffer} - A buffer pointing to the handle of the stream to be closed
* @since 1.3.0
*/
TDengineCursor.prototype.closeStream = function closeStream(stream) {
this._chandle.closeStream(stream);
}
src/connector/nodejs/package-lock.json 已删除 100644 → 0
浏览文件 @ 4697cdfd
{
"name": "td2.0-connector",
"version": "2.0.6",
"lockfileVersion": 1,
"requires": true,
"dependencies": {
"array-index": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/array-index/-/array-index-1.0.0.tgz",
"integrity": "sha1-7FanSe4QPk4Ix5C5w1PfFgVbl/k=",
"requires": {
"debug": "^2.2.0",
"es6-symbol": "^3.0.2"
},
"dependencies": {
"debug": {
"version": "2.6.9",
"resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz",
"integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==",
"requires": {
"ms": "2.0.0"
}
},
"ms": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz",
"integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g="
}
}
},
"d": {
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/d/-/d-1.0.1.tgz",
"integrity": "sha512-m62ShEObQ39CfralilEQRjH6oAMtNCV1xJyEx5LpRYUVN+EviphDgUc/F3hnYbADmkiNs67Y+3ylmlG7Lnu+FA==",
"requires": {
"es5-ext": "^0.10.50",
"type": "^1.0.1"
}
},
"debug": {
"version": "4.3.1",
"resolved": "https://registry.npmjs.org/debug/-/debug-4.3.1.tgz",
"integrity": "sha512-doEwdvm4PCeK4K3RQN2ZC2BYUBaxwLARCqZmMjtF8a51J2Rb0xpVloFRnCODwqjpwnAoao4pelN8l3RJdv3gRQ==",
"requires": {
"ms": "2.1.2"
}
},
"es5-ext": {
"version": "0.10.53",
"resolved": "https://registry.npmjs.org/es5-ext/-/es5-ext-0.10.53.tgz",
"integrity": "sha512-Xs2Stw6NiNHWypzRTY1MtaG/uJlwCk8kH81920ma8mvN8Xq1gsfhZvpkImLQArw8AHnv8MT2I45J3c0R8slE+Q==",
"requires": {
"es6-iterator": "~2.0.3",
"es6-symbol": "~3.1.3",
"next-tick": "~1.0.0"
}
},
"es6-iterator": {
"version": "2.0.3",
"resolved": "https://registry.npmjs.org/es6-iterator/-/es6-iterator-2.0.3.tgz",
"integrity": "sha1-p96IkUGgWpSwhUQDstCg+/qY87c=",
"requires": {
"d": "1",
"es5-ext": "^0.10.35",
"es6-symbol": "^3.1.1"
}
},
"es6-symbol": {
"version": "3.1.3",
"resolved": "https://registry.npmjs.org/es6-symbol/-/es6-symbol-3.1.3.tgz",
"integrity": "sha512-NJ6Yn3FuDinBaBRWl/q5X/s4koRHBrgKAu+yGI6JCBeiu3qrcbJhwT2GeR/EXVfylRk8dpQVJoLEFhK+Mu31NA==",
"requires": {
"d": "^1.0.1",
"ext": "^1.1.2"
}
},
"ext": {
"version": "1.4.0",
"resolved": "https://registry.npmjs.org/ext/-/ext-1.4.0.tgz",
"integrity": "sha512-Key5NIsUxdqKg3vIsdw9dSuXpPCQ297y6wBjL30edxwPgt2E44WcWBZey/ZvUc6sERLTxKdyCu4gZFmUbk1Q7A==",
"requires": {
"type": "^2.0.0"
},
"dependencies": {
"type": {
"version": "2.1.0",
"resolved": "https://registry.npmjs.org/type/-/type-2.1.0.tgz",
"integrity": "sha512-G9absDWvhAWCV2gmF1zKud3OyC61nZDwWvBL2DApaVFogI07CprggiQAOOjvp2NRjYWFzPyu7vwtDrQFq8jeSA=="
}
}
},
"ffi-napi": {
"version": "3.1.0",
"resolved": "https://registry.npmjs.org/ffi-napi/-/ffi-napi-3.1.0.tgz",
"integrity": "sha512-EsHO+sP2p/nUC/3l/l8m9niee1BLm4asUFDzkkBGR4kYVgp2KqdAYUomZhkKtzim4Fq7mcYHjpUaIHsMqs+E1g==",
"requires": {
"debug": "^4.1.1",
"get-uv-event-loop-napi-h": "^1.0.5",
"node-addon-api": "^2.0.0",
"node-gyp-build": "^4.2.1",
"ref-napi": "^2.0.1",
"ref-struct-di": "^1.1.0"
},
"dependencies": {
"ref-napi": {
"version": "2.1.2",
"resolved": "https://registry.npmjs.org/ref-napi/-/ref-napi-2.1.2.tgz",
"integrity": "sha512-aFl+vrIuLWUXMUTQGAwGAuSNLX3Ub5W3iVP8b7KyFFZUdn4+i4U1TXXTop0kCTUfGNu8glBGVz4lowkwMcPVVA==",
"requires": {
"debug": "^4.1.1",
"get-symbol-from-current-process-h": "^1.0.2",
"node-addon-api": "^2.0.0",
"node-gyp-build": "^4.2.1"
}
}
}
},
"get-symbol-from-current-process-h": {
"version": "1.0.2",
"resolved": "https://registry.npmjs.org/get-symbol-from-current-process-h/-/get-symbol-from-current-process-h-1.0.2.tgz",
"integrity": "sha512-syloC6fsCt62ELLrr1VKBM1ggOpMdetX9hTrdW77UQdcApPHLmf7CI7OKcN1c9kYuNxKcDe4iJ4FY9sX3aw2xw=="
},
"get-uv-event-loop-napi-h": {
"version": "1.0.6",
"resolved": "https://registry.npmjs.org/get-uv-event-loop-napi-h/-/get-uv-event-loop-napi-h-1.0.6.tgz",
"integrity": "sha512-t5c9VNR84nRoF+eLiz6wFrEp1SE2Acg0wS+Ysa2zF0eROes+LzOfuTaVHxGy8AbS8rq7FHEJzjnCZo1BupwdJg==",
"requires": {
"get-symbol-from-current-process-h": "^1.0.1"
}
},
"ms": {
"version": "2.1.2",
"resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz",
"integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w=="
},
"next-tick": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/next-tick/-/next-tick-1.0.0.tgz",
"integrity": "sha1-yobR/ogoFpsBICCOPchCS524NCw="
},
"node-addon-api": {
"version": "2.0.2",
"resolved": "https://registry.npmjs.org/node-addon-api/-/node-addon-api-2.0.2.tgz",
"integrity": "sha512-Ntyt4AIXyaLIuMHF6IOoTakB3K+RWxwtsHNRxllEoA6vPwP9o4866g6YWDLUdnucilZhmkxiHwHr11gAENw+QA=="
},
"node-gyp-build": {
"version": "4.2.3",
"resolved": "https://registry.npmjs.org/node-gyp-build/-/node-gyp-build-4.2.3.tgz",
"integrity": "sha512-MN6ZpzmfNCRM+3t57PTJHgHyw/h4OWnZ6mR8P5j/uZtqQr46RRuDE/P+g3n0YR/AiYXeWixZZzaip77gdICfRg=="
},
"ref-array-napi": {
"version": "1.2.1",
"resolved": "https://registry.npmjs.org/ref-array-napi/-/ref-array-napi-1.2.1.tgz",
"integrity": "sha512-jQp2WWSucmxkqVfoNfm7yDlDeGu3liAbzqfwjNybL80ooLOCnCZpAK2woDInY+lxNOK/VlIVSqeDEYb4gVPuNQ==",
"requires": {
"array-index": "1",
"debug": "2",
"ref-napi": "^1.4.2"
},
"dependencies": {
"debug": {
"version": "2.6.9",
"resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz",
"integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==",
"requires": {
"ms": "2.0.0"
}
},
"ms": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz",
"integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g="
},
"ref-napi": {
"version": "1.5.2",
"resolved": "https://registry.npmjs.org/ref-napi/-/ref-napi-1.5.2.tgz",
"integrity": "sha512-hwyNmWpUkt1bDWDW4aiwCoC+SJfJO69UIdjqssNqdaS0sYJpgqzosGg/rLtk69UoQ8drZdI9yyQefM7eEMM3Gw==",
"requires": {
"debug": "^3.1.0",
"node-addon-api": "^2.0.0",
"node-gyp-build": "^4.2.1"
},
"dependencies": {
"debug": {
"version": "3.2.7",
"resolved": "https://registry.npmjs.org/debug/-/debug-3.2.7.tgz",
"integrity": "sha512-CFjzYYAi4ThfiQvizrFQevTTXHtnCqWfe7x1AhgEscTz6ZbLbfoLRLPugTQyBth6f8ZERVUSyWHFD/7Wu4t1XQ==",
"requires": {
"ms": "^2.1.1"
}
},
"ms": {
"version": "2.1.3",
"resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz",
"integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA=="
}
}
}
}
},
"ref-napi": {
"version": "3.0.1",
"resolved": "https://registry.npmjs.org/ref-napi/-/ref-napi-3.0.1.tgz",
"integrity": "sha512-W3rcb0E+tlO9u9ySFnX5vifInwwPGToOfFgTZUHJBNiOBsW0NNvgHz2zJN7ctABo/2yIlgdPQUvuqqfORIF4LA==",
"requires": {
"debug": "^4.1.1",
"get-symbol-from-current-process-h": "^1.0.2",
"node-addon-api": "^2.0.0",
"node-gyp-build": "^4.2.1"
}
},
"ref-struct-di": {
"version": "1.1.1",
"resolved": "https://registry.npmjs.org/ref-struct-di/-/ref-struct-di-1.1.1.tgz",
"integrity": "sha512-2Xyn/0Qgz89VT+++WP0sTosdm9oeowLP23wRJYhG4BFdMUrLj3jhwHZNEytYNYgtPKLNTP3KJX4HEgBvM1/Y2g==",
"requires": {
"debug": "^3.1.0"
},
"dependencies": {
"debug": {
"version": "3.2.7",
"resolved": "https://registry.npmjs.org/debug/-/debug-3.2.7.tgz",
"integrity": "sha512-CFjzYYAi4ThfiQvizrFQevTTXHtnCqWfe7x1AhgEscTz6ZbLbfoLRLPugTQyBth6f8ZERVUSyWHFD/7Wu4t1XQ==",
"requires": {
"ms": "^2.1.1"
}
}
}
},
"ref-struct-napi": {
"version": "1.1.1",
"resolved": "https://registry.npmjs.org/ref-struct-napi/-/ref-struct-napi-1.1.1.tgz",
"integrity": "sha512-YgS5/d7+kT5zgtySYI5ieH0hREdv+DabgDvoczxsui0f9VLm0rrDcWEj4DHKehsH+tJnVMsLwuyctWgvdEcVRw==",
"requires": {
"debug": "2",
"ref-napi": "^1.4.2"
},
"dependencies": {
"debug": {
"version": "2.6.9",
"resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz",
"integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==",
"requires": {
"ms": "2.0.0"
}
},
"ms": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz",
"integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g="
},
"ref-napi": {
"version": "1.5.2",
"resolved": "https://registry.npmjs.org/ref-napi/-/ref-napi-1.5.2.tgz",
"integrity": "sha512-hwyNmWpUkt1bDWDW4aiwCoC+SJfJO69UIdjqssNqdaS0sYJpgqzosGg/rLtk69UoQ8drZdI9yyQefM7eEMM3Gw==",
"requires": {
"debug": "^3.1.0",
"node-addon-api": "^2.0.0",
"node-gyp-build": "^4.2.1"
},
"dependencies": {
"debug": {
"version": "3.2.7",
"resolved": "https://registry.npmjs.org/debug/-/debug-3.2.7.tgz",
"integrity": "sha512-CFjzYYAi4ThfiQvizrFQevTTXHtnCqWfe7x1AhgEscTz6ZbLbfoLRLPugTQyBth6f8ZERVUSyWHFD/7Wu4t1XQ==",
"requires": {
"ms": "^2.1.1"
}
},
"ms": {
"version": "2.1.3",
"resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz",
"integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA=="
}
}
}
}
},
"type": {
"version": "1.2.0",
"resolved": "https://registry.npmjs.org/type/-/type-1.2.0.tgz",
"integrity": "sha512-+5nt5AAniqsCnu2cEQQdpzCAh33kVx8n0VoFidKpB1dVVLAN/F+bgVOqOJqOnEnrhp222clB5p3vUlD+1QAnfg=="
}
}
}
src/connector/nodejs/package.json
浏览文件 @ f0b77fa3
{
"name": "td2.0-connector",
"version": "2.0.6",
"version": "2.0.7",
"description": "A Node.js connector for TDengine.",
"main": "tdengine.js",
"directories": {
......
Markdown is supported
0% 或 .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑！
想要评论请 注册 或