Skip to content

T
TDengine

taosdata / TDengine
大约 1 年 前同步成功

通知 1185
Star 22015
Fork 4786
T
TDengine

体验新版 GitCode，发现更多精彩内容 >>

提交 41290f9a 编写于 作者: S Shengliang Guan
浏览文件
操作

Merge remote-tracking branch 'origin/3.0' into feature/dnode3

上级 67d3e569 caad930c
展开全部 显示空白变更内容
内联 并排
Showing with 18299 addition and 943 deletion
Jenkinsfile
浏览文件 @ 41290f9a
......@@ -41,72 +41,72 @@ def pre_test(){
killall -9 taosd ||echo "no taosd running"
killall -9 gdb || echo "no gdb running"
killall -9 python3.8 || echo "no python program running"
cd ${WORKSPACE}
cd ${WKC}
git reset --hard HEAD~10 >/dev/null
'''
script {
if (env.CHANGE_TARGET == 'master') {
sh '''
cd ${WORKSPACE}
cd ${WKC}
git checkout master
'''
}
else if(env.CHANGE_TARGET == '2.0'){
sh '''
cd ${WORKSPACE}
cd ${WKC}
git checkout 2.0
'''
}
else if(env.CHANGE_TARGET == '3.0'){
sh '''
cd ${WORKSPACE}
cd ${WKC}
git checkout 3.0
'''
}
else{
sh '''
cd ${WORKSPACE}
cd ${WKC}
git checkout develop
'''
}
}
sh'''
cd ${WORKSPACE}
cd ${WKC}
git pull >/dev/null
git fetch origin +refs/pull/${CHANGE_ID}/merge
git checkout -qf FETCH_HEAD
git clean -dfx
'''
// script {
// if (env.CHANGE_TARGET == 'master') {
// sh '''
// cd ${WK}
// git checkout master
// '''
// }
// else if(env.CHANGE_TARGET == '2.0'){
// sh '''
// cd ${WK}
// git checkout 2.0
// '''
// }
// else if(env.CHANGE_TARGET == '3.0'){
// sh '''
// cd ${WK}
// git checkout 3.0
// '''
// }
// else{
// sh '''
// cd ${WK}
// git checkout develop
// '''
// }
// }
// sh '''
// cd ${WK}
// git pull >/dev/null
script {
if (env.CHANGE_TARGET == 'master') {
sh '''
cd ${WK}
git checkout master
'''
}
else if(env.CHANGE_TARGET == '2.0'){
sh '''
cd ${WK}
git checkout 2.0
'''
}
else if(env.CHANGE_TARGET == '3.0'){
sh '''
cd ${WK}
git checkout 3.0
'''
}
else{
sh '''
cd ${WK}
git checkout develop
'''
}
}
sh '''
cd ${WK}
git pull >/dev/null
git clean -dfx
export TZ=Asia/Harbin
date
mkdir debug
......@@ -138,270 +138,228 @@ pipeline {
abort_previous()
abortPreviousBuilds()
}
sh'''
rm -rf ${WORKSPACE}.tes
cp -r ${WORKSPACE} ${WORKSPACE}.tes
cd ${WORKSPACE}.tes
git fetch
'''
script {
if (env.CHANGE_TARGET == 'master') {
sh '''
git checkout master
'''
}
else if(env.CHANGE_TARGET == '2.0'){
sh '''
git checkout 2.0
'''
}
else if(env.CHANGE_TARGET == '3.0'){
sh '''
git checkout 3.0
'''
}
else{
sh '''
git checkout develop
'''
}
}
sh'''
git fetch origin +refs/pull/${CHANGE_ID}/merge
git checkout -qf FETCH_HEAD
'''
script{
skipbuild='2'
skipbuild=sh(script: "git log -2 --pretty=%B | fgrep -ie '[skip ci]' -e '[ci skip]' && echo 1 || echo 2", returnStdout:true)
println skipbuild
}
sh'''
rm -rf ${WORKSPACE}.tes
'''
}
}
stage('Parallel test stage') {
//only build pr
options { skipDefaultCheckout() }
when {
allOf{
changeRequest()
expression{
return skipbuild.trim() == '2'
}
pre_test()
}
}
parallel {
stage('python_1_s1') {
agent{label " slave1 || slave11 "}
steps {
// stage('Parallel test stage') {
// //only build pr
// options { skipDefaultCheckout() }
// when {
// allOf{
// changeRequest()
pre_test()
// timeout(time: 55, unit: 'MINUTES'){
// sh '''
// date
// cd ${WKC}/tests
// ./test-all.sh p1
// date'''
// }
// }
// parallel {
// stage('python_1_s1') {
// agent{label " slave1 || slave11 "}
// steps {
}
}
stage('python_2_s5') {
agent{label " slave5 || slave15 "}
steps {
// pre_test()
// // timeout(time: 55, unit: 'MINUTES'){
// // sh '''
// // date
// // cd ${WKC}/tests
// // ./test-all.sh p1
// // date'''
// // }
pre_test()
// timeout(time: 55, unit: 'MINUTES'){
// sh '''
// date
// cd ${WKC}/tests
// ./test-all.sh p2
// date'''
// }
}
}
stage('python_3_s6') {
agent{label " slave6 || slave16 "}
steps {
pre_test()
// timeout(time: 55, unit: 'MINUTES'){
// sh '''
// date
// cd ${WKC}/tests
// ./test-all.sh p3
// date'''
// }
}
}
stage('test_b1_s2') {
agent{label " slave2 || slave12 "}
steps {
pre_test()
// timeout(time: 55, unit: 'MINUTES'){
// stage('python_2_s5') {
// agent{label " slave5 || slave15 "}
// steps {
// sh '''
// rm -rf /var/lib/taos/*
// rm -rf /var/log/taos/*
// nohup taosd >/dev/null &
// sleep 10
// '''
// sh '''
// cd ${WKC}/tests/examples/nodejs
// npm install td2.0-connector > /dev/null 2>&1
// node nodejsChecker.js host=localhost
// node test1970.js
// cd ${WKC}/tests/connectorTest/nodejsTest/nanosupport
// npm install td2.0-connector > /dev/null 2>&1
// node nanosecondTest.js
// pre_test()
// // timeout(time: 55, unit: 'MINUTES'){
// // sh '''
// // date
// // cd ${WKC}/tests
// // ./test-all.sh p2
// // date'''
// // }
// }
// }
// stage('python_3_s6') {
// agent{label " slave6 || slave16 "}
// steps {
// pre_test()
// // timeout(time: 55, unit: 'MINUTES'){
// '''
// sh '''
// cd ${WKC}/tests/examples/C#/taosdemo
// mcs -out:taosdemo *.cs > /dev/null 2>&1
// echo '' |./taosdemo -c /etc/taos
// cd ${WKC}/tests/connectorTest/C#Test/nanosupport
// mcs -out:nano *.cs > /dev/null 2>&1
// echo '' |./nano
// '''
// sh '''
// cd ${WKC}/tests/gotest
// bash batchtest.sh
// '''
// sh '''
// cd ${WKC}/tests
// ./test-all.sh b1fq
// date'''
// // sh '''
// // date
// // cd ${WKC}/tests
// // ./test-all.sh p3
// // date'''
// // }
// }
}
}
stage('test_crash_gen_s3') {
agent{label " slave3 || slave13 "}
// }
// stage('test_b1_s2') {
// agent{label " slave2 || slave12 "}
// steps {
// pre_test()
// // timeout(time: 55, unit: 'MINUTES'){
steps {
pre_test()
// timeout(time: 60, unit: 'MINUTES'){
// sh '''
// cd ${WKC}/tests/pytest
// ./crash_gen.sh -a -p -t 4 -s 2000
// '''
// // sh '''
// // rm -rf /var/lib/taos/*
// // rm -rf /var/log/taos/*
// // nohup taosd >/dev/null &
// // sleep 10
// // '''
// // sh '''
// // cd ${WKC}/tests/examples/nodejs
// // npm install td2.0-connector > /dev/null 2>&1
// // node nodejsChecker.js host=localhost
// // node test1970.js
// // cd ${WKC}/tests/connectorTest/nodejsTest/nanosupport
// // npm install td2.0-connector > /dev/null 2>&1
// // node nanosecondTest.js
// // '''
// // sh '''
// // cd ${WKC}/tests/examples/C#/taosdemo
// // mcs -out:taosdemo *.cs > /dev/null 2>&1
// // echo '' |./taosdemo -c /etc/taos
// // cd ${WKC}/tests/connectorTest/C#Test/nanosupport
// // mcs -out:nano *.cs > /dev/null 2>&1
// // echo '' |./nano
// // '''
// // sh '''
// // cd ${WKC}/tests/gotest
// // bash batchtest.sh
// // '''
// // sh '''
// // cd ${WKC}/tests
// // ./test-all.sh b1fq
// // date'''
// // }
// }
// }
// timeout(time: 60, unit: 'MINUTES'){
// stage('test_crash_gen_s3') {
// agent{label " slave3 || slave13 "}
// steps {
// pre_test()
// // timeout(time: 60, unit: 'MINUTES'){
// // sh '''
// // cd ${WKC}/tests/pytest
// // ./crash_gen.sh -a -p -t 4 -s 2000
// // '''
// // }
// // timeout(time: 60, unit: 'MINUTES'){
// // // sh '''
// // // cd ${WKC}/tests/pytest
// // // rm -rf /var/lib/taos/*
// // // rm -rf /var/log/taos/*
// // // ./handle_crash_gen_val_log.sh
// // // '''
// // sh '''
// // cd ${WKC}/tests/pytest
// // rm -rf /var/lib/taos/*
// // rm -rf /var/log/taos/*
// // ./handle_crash_gen_val_log.sh
// // ./handle_taosd_val_log.sh
// // '''
// // }
// // timeout(time: 55, unit: 'MINUTES'){
// // sh '''
// // date
// // cd ${WKC}/tests
// // ./test-all.sh b2fq
// // date
// // '''
// sh '''
// cd ${WKC}/tests/pytest
// rm -rf /var/lib/taos/*
// rm -rf /var/log/taos/*
// ./handle_taosd_val_log.sh
// '''
// // }
// }
// timeout(time: 55, unit: 'MINUTES'){
// sh '''
// date
// cd ${WKC}/tests
// ./test-all.sh b2fq
// date
// '''
// }
}
}
stage('test_valgrind_s4') {
agent{label " slave4 || slave14 "}
// stage('test_valgrind_s4') {
// agent{label " slave4 || slave14 "}
steps {
pre_test()
// catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') {
// sh '''
// cd ${WKC}/tests/pytest
// ./valgrind-test.sh 2>&1 > mem-error-out.log
// ./handle_val_log.sh
// '''
// steps {
// pre_test()
// // catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') {
// // sh '''
// // cd ${WKC}/tests/pytest
// // ./valgrind-test.sh 2>&1 > mem-error-out.log
// // ./handle_val_log.sh
// // '''
// // }
// // timeout(time: 55, unit: 'MINUTES'){
// // sh '''
// // date
// // cd ${WKC}/tests
// // ./test-all.sh b3fq
// // date'''
// // sh '''
// // date
// // cd ${WKC}/tests
// // ./test-all.sh full example
// // date'''
// // }
// }
// timeout(time: 55, unit: 'MINUTES'){
// sh '''
// date
// cd ${WKC}/tests
// ./test-all.sh b3fq
// date'''
// sh '''
// date
// cd ${WKC}/tests
// ./test-all.sh full example
// date'''
// }
}
}
stage('test_b4_s7') {
agent{label " slave7 || slave17 "}
steps {
pre_test()
// timeout(time: 55, unit: 'MINUTES'){
// stage('test_b4_s7') {
// agent{label " slave7 || slave17 "}
// steps {
// pre_test()
// // timeout(time: 55, unit: 'MINUTES'){
// sh '''
// date
// cd ${WKC}/tests
// ./test-all.sh b4fq
// cd ${WKC}/tests
// ./test-all.sh p4
// cd ${WKC}/tests
// ./test-all.sh full jdbc
// cd ${WKC}/tests
// ./test-all.sh full unit
// date'''
// // sh '''
// // date
// // cd ${WKC}/tests
// // ./test-all.sh b4fq
// // cd ${WKC}/tests
// // ./test-all.sh p4
// // cd ${WKC}/tests
// // ./test-all.sh full jdbc
// // cd ${WKC}/tests
// // ./test-all.sh full unit
// // date'''
// // }
// }
}
}
stage('test_b5_s8') {
agent{label " slave8 || slave18 "}
steps {
pre_test()
// timeout(time: 55, unit: 'MINUTES'){
// }
// stage('test_b5_s8') {
// agent{label " slave8 || slave18 "}
// steps {
// pre_test()
// // timeout(time: 55, unit: 'MINUTES'){
// sh '''
// date
// cd ${WKC}/tests
// ./test-all.sh b5fq
// date'''
// // sh '''
// // date
// // cd ${WKC}/tests
// // ./test-all.sh b5fq
// // date'''
// // }
// }
}
}
stage('test_b6_s9') {
agent{label " slave9 || slave19 "}
steps {
pre_test()
// timeout(time: 55, unit: 'MINUTES'){
// }
// stage('test_b6_s9') {
// agent{label " slave9 || slave19 "}
// steps {
// pre_test()
// // timeout(time: 55, unit: 'MINUTES'){
// sh '''
// date
// cd ${WKC}/tests
// ./test-all.sh b6fq
// date'''
// // sh '''
// // date
// // cd ${WKC}/tests
// // ./test-all.sh b6fq
// // date'''
// // }
// }
}
}
stage('test_b7_s10') {
agent{label " slave10 || slave20 "}
steps {
pre_test()
// timeout(time: 55, unit: 'MINUTES'){
// }
// stage('test_b7_s10') {
// agent{label " slave10 || slave20 "}
// steps {
// pre_test()
// // timeout(time: 55, unit: 'MINUTES'){
// sh '''
// date
// cd ${WKC}/tests
// ./test-all.sh b7fq
// date'''
// // sh '''
// // date
// // cd ${WKC}/tests
// // ./test-all.sh b7fq
// // date'''
// // }
// }
// }
// }
// }
}
}
}
}
}
post {
success {
......
cmake/lucene_CMakeLists.txt.in
浏览文件 @ 41290f9a
......@@ -2,7 +2,7 @@
# lucene
ExternalProject_Add(lucene
GIT_REPOSITORY https://github.com/taosdata-contrib/LucenePlusPlus.git
GIT_TAG rel_3.0.8
GIT_TAG rel_3.0.8_td
SOURCE_DIR "${CMAKE_SOURCE_DIR}/deps/lucene"
BINARY_DIR ""
#BUILD_IN_SOURCE TRUE
......
deps/test/CMakeLists.txt
浏览文件 @ 41290f9a
......@@ -2,3 +2,7 @@
if(${BUILD_WITH_ROCKSDB})
add_subdirectory(rocksdb)
endif(${BUILD_WITH_ROCKSDB})
if(${BUILD_WITH_LUCENE})
add_subdirectory(lucene)
endif(${BUILD_WITH_LUCENE})
deps/test/lucene/CMakeLists.txt 0 → 100644
浏览文件 @ 41290f9a
add_executable(luceneTest "")
target_sources(luceneTest
PRIVATE
"${CMAKE_CURRENT_SOURCE_DIR}/main.cpp"
)
target_link_libraries(luceneTest lucene++)
\ No newline at end of file
deps/test/lucene/main.cpp 0 → 100644
浏览文件 @ 41290f9a
#include <iostream>
int main(int argc, char const *argv[]) {
std::cout << "Hello, this is lucene test" << std::endl;
return 0;
}
docs/scratch.md 0 → 100644
浏览文件 @ 41290f9a
```plantuml
@startuml create_table
skinparam sequenceMessageAlign center
skinparam responseMessageBelowArrow true
participant APP as app
box "dnode1"
participant RPC as rpc
participant VNODE as vnode
participant SYNC as sync
end box
box "dnode2"
participant SYNC as sync2
participant VNODE as vnode2
end box
box "dnode3"
participant SYNC as sync3
participant VNODE as vnode3
end box
' APP send request to dnode and RPC in dnode recv the request
app ->rpc: create table req
' RPC call vnodeProcessReq() function to process the request
rpc -> vnode: vnodeProcessReq
note right
callback function
run in RPC module
threads. The function
only puts the request
to a vnode queue.
end note
' VNODE call vnodeProcessReqs() function to integrate requests and process as a whole
vnode -> vnode: vnodeProcessReqs()
note right
integrate reqs and
process as a whole
end note
' sync the request to other nodes
vnode -> sync: syncProcessReqs()
' make request persistent
' sync -->vnode: walWrite()\n(callback function)
' replicate requests to other DNODES
sync -> sync2: replication req
sync -> sync3: replication req
sync2 -> vnode2: walWrite()\n(callback function)
sync2 --> sync: replication rsp\n(confirm)
sync3 -> vnode3: walWrite()\n(callback function)
sync3 --> sync: replication rsp\n(confirm)
' send apply request
sync -> sync2: apply req
sync -> sync3: apply req
' vnode apply
sync2 -> vnode2: vnodeApplyReqs()
sync3 -> vnode3: vnodeApplyReqs()
' call apply request
sync --> vnode: vnodeApplyReqs()\n(callback function)
' send response
vnode --> rpc: rpcSendRsp()
' dnode send response to APP
rpc --> app: create table rsp
@enduml
```
## Leader处理强一致写入请求
```plantuml
@startuml leader_process_stict_consistency
box "dnode1"
participant CRPC as crpc
participant VNODE as vnode
participant SYNC as sync
end box
-> crpc: create table/submit req
' In CRPC threads
group #pink "In CRPC threads"
crpc -> vnode:vnodeProcessReq()
note right
A callback function
run by CRPC thread
to put the request
to a vnode queue
end note
end
' In VNODE worker threads
group #lightblue "In VNODE worker threads"
vnode -> vnode: vnodeProcessReqs()
note right
VNODE process requests
accumulated in a
vnode write queue and
process the batch reqs
as a whole
end note
vnode -> sync: syncProcessReqs()
sync -> : replication req1
sync -> : replication req2
end
group #red "SYNC threads"
sync <- : replication rsp1
sync <- : replication rsp2
sync -> vnode: notify apply
sync -> : apply rsp1
sync -> : apply rsp2
end
group #lightblue "In VNODE worker threads"
vnode -> vnode: vnodeApplyReqs()
vnode -> crpc:
end
<- crpc: create table/submit rsp
@enduml
```
## Follower处理强一致写入请求
```plantuml
@startuml follower_process_strict_consistency
participant SYNC as sync
participant VNODE as vnode
group #pink "SYNC threads"
-> sync: replication req
sync -> sync: syncProcessReqs()
note right
In the replication
only data is
persisted and response
is sent back
end note
<- sync: replication rsp
-> sync: apply req
sync -> vnode: notify apply
end
group #lightblue "VNODE worker threads"
vnode -> vnode: vnodeApplyReqs()
end
@enduml
```
## Leader处理最终一致写入请求
```plantuml
@startuml leader_process_eventual_consistency
box "dnode1"
participant CRPC as crpc
participant VNODE as vnode
participant SYNC as sync
end box
-> crpc: create table/submit req
' In CRPC threads
group #pink "In CRPC threads"
crpc -> vnode:vnodeProcessReq()
note right
A callback function
run by CRPC thread
to put the request
to a vnode queue
end note
end
' In VNODE worker threads
group #lightblue "In VNODE worker threads"
vnode -> vnode: vnodeProcessReqs()
note right
VNODE process requests
accumulated in a
vnode write queue and
process the batch reqs
as a whole
end note
vnode -> sync: syncProcessReqs()
sync -> : replication req1
sync -> : replication req2
sync -> vnode: notify apply
end
group #lightblue "In VNODE worker threads"
vnode -> vnode: vnodeApplyReqs()
vnode -> crpc:
end
<- crpc: create table/submit rsp
@enduml
```
## Follower处理最终一致写入请求
```plantuml
@startuml follower_process_eventual_consistency
participant SYNC as sync
participant VNODE as vnode
group #pink "SYNC threads"
-> sync: replication rsp
sync -> sync: syncProcessReqs()
sync -> vnode: notify VNODE \nthread to process\n the reqs
end
group #lightblue "VNODE worker threads"
vnode -> vnode: vnodeApplyReqs()
end
@enduml
```
\ No newline at end of file
docs/vnode_write.md 0 → 100644
浏览文件 @ 41290f9a
<center><h1>VNODE Write Processes</h1></center>
## META Operations
META data write operations including:
1. create table
2. drop table
3. alter table
We take create table as an example to figure out the whole process.
```plantuml
@startuml create_table
skinparam sequenceMessageAlign center
skinparam responseMessageBelowArrow true
participant APP as app
box "dnode1"
participant RPC as rpc
participant VNODE as vnode
participant SYNC as sync
end box
box "dnode2"
participant SYNC as sync2
participant VNODE as vnode2
end box
box "dnode3"
participant SYNC as sync3
participant VNODE as vnode3
end box
' APP send request to dnode and RPC in dnode recv the request
app ->rpc: create table req
' RPC call vnodeProcessReq() function to process the request
rpc -> vnode: vnodeProcessReq
note right
callback function
run in RPC module
threads. The function
only puts the request
to a vnode queue.
end note
' VNODE call vnodeProcessReqs() function to integrate requests and process as a whole
vnode -> vnode: vnodeProcessReqs()
note right
integrate reqs and
process as a whole
end note
' sync the request to other nodes
vnode -> sync: syncProcessReqs()
' make request persistent
' sync -->vnode: walWrite()\n(callback function)
' replicate requests to other DNODES
sync -> sync2: replication req
sync -> sync3: replication req
sync2 -> vnode2: walWrite()\n(callback function)
sync2 --> sync: replication rsp\n(confirm)
sync3 -> vnode3: walWrite()\n(callback function)
sync3 --> sync: replication rsp\n(confirm)
' send apply request
sync -> sync2: apply req
sync -> sync3: apply req
' vnode apply
sync2 -> vnode2: vnodeApplyReqs()
sync3 -> vnode3: vnodeApplyReqs()
' call apply request
sync --> vnode: vnodeApplyReqs()\n(callback function)
' send response
vnode --> rpc: rpcSendRsp()
' dnode send response to APP
rpc --> app: create table rsp
@enduml
```
## Time-series data Operations
There are only one operations for time-series data: data insert. We will figure out the whole process.
```plantuml
@startuml create_table
skinparam sequenceMessageAlign center
skinparam responseMessageBelowArrow true
participant APP as app
box "dnode1"
participant RPC as rpc
participant VNODE as vnode
participant SYNC as sync
end box
box "dnode2"
participant SYNC as sync2
participant VNODE as vnode2
end box
box "dnode3"
participant SYNC as sync3
participant VNODE as vnode3
end box
' APP send request to dnode and RPC in dnode recv the request
app ->rpc: insert data req
' RPC call vnodeProcessReq() function to process the request
rpc -> vnode: vnodeProcessReq
note right
callback function
run in RPC module
threads. The function
only puts the request
to a vnode queue.
end note
' VNODE call vnodeProcessReqs() function to integrate requests and process as a whole
vnode -> vnode: vnodeProcessReqs()
note right
integrate reqs and
process as a whole
end note
' sync the request to other nodes
vnode -> sync: syncProcessReqs()
' ' make request persistent
' ' sync -->vnode: walWrite()\n(callback function)
' ' replicate requests to other DNODES
sync -> sync2: replication req
sync -> sync3: replication req
' vnode apply
sync2 -> vnode2: vnodeApplyReqs()
sync3 -> vnode3: vnodeApplyReqs()
' call apply request
sync --> vnode: vnodeApplyReqs()\n(callback function)
' send response
vnode --> rpc: rpcSendRsp()
' dnode send response to APP
rpc --> app: insert data rsp
@enduml
```
## vnodeProcessReqs()
```plantuml
@startuml vnodeProcessReqs()
participant VNODE as v
participant SYNC as s
group vnodeProcessReqs()
' Group requests and get a request batch to process as a whole
v -> v: vnodeGetReqsFromQueue()
note right
integrate all write
requests as a batch
to process as a whole
end note
' VNODE call syncProcessReqs() function to process the batch request
v -> s: syncProcessReqs()
group syncProcessReqs()
' Check if current node is leader
alt not leader
return NOT_LEADER
end
s -> s: syncAppendReqsToLogStore()
group syncAppendReqsToLogStore()
s -> v: walWrite()
note right
There must be a
callback function
provided by VNODE
to persist the
requests in WAL
end note
alt (no unapplied reqs) AND (only one node OR no meta requests)
s -> v: vnodeApplyReqs()
note right
just use the woker
thread to apply
the requests. This
is a callback function
provided by VNODE
end note
else other cases need to wait response
s -> s:
note right
save the requests in log store
and wait for comfirmation or
other cases
end note
s ->]: send replication requests
s ->]: send replication requests
end
end
end
end
@enduml
```
<!-- ## syncProcessReplicationReq()
```plantuml
@startuml syncProcessReplicationReq
participant SYNC as s
participant VNODE as v
-> s: replication request
s -> s:
note right
process the request
to get the request
batch
end note
s -> s: syncAppendReqToLogStore()
s -> v: walWrite()
alt has meta req
<- s: comfirmation
else
s -> v: vnodeApplyReqs()
end
@enduml -->
<!-- ``` -->
## vnodeApplyReqs()
The function *vnodeApplyReqs()* is the actual function running by a vnode to process the requests.
```plantuml
@startuml vnodeApplyReqs()
skinparam sequenceMessageAlign left
skinparam responseMessageBelowArrow true
participant VNODE as vnode
participant TQ as tq
participant TSDB as tsdb
participant META as meta
group vnodeApplyReqs()
autonumber
loop nReqs
' Copy request message to vnode buffer pool
vnode -> vnode: vnodeCopyReq()
note right
copy request to
vnode buffer pool
end note
vnode -> tq: tqPush()
note right
push the request
to TQ so consumers
can consume
end note
alt META_REQ
autonumber 3
vnode -> meta: metaApplyReq()
else TS_REQ
autonumber 3
vnode -> tsdb: tsdbApplyReq()
end
end
' Check if need to commit
alt vnode buffer pool is full
group vnodeCommit()
autonumber 4.1
vnode -> tq: tqCommit()
note right
tqCommit may renew wal
end note
vnode -> meta: metaCommit();
note right
commit meta data
end note
vnode -> tsdb: tsdbCommit();
note right
commit time-series data
end note
end
end
end
@enduml
```
<!-- meta操作：建表，删表，改表（队队列/同步）
数据写入
快照文件与sync的结合
vnodeOpen（）
vnodeClose（）
sync.h -->
include/client/taos.h
浏览文件 @ 41290f9a
......@@ -46,6 +46,11 @@ typedef void **TAOS_ROW;
#define TSDB_DATA_TYPE_USMALLINT 12 // 2 bytes
#define TSDB_DATA_TYPE_UINT 13 // 4 bytes
#define TSDB_DATA_TYPE_UBIGINT 14 // 8 bytes
#define TSDB_DATA_TYPE_VARCHAR 15 // string
#define TSDB_DATA_TYPE_JSON 16 // json
#define TSDB_DATA_TYPE_DECIMAL 17 // decimal
#define TSDB_DATA_TYPE_BLOB 18 // binary string
#define TSDB_DATA_TYPE_LONGBLOB 19 // long binary string
typedef enum {
TSDB_OPTION_LOCALE,
......@@ -157,7 +162,6 @@ DLL_EXPORT int taos_errno(TAOS_RES *tres);
DLL_EXPORT void taos_query_a(TAOS *taos, const char *sql, void (*fp)(void *param, TAOS_RES *, int code), void *param);
DLL_EXPORT void taos_fetch_rows_a(TAOS_RES *res, void (*fp)(void *param, TAOS_RES *, int numOfRows), void *param);
//DLL_EXPORT void taos_fetch_row_a(TAOS_RES *res, void (*fp)(void *param, TAOS_RES *, TAOS_ROW row), void *param);
typedef void (*TAOS_SUBSCRIBE_CALLBACK)(TAOS_SUB* tsub, TAOS_RES *res, void* param, int code);
DLL_EXPORT TAOS_SUB *taos_subscribe(TAOS* taos, int restart, const char* topic, const char *sql, TAOS_SUBSCRIBE_CALLBACK fp, void *param, int interval);
......
include/common/common.h
浏览文件 @ 41290f9a
......@@ -17,6 +17,8 @@
#define TDENGINE_COMMON_H
#include "taosdef.h"
#include "taosmsg.h"
#include "tarray.h"
//typedef struct STimeWindow {
// TSKEY skey;
......@@ -36,4 +38,32 @@
// int16_t bytes;
//} SSchema;
typedef struct SColumnDataAgg {
int16_t colId;
int64_t sum;
int64_t max;
int64_t min;
int16_t maxIndex;
int16_t minIndex;
int16_t numOfNull;
} SColumnDataAgg;
typedef struct SDataBlockInfo {
STimeWindow window;
int32_t rows;
int32_t numOfCols;
int64_t uid;
} SDataBlockInfo;
typedef struct SSDataBlock {
SColumnDataAgg *pBlockAgg;
SArray *pDataBlock; // SArray<SColumnInfoData>
SDataBlockInfo info;
} SSDataBlock;
typedef struct SColumnInfoData {
SColumnInfo info; // TODO filter info needs to be removed
char *pData; // the corresponding block data in memory
} SColumnInfoData;
#endif // TDENGINE_COMMON_H
include/common/taosmsg.h
浏览文件 @ 41290f9a
......@@ -456,7 +456,6 @@ typedef struct SColumnInfo {
typedef struct STableIdInfo {
uint64_t uid;
int32_t tid;
TSKEY key; // last accessed ts, for subscription
} STableIdInfo;
......
include/common/tvariant.h
浏览文件 @ 41290f9a
......@@ -36,6 +36,8 @@ typedef struct SVariant {
};
} SVariant;
int32_t toInteger(const char* z, int32_t n, int32_t base, int64_t* value, bool* issigned);
bool taosVariantIsValid(SVariant *pVar);
void taosVariantCreate(SVariant *pVar, char* z, int32_t n, int32_t type);
......
include/libs/catalog/catalog.h
浏览文件 @ 41290f9a
......@@ -33,7 +33,6 @@ struct SCatalog;
typedef struct SMetaReq {
char clusterId[TSDB_CLUSTER_ID_LEN];
SArray *pTableName; // table full name
SArray *pVgroup; // vgroup id
SArray *pUdf; // udf name
bool qNodeEpset; // valid qnode
} SMetaReq;
......@@ -60,7 +59,6 @@ typedef struct STableComInfo {
typedef struct SCTableMeta {
int32_t vgId:24;
int8_t tableType;
uint32_t tid;
uint64_t uid;
uint64_t suid;
} SCTableMeta;
......@@ -71,7 +69,6 @@ typedef struct SCTableMeta {
typedef struct STableMeta {
int32_t vgId:24;
int8_t tableType;
uint32_t tid;
uint64_t uid;
uint64_t suid;
// if the table is TSDB_CHILD_TABLE, the following information is acquired from the corresponding super table meta info
......
include/libs/function/function.h 0 → 100644
浏览文件 @ 41290f9a
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef TDENGINE_TFUNCTION_H
#define TDENGINE_TFUNCTION_H
#ifdef __cplusplus
extern "C" {
#endif
#include "common.h"
#include "tvariant.h"
#include "tbuffer.h"
#define FUNCTION_SCALAR 1
#define FUNCTION_AGG 2
#define TOP_BOTTOM_QUERY_LIMIT 100
#define FUNCTIONS_NAME_MAX_LENGTH 16
#define FUNCTION_INVALID_ID -1
#define FUNCTION_COUNT 0
#define FUNCTION_SUM 1
#define FUNCTION_AVG 2
#define FUNCTION_MIN 3
#define FUNCTION_MAX 4
#define FUNCTION_STDDEV 5
#define FUNCTION_PERCT 6
#define FUNCTION_APERCT 7
#define FUNCTION_FIRST 8
#define FUNCTION_LAST 9
#define FUNCTION_LAST_ROW 10
#define FUNCTION_TOP 11
#define FUNCTION_BOTTOM 12
#define FUNCTION_SPREAD 13
#define FUNCTION_TWA 14
#define FUNCTION_LEASTSQR 15
#define FUNCTION_TS 16
#define FUNCTION_TS_DUMMY 17
#define FUNCTION_TAG_DUMMY 18
#define FUNCTION_TS_COMP 19
#define FUNCTION_TAG 20
#define FUNCTION_PRJ 21
#define FUNCTION_TAGPRJ 22
#define FUNCTION_ARITHM 23
#define FUNCTION_DIFF 24
#define FUNCTION_FIRST_DST 25
#define FUNCTION_LAST_DST 26
#define FUNCTION_STDDEV_DST 27
#define FUNCTION_INTERP 28
#define FUNCTION_RATE 29
#define FUNCTION_IRATE 30
#define FUNCTION_TID_TAG 31
#define FUNCTION_DERIVATIVE 32
#define FUNCTION_BLKINFO 33
#define FUNCTION_HISTOGRAM 34
#define FUNCTION_HLL 35
#define FUNCTION_MODE 36
#define FUNCTION_SAMPLE 37
typedef struct SPoint1 {
int64_t key;
union{double val; char* ptr;};
} SPoint1;
struct SQLFunctionCtx;
struct SResultRowCellInfo;
//for selectivity query, the corresponding tag value is assigned if the data is qualified
typedef struct SExtTagsInfo {
int16_t tagsLen; // keep the tags data for top/bottom query result
int16_t numOfTagCols;
struct SQLFunctionCtx **pTagCtxList;
} SExtTagsInfo;
// sql function runtime context
typedef struct SQLFunctionCtx {
int32_t size; // number of rows
void * pInput; // input data buffer
uint32_t order; // asc|desc
int16_t inputType;
int16_t inputBytes;
int16_t outputType;
int16_t outputBytes; // size of results, determined by function and input column data type
int32_t interBufBytes; // internal buffer size
bool hasNull; // null value exist in current block
bool requireNull; // require null in some function
bool stableQuery;
int16_t functionId; // function id
char * pOutput; // final result output buffer, point to sdata->data
uint8_t currentStage; // record current running step, default: 0
int64_t startTs; // timestamp range of current query when function is executed on a specific data block
int32_t numOfParams;
SVariant param[4]; // input parameter, e.g., top(k, 20), the number of results for top query is kept in param
int64_t *ptsList; // corresponding timestamp array list
void *ptsOutputBuf; // corresponding output buffer for timestamp of each result, e.g., top/bottom*/
SVariant tag;
bool isSmaSet;
SColumnDataAgg sma;
struct SResultRowCellInfo *resultInfo;
SExtTagsInfo tagInfo;
SPoint1 start;
SPoint1 end;
} SQLFunctionCtx;
enum {
TEXPR_NODE_DUMMY = 0x0,
TEXPR_BINARYEXPR_NODE= 0x1,
TEXPR_UNARYEXPR_NODE = 0x2,
TEXPR_COL_NODE = 0x4,
TEXPR_VALUE_NODE = 0x8,
};
typedef struct tExprNode {
uint8_t nodeType;
union {
struct {
union {
int32_t optr; // binary operator
int32_t functionId;// unary operator
};
void *info; // support filter operation on this expression only available for leaf node
struct tExprNode *pLeft; // left child pointer
struct tExprNode *pRight; // right child pointer
} _node;
SSchema *pSchema;// column node
struct SVariant *pVal; // value node
};
} tExprNode;
void exprTreeToBinary(SBufferWriter* bw, tExprNode* pExprTree);
void tExprTreeDestroy(tExprNode *pNode, void (*fp)(void *));
typedef struct SAggFunctionInfo {
char name[FUNCTIONS_NAME_MAX_LENGTH];
int8_t type; // Scalar function or aggregation function
uint8_t functionId; // Function Id
int8_t sFunctionId; // Transfer function for super table query
uint16_t status;
bool (*init)(SQLFunctionCtx *pCtx, struct SResultRowCellInfo* pResultCellInfo); // setup the execute environment
void (*exec)(SQLFunctionCtx *pCtx);
// finalizer must be called after all exec has been executed to generated final result.
void (*xFinalize)(SQLFunctionCtx *pCtx);
void (*mergeFunc)(SQLFunctionCtx *pCtx);
int32_t (*dataReqFunc)(SQLFunctionCtx *pCtx, STimeWindow* w, int32_t colId);
} SAggFunctionInfo;
typedef struct SScalarFunctionInfo {
char name[FUNCTIONS_NAME_MAX_LENGTH];
int8_t type; // scalar function or aggregation function
uint8_t functionId; // index of scalar function
bool (*init)(SQLFunctionCtx *pCtx, struct SResultRowCellInfo* pResultCellInfo); // setup the execute environment
void (*exec)(SQLFunctionCtx *pCtx);
} SScalarFunctionInfo;
typedef struct SResultDataInfo {
int16_t type;
int16_t bytes;
int32_t intermediateBytes;
} SResultDataInfo;
int32_t getResultDataInfo(int32_t dataType, int32_t dataBytes, int32_t functionId, int32_t param, SResultDataInfo* pInfo, int16_t extLength,
bool isSuperTable);
/**
* If the given name is a valid built-in sql function, the value of true will be returned.
* @param name
* @param len
* @return
*/
int32_t qIsBuiltinFunction(const char* name, int32_t len);
bool qIsValidUdf(SArray* pUdfInfo, const char* name, int32_t len, int32_t* functionId);
const char* qGetFunctionName(int32_t functionId);
#ifdef __cplusplus
}
#endif
#endif // TDENGINE_TFUNCTION_H
include/libs/parser/parser.h
浏览文件 @ 41290f9a
......@@ -23,6 +23,29 @@ extern "C" {
#include "catalog.h"
#include "common.h"
#include "tname.h"
#include "tvariant.h"
typedef struct SColumn {
uint64_t tableUid;
int32_t columnIndex;
SColumnInfo info;
} SColumn;
// the structure for sql function in select clause
typedef struct SSqlExpr {
char token[TSDB_COL_NAME_LEN]; // original token
SSchema resSchema;
SColIndex colInfo;
uint64_t uid; // table uid, todo refactor use the pointer
int32_t interBytes; // inter result buffer size
int16_t numOfParams; // argument value of each function
SVariant param[3]; // parameters are not more than 3
} SSqlExpr;
typedef struct SExprInfo {
SSqlExpr base;
struct tExprNode *pExpr;
} SExprInfo;
//typedef struct SInterval {
// int32_t tz; // query client timezone
......@@ -95,10 +118,7 @@ typedef struct STagCond {
typedef struct STableMetaInfo {
STableMeta *pTableMeta; // table meta, cached in client side and acquired by name
uint32_t tableMetaSize;
size_t tableMetaCapacity;
SVgroupsInfo *vgroupList;
SArray *pVgroupTables; // SArray<SVgroupTableInfo>
/*
* 1. keep the vgroup index during the multi-vnode super table projection query
......@@ -110,6 +130,20 @@ typedef struct STableMetaInfo {
SArray *tagColList; // SArray<SColumn*>, involved tag columns
} STableMetaInfo;
typedef struct SQueryAttrInfo {
bool stableQuery;
bool groupbyColumn;
bool simpleAgg;
bool arithmeticOnAgg;
bool projectionQuery;
bool hasFilter;
bool onlyTagQuery;
bool orderProjectQuery;
bool stateWindow;
bool globalMerge;
bool multigroupResult;
} SQueryAttrInfo;
typedef struct SQueryStmtInfo {
int16_t command; // the command may be different for each subclause, so keep it seperately.
uint32_t type; // query/insert type
......@@ -152,19 +186,15 @@ typedef struct SQueryStmtInfo {
SArray *pUpstream; // SArray<struct SQueryStmtInfo>
struct SQueryStmtInfo *pDownstream;
int32_t havingFieldNum;
bool stableQuery;
bool groupbyColumn;
bool simpleAgg;
bool arithmeticOnAgg;
bool projectionQuery;
bool hasFilter;
bool onlyTagQuery;
bool orderProjectQuery;
bool stateWindow;
bool globalMerge;
bool multigroupResult;
SQueryAttrInfo info;
} SQueryStmtInfo;
typedef struct SColumnIndex {
int16_t tableIndex;
int16_t columnIndex;
int16_t type; // normal column/tag/ user input constant column
} SColumnIndex;
struct SInsertStmtInfo;
/**
......@@ -206,6 +236,17 @@ int32_t qParseInsertSql(const char* pStr, size_t length, struct SInsertStmtInfo*
*/
int32_t qParserConvertSql(const char* pStr, size_t length, char** pConvertSql);
void assignExprInfo(SExprInfo* dst, const SExprInfo* src);
void columnListCopy(SArray* dst, const SArray* src, uint64_t uid);
void columnListDestroy(SArray* pColumnList);
void dropAllExprInfo(SArray* pExprInfo);
SExprInfo* createExprInfo(STableMetaInfo* pTableMetaInfo, int16_t functionId, SColumnIndex* pColIndex, struct tExprNode* pParamExpr, SSchema* pResSchema, int16_t interSize);
int32_t copyExprInfoList(SArray* dst, const SArray* src, uint64_t uid, bool deepcopy);
STableMetaInfo* getMetaInfo(SQueryStmtInfo* pQueryInfo, int32_t tableIndex);
int32_t getNewResColId();
#ifdef __cplusplus
}
#endif
......
include/libs/planner/planner.h
浏览文件 @ 41290f9a
......@@ -24,8 +24,8 @@ extern "C" {
#define QUERY_TYPE_PARTIAL 2
struct SEpSet;
struct SQueryNode;
struct SQueryPhyNode;
struct SQueryPlanNode;
struct SQueryDistPlanNode;
struct SQueryStmtInfo;
typedef struct SSubquery {
......@@ -33,7 +33,7 @@ typedef struct SSubquery {
int32_t type; // QUERY_TYPE_MERGE|QUERY_TYPE_PARTIAL
int32_t level; // the execution level of current subquery, starting from 0.
SArray *pUpstream; // the upstream,from which to fetch the result
struct SQueryPhyNode *pNode; // physical plan of current subquery
struct SQueryDistPlanNode *pNode; // physical plan of current subquery
} SSubquery;
typedef struct SQueryJob {
......@@ -48,7 +48,7 @@ typedef struct SQueryJob {
* @param pQueryNode
* @return
*/
int32_t qOptimizeQueryPlan(struct SQueryNode* pQueryNode);
int32_t qOptimizeQueryPlan(struct SQueryPlanNode* pQueryNode);
/**
* Create the query plan according to the bound AST, which is in the form of pQueryInfo
......@@ -56,14 +56,14 @@ int32_t qOptimizeQueryPlan(struct SQueryNode* pQueryNode);
* @param pQueryNode
* @return
*/
int32_t qCreateQueryPlan(const struct SQueryStmtInfo* pQueryInfo, struct SQueryNode* pQueryNode);
int32_t qCreateQueryPlan(const struct SQueryStmtInfo* pQueryInfo, struct SQueryPlanNode* pQueryNode);
/**
* Convert the query plan to string, in order to display it in the shell.
* @param pQueryNode
* @return
*/
int32_t qQueryPlanToString(struct SQueryNode* pQueryNode, char** str);
int32_t qQueryPlanToString(struct SQueryPlanNode* pQueryNode, char** str);
/**
* Restore the SQL statement according to the logic query plan.
......@@ -71,7 +71,7 @@ int32_t qQueryPlanToString(struct SQueryNode* pQueryNode, char** str);
* @param sql
* @return
*/
int32_t qQueryPlanToSql(struct SQueryNode* pQueryNode, char** sql);
int32_t qQueryPlanToSql(struct SQueryPlanNode* pQueryNode, char** sql);
/**
* Create the physical plan for the query, according to the logic plan.
......@@ -79,7 +79,7 @@ int32_t qQueryPlanToSql(struct SQueryNode* pQueryNode, char** sql);
* @param pPhyNode
* @return
*/
int32_t qCreatePhysicalPlan(struct SQueryNode* pQueryNode, struct SEpSet* pQnode, struct SQueryPhyNode *pPhyNode);
int32_t qCreatePhysicalPlan(struct SQueryPlanNode* pQueryNode, struct SEpSet* pQnode, struct SQueryDistPlanNode *pPhyNode);
/**
* Convert to physical plan to string to enable to print it out in the shell.
......@@ -87,20 +87,20 @@ int32_t qCreatePhysicalPlan(struct SQueryNode* pQueryNode, struct SEpSet* pQnode
* @param str
* @return
*/
int32_t qPhyPlanToString(struct SQueryPhyNode *pPhyNode, char** str);
int32_t qPhyPlanToString(struct SQueryDistPlanNode *pPhyNode, char** str);
/**
* Destroy the query plan object.
* @return
*/
void* qDestroyQueryPlan(struct SQueryNode* pQueryNode);
void* qDestroyQueryPlan(struct SQueryPlanNode* pQueryNode);
/**
* Destroy the physical plan.
* @param pQueryPhyNode
* @return
*/
void* qDestroyQueryPhyPlan(struct SQueryPhyNode* pQueryPhyNode);
void* qDestroyQueryPhyPlan(struct SQueryDistPlanNode* pQueryPhyNode);
/**
* Create the query job from the physical execution plan
......@@ -108,7 +108,7 @@ void* qDestroyQueryPhyPlan(struct SQueryPhyNode* pQueryPhyNode);
* @param pJob
* @return
*/
int32_t qCreateQueryJob(const struct SQueryPhyNode* pPhyNode, struct SQueryJob** pJob);
int32_t qCreateQueryJob(const struct SQueryDistPlanNode* pPhyNode, struct SQueryJob** pJob);
#ifdef __cplusplus
}
......
include/libs/sync/sync.h
浏览文件 @ 41290f9a
......@@ -22,7 +22,6 @@ extern "C" {
#include <stdint.h>
#include "taosdef.h"
#include "wal.h"
typedef int64_t SyncNodeId;
typedef int32_t SyncGroupId;
......@@ -41,6 +40,7 @@ typedef struct {
} SSyncBuffer;
typedef struct {
SyncNodeId nodeId;
uint16_t nodePort; // node sync Port
char nodeFqdn[TSDB_FQDN_LEN]; // node FQDN
} SNodeInfo;
......@@ -62,32 +62,59 @@ typedef struct SSyncFSM {
void* pData;
// apply committed log, bufs will be free by raft module
int (*applyLog)(struct SSyncFSM *fsm, SyncIndex index, const SSyncBuffer *buf, void *pData);
int (*applyLog)(struct SSyncFSM* fsm, SyncIndex index, const SSyncBuffer* buf, void* pData);
// cluster commit callback
int (*onClusterChanged)(struct SSyncFSM *fsm, const SSyncCluster* cluster, void *pData);
int (*onClusterChanged)(struct SSyncFSM* fsm, const SSyncCluster* cluster, void* pData);
// fsm return snapshot in ppBuf, bufs will be free by raft module
// TODO: getSnapshot SHOULD be async?
int (*getSnapshot)(struct SSyncFSM *fsm, SSyncBuffer **ppBuf, int* objId, bool *isLast);
int (*getSnapshot)(struct SSyncFSM* fsm, SSyncBuffer** ppBuf, int* objId, bool* isLast);
// fsm apply snapshot with pBuf data
int (*applySnapshot)(struct SSyncFSM *fsm, SSyncBuffer *pBuf, int objId, bool isLast);
int (*applySnapshot)(struct SSyncFSM* fsm, SSyncBuffer* pBuf, int objId, bool isLast);
// call when restore snapshot and log done
int (*onRestoreDone)(struct SSyncFSM *fsm);
int (*onRestoreDone)(struct SSyncFSM* fsm);
void (*onRollback)(struct SSyncFSM *fsm, SyncIndex index, const SSyncBuffer *buf);
void (*onRollback)(struct SSyncFSM* fsm, SyncIndex index, const SSyncBuffer* buf);
void (*onRoleChanged)(struct SSyncFSM *fsm, const SNodesRole* pRole);
void (*onRoleChanged)(struct SSyncFSM* fsm, const SNodesRole* pRole);
} SSyncFSM;
typedef struct SSyncLogStore {
void* pData;
// write log with given index
int32_t (*logWrite)(struct SSyncLogStore* logStore, SyncIndex index, SSyncBuffer* pBuf);
// mark log with given index has been commtted
int32_t (*logCommit)(struct SSyncLogStore* logStore, SyncIndex index);
// prune log before given index
int32_t (*logPrune)(struct SSyncLogStore* logStore, SyncIndex index);
// rollback log after given index
int32_t (*logRollback)(struct SSyncLogStore* logStore, SyncIndex index);
} SSyncLogStore;
typedef struct SSyncServerState {
SNodeInfo voteFor;
SyncNodeId voteFor;
SSyncTerm term;
} SSyncServerState;
typedef struct SSyncClusterConfig {
// Log index number of current cluster config.
SyncIndex index;
// Log index number of previous cluster config.
SyncIndex prevIndex;
// current cluster
const SSyncCluster* cluster;
} SSyncClusterConfig;
typedef struct SStateManager {
void* pData;
......@@ -95,35 +122,38 @@ typedef struct SStateManager {
const SSyncServerState* (*readServerState)(struct SStateManager* stateMng);
void (*saveCluster)(struct SStateManager* stateMng, const SSyncCluster* cluster);
void (*saveCluster)(struct SStateManager* stateMng, const SSyncClusterConfig* cluster);
const SSyncCluster* (*readCluster)(struct SStateManager* stateMng);
const SSyncClusterConfig* (*readCluster)(struct SStateManager* stateMng);
} SStateManager;
typedef struct {
SyncGroupId vgId;
twalh walHandle;
SyncIndex snapshotIndex;
SSyncCluster syncCfg;
SSyncFSM fsm;
SSyncLogStore logStore;
SStateManager stateManager;
} SSyncInfo;
struct SSyncNode;
typedef struct SSyncNode SSyncNode;
int32_t syncInit();
void syncCleanUp();
SyncNodeId syncStart(const SSyncInfo*);
SSyncNode syncStart(const SSyncInfo*);
void syncStop(SyncNodeId);
int32_t syncPropose(SyncNodeId nodeId, SSyncBuffer buffer, void* pData, bool isWeak);
int32_t syncPropose(SSyncNode syncNode, SSyncBuffer buffer, void* pData, bool isWeak);
int32_t syncAddNode(SyncNodeId nodeId, const SNodeInfo *pNode);
int32_t syncAddNode(SSyncNode syncNode, const SNodeInfo *pNode);
int32_t syncRemoveNode(SyncNodeId nodeId, const SNodeInfo *pNode);
int32_t syncRemoveNode(SSyncNode syncNode, const SNodeInfo *pNode);
extern int32_t syncDebugFlag;
......
include/os/os.h
浏览文件 @ 41290f9a
......@@ -44,6 +44,7 @@ extern "C" {
#include <errno.h>
#include <float.h>
#include <math.h>
#include <sys/stat.h>
#include "osAtomic.h"
#include "osDef.h"
......
include/server/vnode/tq/tq.h
浏览文件 @ 41290f9a
......@@ -23,64 +23,79 @@ extern "C" {
#endif
typedef struct tmqMsgHead {
int32_t headLen;
int32_t protoVer;
int32_t msgType;
int64_t cgId;
int64_t topicId;
int64_t clientId;
int32_t checksum;
int32_t msgType;
} tmqMsgHead;
typedef struct tmqOneAck {
int64_t topicId;
int64_t consumeOffset;
} tmqOneAck;
typedef struct tmqAcks {
int32_t ackNum;
//should be sorted
tmqOneAck acks[];
} tmqAcks;
//TODO: put msgs into common
typedef struct tmqConnectReq {
tmqMsgHead head;
tmqAcks acks;
} tmqConnectReq;
typedef struct tmqConnectResp {
typedef struct tmqConnectRsp {
tmqMsgHead head;
int8_t status;
} tmqConnectResp;
} tmqConnectRsp;
typedef struct tmqDisconnectReq {
tmqMsgHead head;
} tmqDisconnectReq;
typedef struct tmqDisconnectResp {
typedef struct tmqDisconnectRsp {
tmqMsgHead head;
int8_t status;
} tmqDiconnectResp;
} tmqDiconnectRsp;
typedef struct tmqConsumeReq {
tmqMsgHead head;
int64_t commitOffset;
tmqAcks acks;
} tmqConsumeReq;
typedef struct tmqConsumeResp {
typedef struct tmqMsgContent {
int64_t topicId;
int64_t msgLen;
char msg[];
} tmqMsgContent;
typedef struct tmqConsumeRsp {
tmqMsgHead head;
char content[];
} tmqConsumeResp;
int64_t bodySize;
tmqMsgContent msgs[];
} tmqConsumeRsp;
//
typedef struct tmqMnodeSubscribeReq {
tmqMsgHead head;
int64_t topicLen;
char topic[];
} tmqSubscribeReq;
typedef struct tmqMnodeSubscribeResp {
typedef struct tmqMnodeSubscribeRsp {
tmqMsgHead head;
int64_t vgId;
char ep[]; //TSDB_EP_LEN
} tmqSubscribeResp;
} tmqSubscribeRsp;
typedef struct tmqHeartbeatReq {
} tmqHeartbeatReq;
typedef struct tmqHeartbeatResp {
typedef struct tmqHeartbeatRsp {
} tmqHeartbeatResp;
} tmqHeartbeatRsp;
typedef struct tqTopicVhandle {
//name
......@@ -92,33 +107,57 @@ typedef struct tqTopicVhandle {
} tqTopicVhandle;
typedef struct STQ {
//the set for topics
//key=topicName: str
//value=tqTopicVhandle
//the collection of group handle
//a map
//key=<topic: str, cgId: int64_t>
//value=consumeOffset: int64_t
} STQ;
#define TQ_BUFFER_SIZE 8
//TODO: define a serializer and deserializer
typedef struct tqBufferItem {
int64_t offset;
//executors are identical but not concurrent
//so it must be a copy in each item
void* executor;
int64_t size;
void* content;
} tqBufferItem;
typedef struct tqGroupHandle {
char* topic; //c style, end with '\0'
int64_t cgId;
void* ahandle;
int64_t consumeOffset;
typedef struct tqBufferHandle {
//char* topic; //c style, end with '\0'
//int64_t cgId;
//void* ahandle;
int64_t nextConsumeOffset;
int64_t topicId;
int32_t head;
int32_t tail;
tqBufferItem buffer[TQ_BUFFER_SIZE];
} tqBufferHandle;
typedef struct tqListHandle {
tqBufferHandle* bufHandle;
struct tqListHandle* next;
} tqListHandle;
typedef struct tqGroupHandle {
int64_t cId;
int64_t cgId;
void* ahandle;
int32_t topicNum;
tqListHandle *head;
} tqGroupHandle;
typedef struct tqQueryExec {
void* src;
tqBufferItem* dest;
void* executor;
} tqQueryExec;
typedef struct tqQueryMsg {
tqQueryExec *exec;
struct tqQueryMsg *next;
} tqQueryMsg;
//init in each vnode
STQ* tqInit(void* ref_func(void*), void* unref_func(void*));
void tqCleanUp(STQ*);
......@@ -127,20 +166,33 @@ void tqCleanUp(STQ*);
int tqPushMsg(STQ*, void* msg, int64_t version);
int tqCommit(STQ*);
//void* will be replace by a msg type
int tqHandleConsumeMsg(STQ*, tmqConsumeReq* msg);
int tqConsume(STQ*, tmqConsumeReq*);
tqGroupHandle* tqFindGHandleBycId(STQ*, int64_t cId);
tqGroupHandle* tqGetGroupHandle(STQ*, int64_t cId);
int tqOpenTCGroup(STQ*, int64_t topicId, int64_t cgId, int64_t cId);
int tqCloseTCGroup(STQ*, int64_t topicId, int64_t cgId, int64_t cId);
int tqMoveOffsetToNext(tqGroupHandle*);
int tqResetOffset(STQ*, int64_t topicId, int64_t cgId, int64_t offset);
int tqFetchMsg(tqGroupHandle*, void*);
int tqRegisterContext(tqGroupHandle*, void*);
int tqLaunchQuery(STQ*, int64_t topicId, int64_t cgId, void* query);
int tqLaunchQuery(tqGroupHandle*);
int tqSendLaunchQuery(STQ*, int64_t topicId, int64_t cgId, void* query);
int tqSerializeGroupHandle(tqGroupHandle *gHandle, void** ppBytes, int32_t offset);
int tqSerializeListHandle(tqListHandle *listHandle, void** ppBytes, int32_t offset);
int tqSerializeBufHandle(tqBufferHandle *bufHandle, void** ppBytes, int32_t offset);
int tqSerializeBufItem(tqBufferItem *bufItem, void** ppBytes, int32_t offset);
int tqDeserializeGroupHandle(const void* pBytes, tqGroupHandle **pGhandle);
int tqDeserializeListHandle(const void* pBytes, tqListHandle **pListHandle);
int tqDeserializeBufHandle(const void* pBytes, tqBufferHandle **pBufHandle);
int tqDeserializeBufItem(const void* pBytes, tqBufferItem **pBufItem);
int tqGetGHandleSSize(const tqGroupHandle *gHandle);
int tqListHandleSSize(const tqListHandle *listHandle);
int tqBufHandleSSize(const tqBufferHandle *bufHandle);
int tqBufItemSSize(const tqBufferItem *bufItem);
#ifdef __cplusplus
}
#endif
......
include/util/tdef.h
浏览文件 @ 41290f9a
......@@ -24,12 +24,7 @@ extern "C" {
#define TSDB__packed
#ifdef TSKEY32
#define TSKEY int32_t;
#else
#define TSKEY int64_t
#endif
#define TSKEY_INITIAL_VAL INT64_MIN
// Bytes for each type.
......@@ -43,7 +38,6 @@ extern const int32_t TYPE_BYTES[15];
#define FLOAT_BYTES sizeof(float)
#define DOUBLE_BYTES sizeof(double)
#define POINTER_BYTES sizeof(void *) // 8 by default assert(sizeof(ptrdiff_t) == sizseof(void*)
#define TSDB_KEYSIZE sizeof(TSKEY)
#define TSDB_NCHAR_SIZE sizeof(int32_t)
......@@ -88,6 +82,7 @@ extern const int32_t TYPE_BYTES[15];
#define TSDB_ERR -1
#define TS_PATH_DELIMITER "."
#define TS_ESCAPE_CHAR '`'
#define TSDB_TIME_PRECISION_MILLI 0
#define TSDB_TIME_PRECISION_MICRO 1
......@@ -132,11 +127,12 @@ do { \
#define TSDB_RELATION_MATCH 14
#define TSDB_RELATION_NMATCH 15
#define TSDB_BINARY_OP_ADD 30
#define TSDB_BINARY_OP_SUBTRACT 31
#define TSDB_BINARY_OP_MULTIPLY 32
#define TSDB_BINARY_OP_DIVIDE 33
#define TSDB_BINARY_OP_REMAINDER 34
#define TSDB_BINARY_OP_ADD 4000
#define TSDB_BINARY_OP_SUBTRACT 4001
#define TSDB_BINARY_OP_MULTIPLY 4002
#define TSDB_BINARY_OP_DIVIDE 4003
#define TSDB_BINARY_OP_REMAINDER 4004
#define TSDB_BINARY_OP_CONCAT 4005
#define IS_RELATION_OPTR(op) (((op) >= TSDB_RELATION_LESS) && ((op) < TSDB_RELATION_IN))
......@@ -386,43 +382,6 @@ do { \
#define TSDB_DATA_TYPE_UINT 13 // 4 bytes
#define TSDB_DATA_TYPE_UBIGINT 14 // 8 bytes
// ----------------- For variable data types such as TSDB_DATA_TYPE_BINARY and TSDB_DATA_TYPE_NCHAR
//typedef int32_t VarDataOffsetT;
//typedef int16_t VarDataLenT; // maxVarDataLen: 32767
//typedef uint16_t TDRowLenT; // not including overhead: 0 ~ 65535
//typedef uint32_t TDRowTLenT; // total length, including overhead
//
//typedef struct tstr {
// VarDataLenT len;
// char data[];
//} tstr;
//
//#pragma pack(push, 1)
//typedef struct {
// VarDataLenT len;
// uint8_t data;
//} SBinaryNullT;
//
//typedef struct {
// VarDataLenT len;
// uint32_t data;
//} SNCharNullT;
//#pragma pack(pop)
//
//#define VARSTR_HEADER_SIZE sizeof(VarDataLenT)
//
//#define varDataLen(v) ((VarDataLenT *)(v))[0]
//#define varDataTLen(v) (sizeof(VarDataLenT) + varDataLen(v))
//#define varDataVal(v) ((void *)((char *)v + VARSTR_HEADER_SIZE))
//#define varDataCopy(dst, v) memcpy((dst), (void*) (v), varDataTLen(v))
//#define varDataLenByData(v) (*(VarDataLenT *)(((char*)(v)) - VARSTR_HEADER_SIZE))
//#define varDataSetLen(v, _len) (((VarDataLenT *)(v))[0] = (VarDataLenT) (_len))
//#define IS_VAR_DATA_TYPE(t) (((t) == TSDB_DATA_TYPE_BINARY) || ((t) == TSDB_DATA_TYPE_NCHAR))
//
//#define varDataNetLen(v) (htons(((VarDataLenT *)(v))[0]))
//#define varDataNetTLen(v) (sizeof(VarDataLenT) + varDataNetLen(v))
enum { TRANS_STAT_INIT = 0, TRANS_STAT_EXECUTING, TRANS_STAT_EXECUTED, TRANS_STAT_ROLLBACKING, TRANS_STAT_ROLLBACKED };
enum { TRANS_OPER_INIT = 0, TRANS_OPER_EXECUTE, TRANS_OPER_ROLLBACK };
......
include/util/tpagedfile.h 0 → 100644
浏览文件 @ 41290f9a
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef TDENGINE_TPAGEDFILE_H
#define TDENGINE_TPAGEDFILE_H
#ifdef __cplusplus
extern "C" {
#endif
#include "tlist.h"
#include "thash.h"
#include "os.h"
#include "tlockfree.h"
typedef struct SArray* SIDList;
typedef struct SPageDiskInfo {
int32_t offset;
int32_t length;
} SPageDiskInfo;
typedef struct SPageInfo {
SListNode* pn; // point to list node
int32_t pageId;
SPageDiskInfo info;
void* pData;
bool used; // set current page is in used
} SPageInfo;
typedef struct SFreeListItem {
int32_t offset;
int32_t len;
} SFreeListItem;
typedef struct SResultBufStatis {
int32_t flushBytes;
int32_t loadBytes;
int32_t getPages;
int32_t releasePages;
int32_t flushPages;
} SResultBufStatis;
typedef struct SDiskbasedResultBuf {
int32_t numOfPages;
int64_t totalBufSize;
int64_t fileSize; // disk file size
FILE* file;
int32_t allocateId; // allocated page id
char* path; // file path
int32_t pageSize; // current used page size
int32_t inMemPages; // numOfPages that are allocated in memory
SHashObj* groupSet; // id hash table
SHashObj* all;
SList* lruList;
void* emptyDummyIdList; // dummy id list
void* assistBuf; // assistant buffer for compress/decompress data
SArray* pFree; // free area in file
bool comp; // compressed before flushed to disk
int32_t nextPos; // next page flush position
uint64_t qId; // for debug purpose
SResultBufStatis statis;
} SDiskbasedResultBuf;
#define DEFAULT_INTERN_BUF_PAGE_SIZE (1024L) // in bytes
#define PAGE_INFO_INITIALIZER (SPageDiskInfo){-1, -1}
#define DEFAULT_PAGE_SIZE (16384L)
typedef struct SFilePage {
int64_t num;
char data[];
} SFilePage;
/**
* create disk-based result buffer
* @param pResultBuf
* @param rowSize
* @param pagesize
* @param inMemPages
* @param handle
* @return
*/
int32_t createDiskbasedResultBuffer(SDiskbasedResultBuf** pResultBuf, int32_t pagesize, int32_t inMemBufSize, uint64_t qId, const char* dir);
/**
*
* @param pResultBuf
* @param groupId
* @param pageId
* @return
*/
SFilePage* getNewDataBuf(SDiskbasedResultBuf* pResultBuf, int32_t groupId, int32_t* pageId);
/**
*
* @param pResultBuf
* @param groupId
* @return
*/
SIDList getDataBufPagesIdList(SDiskbasedResultBuf* pResultBuf, int32_t groupId);
/**
* get the specified buffer page by id
* @param pResultBuf
* @param id
* @return
*/
SFilePage* getResBufPage(SDiskbasedResultBuf* pResultBuf, int32_t id);
/**
* release the referenced buf pages
* @param pResultBuf
* @param page
*/
void releaseResBufPage(SDiskbasedResultBuf* pResultBuf, void* page);
/**
*
* @param pResultBuf
* @param pi
*/
void releaseResBufPageInfo(SDiskbasedResultBuf* pResultBuf, SPageInfo* pi);
/**
* get the total buffer size in the format of disk file
* @param pResultBuf
* @return
*/
size_t getResBufSize(const SDiskbasedResultBuf* pResultBuf);
/**
* get the number of groups in the result buffer
* @param pResultBuf
* @return
*/
size_t getNumOfResultBufGroupId(const SDiskbasedResultBuf* pResultBuf);
/**
* destroy result buffer
* @param pResultBuf
*/
void destroyResultBuf(SDiskbasedResultBuf* pResultBuf);
/**
*
* @param pList
* @return
*/
SPageInfo* getLastPageInfo(SIDList pList);
#ifdef __cplusplus
}
#endif
#endif // TDENGINE_TPAGEDFILE_H
include/util/tskiplist.h
浏览文件 @ 41290f9a
......@@ -21,7 +21,7 @@ extern "C" {
#endif
#include "os.h"
#include "tdef.h"
//#include "tdef.h"
#include "tarray.h"
#include "tfunctional.h"
......
include/util/tutil.h
浏览文件 @ 41290f9a
......@@ -26,6 +26,7 @@ extern "C" {
#include "tdef.h"
int32_t strdequote(char *src);
int32_t strndequote(char *dst, const char* z, int32_t len);
int32_t strRmquote(char *z, int32_t len);
size_t strtrim(char *src);
char * strnchr(char *haystack, char needle, int32_t len, bool skipquote);
......@@ -40,9 +41,6 @@ char * paGetToken(char *src, char **token, int32_t *tokenLen);
int32_t taosByteArrayToHexStr(char bytes[], int32_t len, char hexstr[]);
int32_t taosHexStrToByteArray(char hexstr[], char bytes[]);
//bool taosGetVersionNumber(char *versionStr, int *versionNubmer);
//int taosCheckVersion(char *input_client_version, char *input_server_version, int compared_segments);
char * taosIpStr(uint32_t ipInt);
uint32_t ip2uint(const char *const ip_addr);
......
source/common/CMakeLists.txt
浏览文件 @ 41290f9a
......@@ -11,3 +11,5 @@ target_link_libraries(
PUBLIC util
INTERFACE api
)
ADD_SUBDIRECTORY(test)
source/common/src/tname.c
浏览文件 @ 41290f9a
......@@ -120,29 +120,6 @@ int64_t taosGetIntervalStartTimestamp(int64_t startTime, int64_t slidingTime, in
#endif
/*
* tablePrefix.columnName
* extract table name and save it in pTable, with only column name in pToken
*/
//void extractTableNameFromToken(SStrToken* pToken, SStrToken* pTable) {
// const char sep = TS_PATH_DELIMITER[0];
//
// if (pToken == pTable || pToken == NULL || pTable == NULL) {
// return;
// }
//
// char* r = strnchr(pToken->z, sep, pToken->n, false);
//
// if (r != NULL) { // record the table name token
// pTable->n = (uint32_t)(r - pToken->z);
// pTable->z = pToken->z;
//
// r += 1;
// pToken->n -= (uint32_t)(r - pToken->z);
// pToken->z = r;
// }
//}
static struct SSchema _s = {
.colId = TSDB_TBNAME_COLUMN_INDEX,
.type = TSDB_DATA_TYPE_BINARY,
......
source/common/src/ttime.c
浏览文件 @ 41290f9a
......@@ -416,12 +416,14 @@ static int32_t getDuration(int64_t val, char unit, int64_t* result, int32_t time
return -1;
}
}
/* get the value in microsecond */
return 0;
}
/*
* n - months
* y - Years
* is not allowed, since the duration of month or year are both variable.
*
* b - nanoseconds;
* u - microseconds;
* a - Millionseconds
......@@ -430,8 +432,6 @@ static int32_t getDuration(int64_t val, char unit, int64_t* result, int32_t time
* h - Hours
* d - Days (24 hours)
* w - Weeks (7 days)
* n - Months (30 days)
* y - Years (365 days)
*/
int32_t parseAbsoluteDuration(char* token, int32_t tokenlen, int64_t* duration, char* unit, int32_t timePrecision) {
errno = 0;
......
source/common/src/tvariant.c
浏览文件 @ 41290f9a
......@@ -15,21 +15,66 @@
#include "os.h"
#include "taos.h"
#include "thash.h"
#include "taosdef.h"
#include "thash.h"
#include "ttime.h"
#include "ttokendef.h"
#include "ttypes.h"
#include "tutil.h"
#include "tvariant.h"
#define SET_EXT_INFO(converted, res, minv, maxv, exti) do { \
if (converted == NULL || exti == NULL || *converted == false) { break; } \
if ((res) < (minv)) { *exti = -1; break; } \
if ((res) > (maxv)) { *exti = 1; break; } \
#define SET_EXT_INFO(converted, res, minv, maxv, exti) \
do { \
if (converted == NULL || exti == NULL || *converted == false) { \
break; \
} \
if ((res) < (minv)) { \
*exti = -1; \
break; \
} \
if ((res) > (maxv)) { \
*exti = 1; \
break; \
} \
assert(0); \
} while (0)
int32_t toInteger(const char* z, int32_t n, int32_t base, int64_t* value, bool* isSigned) {
errno = 0;
char* endPtr = NULL;
int32_t index = 0;
bool specifiedSign = (z[0] == '+' || z[0] == '-');
if (specifiedSign) {
*isSigned = true;
index = 1;
}
uint64_t val = strtoull(&z[index], &endPtr, base);
if (errno == ERANGE || errno == EINVAL) {
errno = 0;
return -1;
}
if (specifiedSign && val > INT64_MAX) {
return -1;
}
if (endPtr - &z[index] != n - index) {
return -1;
}
*isSigned = specifiedSign || (val <= INT64_MAX);
if (*isSigned) {
*value = (z[0] == '-')? -val:val;
} else {
*(uint64_t*) value = val;
}
return 0;
}
void taosVariantCreate(SVariant *pVar, char* z, int32_t n, int32_t type) {
int32_t ret = 0;
memset(pVar, 0, sizeof(SVariant));
......@@ -43,7 +88,6 @@ void taosVariantCreate(SVariant *pVar, char* z, int32_t n, int32_t type) {
} else {
return;
}
break;
}
......@@ -51,38 +95,38 @@ void taosVariantCreate(SVariant *pVar, char* z, int32_t n, int32_t type) {
case TSDB_DATA_TYPE_SMALLINT:
case TSDB_DATA_TYPE_BIGINT:
case TSDB_DATA_TYPE_INT:{
// ret = tStrToInteger(token->z, token->type, token->n, &pVar->i64, true);
// if (ret != 0) {
// SToken t = {0};
// tGetToken(token->z, &t.type);
// if (t.type == TK_MINUS) { // it is a signed number which is greater than INT64_MAX or less than INT64_MIN
// pVar->nType = -1; // -1 means error type
// return;
// }
//
// // data overflow, try unsigned parse the input number
// ret = tStrToInteger(token->z, token->type, token->n, &pVar->i64, false);
// if (ret != 0) {
// pVar->nType = -1; // -1 means error type
// return;
// }
// }
bool sign = true;
break;
int32_t base = 10;
if (type == TK_HEX) {
base = 16;
} else if (type == TK_OCT) {
base = 8;
} else if (type == TK_BIN) {
base = 2;
}
ret = toInteger(z, n, base, &pVar->i64, &sign);
if (ret != 0) {
pVar->nType = -1; // -1 means error type
return;
}
pVar->nType = (sign)? TSDB_DATA_TYPE_BIGINT:TSDB_DATA_TYPE_UBIGINT;
break;
}
case TSDB_DATA_TYPE_DOUBLE:
case TSDB_DATA_TYPE_FLOAT: {
pVar->d = strtod(z, NULL);
break;
}
case TSDB_DATA_TYPE_BINARY: {
pVar->pz = strndup(z, n);
pVar->nLen = strRmquote(pVar->pz, n);
break;
}
case TSDB_DATA_TYPE_TIMESTAMP: {
assert(0);
pVar->i64 = taosGetTimestamp(TSDB_TIME_PRECISION_NANO);
break;
}
......@@ -95,7 +139,6 @@ void taosVariantCreate(SVariant *pVar, char* z, int32_t n, int32_t type) {
pVar->nType = type;
}
/**
* create SVariant from binary string, not ascii data
* @param pVar
......@@ -901,7 +944,7 @@ int32_t taosVariantDump(SVariant *pVariant, char *payload, int16_t type, bool in
*
* It is actually the bigint/binary/bool/nchar type transfer
*/
int32_t tVariantTypeSetType(SVariant *pVariant, char type) {
int32_t taosVariantTypeSetType(SVariant *pVariant, char type) {
if (pVariant == NULL || pVariant->nType == 0) { // value is not set
return 0;
}
......
source/common/src/versionUtil.c 0 → 100644
浏览文件 @ 41290f9a
#include "os.h"
#include "tdef.h"
#include "ulog.h"
#include "taoserror.h"
bool taosGetVersionNumber(char *versionStr, int *versionNubmer) {
if (versionStr == NULL || versionNubmer == NULL) {
return false;
}
int versionNumberPos[5] = {0};
int len = (int)strlen(versionStr);
int dot = 0;
for (int pos = 0; pos < len && dot < 4; ++pos) {
if (versionStr[pos] == '.') {
versionStr[pos] = 0;
versionNumberPos[++dot] = pos + 1;
}
}
if (dot != 3) {
return false;
}
for (int pos = 0; pos < 4; ++pos) {
versionNubmer[pos] = atoi(versionStr + versionNumberPos[pos]);
}
versionStr[versionNumberPos[1] - 1] = '.';
versionStr[versionNumberPos[2] - 1] = '.';
versionStr[versionNumberPos[3] - 1] = '.';
return true;
}
int taosCheckVersion(char *input_client_version, char *input_server_version, int comparedSegments) {
char client_version[TSDB_VERSION_LEN] = {0};
char server_version[TSDB_VERSION_LEN] = {0};
int clientVersionNumber[4] = {0};
int serverVersionNumber[4] = {0};
tstrncpy(client_version, input_client_version, sizeof(client_version));
tstrncpy(server_version, input_server_version, sizeof(server_version));
if (!taosGetVersionNumber(client_version, clientVersionNumber)) {
uError("invalid client version:%s", client_version);
return TSDB_CODE_TSC_INVALID_VERSION;
}
if (!taosGetVersionNumber(server_version, serverVersionNumber)) {
uError("invalid server version:%s", server_version);
return TSDB_CODE_TSC_INVALID_VERSION;
}
for(int32_t i = 0; i < comparedSegments; ++i) {
if (clientVersionNumber[i] != serverVersionNumber[i]) {
uError("the %d-th number of server version:%s not matched with client version:%s", i, server_version,
client_version);
return TSDB_CODE_TSC_INVALID_VERSION;
}
}
return 0;
}
source/common/test/CMakeLists.txt 0 → 100644
浏览文件 @ 41290f9a
MESSAGE(STATUS "build parser unit test")
# GoogleTest requires at least C++11
SET(CMAKE_CXX_STANDARD 11)
AUX_SOURCE_DIRECTORY(${CMAKE_CURRENT_SOURCE_DIR} SOURCE_LIST)
ADD_EXECUTABLE(commonTest ${SOURCE_LIST})
TARGET_LINK_LIBRARIES(
commonTest
PUBLIC os util common gtest
)
TARGET_INCLUDE_DIRECTORIES(
commonTest
PUBLIC "${CMAKE_SOURCE_DIR}/include/libs/common/"
PRIVATE "${CMAKE_SOURCE_DIR}/source/libs/common/inc"
)
source/common/test/commonTests.cpp
浏览文件 @ 41290f9a
#include <gtest/gtest.h>
#include <iostream>
#pragma GCC diagnostic ignored "-Wwrite-strings"
#pragma GCC diagnostic ignored "-Wunused-function"
#pragma GCC diagnostic ignored "-Wunused-variable"
#pragma GCC diagnostic ignored "-Wunused-but-set-variable"
#pragma GCC diagnostic ignored "-Wsign-compare"
#include "os.h"
#include "taos.h"
#include "tvariant.h"
#include "tdef.h"
namespace {
//
} // namespace
int main(int argc, char** argv) {
testing::InitGoogleTest(&argc, argv);
return RUN_ALL_TESTS();
}
TEST(testCase, toInteger_test) {
char* s = "123";
uint32_t type = 0;
int64_t val = 0;
bool sign = true;
int32_t ret = toInteger(s, strlen(s), 10, &val, &sign);
ASSERT_EQ(ret, 0);
ASSERT_EQ(val, 123);
ASSERT_EQ(sign, true);
s = "9223372036854775807";
ret = toInteger(s, strlen(s), 10, &val, &sign);
ASSERT_EQ(ret, 0);
ASSERT_EQ(val, 9223372036854775807);
ASSERT_EQ(sign, true);
s = "9323372036854775807";
ret = toInteger(s, strlen(s), 10, &val, &sign);
ASSERT_EQ(ret, 0);
ASSERT_EQ(val, 9323372036854775807u);
ASSERT_EQ(sign, false);
s = "-9323372036854775807";
ret = toInteger(s, strlen(s), 10, &val, &sign);
ASSERT_EQ(ret, -1);
s = "-1";
ret = toInteger(s, strlen(s), 10, &val, &sign);
ASSERT_EQ(ret, 0);
ASSERT_EQ(val, -1);
ASSERT_EQ(sign, true);
s = "-9223372036854775807";
ret = toInteger(s, strlen(s), 10, &val, &sign);
ASSERT_EQ(ret, 0);
ASSERT_EQ(val, -9223372036854775807);
ASSERT_EQ(sign, true);
s = "1000u";
ret = toInteger(s, strlen(s), 10, &val, &sign);
ASSERT_EQ(ret, -1);
s = "0x10";
ret = toInteger(s, strlen(s), 16, &val, &sign);
ASSERT_EQ(ret, 0);
ASSERT_EQ(val, 16);
ASSERT_EQ(sign, true);
s = "110";
ret = toInteger(s, strlen(s), 2, &val, &sign);
ASSERT_EQ(ret, 0);
ASSERT_EQ(val, 6);
ASSERT_EQ(sign, true);
s = "110";
ret = toInteger(s, strlen(s), 8, &val, &sign);
ASSERT_EQ(ret, 0);
ASSERT_EQ(val, 72);
ASSERT_EQ(sign, true);
//18446744073709551615 UINT64_MAX
s = "18446744073709551615";
ret = toInteger(s, strlen(s), 10, &val, &sign);
ASSERT_EQ(ret, 0);
ASSERT_EQ(val, 18446744073709551615u);
ASSERT_EQ(sign, false);
s = "18446744073709551616";
ret = toInteger(s, strlen(s), 10, &val, &sign);
ASSERT_EQ(ret, -1);
}
source/libs/CMakeLists.txt
浏览文件 @ 41290f9a
......@@ -9,3 +9,4 @@ add_subdirectory(lru)
add_subdirectory(catalog)
add_subdirectory(executor)
add_subdirectory(planner)
add_subdirectory(function)
\ No newline at end of file
source/libs/catalog/src/catalog.c
浏览文件 @ 41290f9a
......@@ -14,3 +14,11 @@
*/
#include "catalogInt.h"
struct SCatalog* getCatalogHandle(const SEpSet* pMgmtEps) {
return NULL;
}
int32_t catalogGetMetaData(struct SCatalog* pCatalog, const SMetaReq* pMetaReq, SMetaData* pMetaData) {
return 0;
}
source/libs/function/CMakeLists.txt 0 → 100644
浏览文件 @ 41290f9a
aux_source_directory(src FUNCTION_SRC)
add_library(function ${FUNCTION_SRC})
target_include_directories(
function
PUBLIC "${CMAKE_SOURCE_DIR}/include/libs/function"
PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc"
)
target_link_libraries(
function
PRIVATE os util common
)
\ No newline at end of file
source/libs/function/inc/taggfunction.h 0 → 100644
浏览文件 @ 41290f9a
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef TDENGINE_TAGGFUNCTION_H
#define TDENGINE_TAGGFUNCTION_H
#ifdef __cplusplus
extern "C" {
#endif
#include "os.h"
#include "tname.h"
#include "taosdef.h"
#include "tvariant.h"
#include "function.h"
#include "tudf.h"
extern SAggFunctionInfo aggFunc[34];
typedef struct SResultRowCellInfo {
int8_t hasResult; // result generated, not NULL value
bool initialized; // output buffer has been initialized
bool complete; // query has completed
uint32_t numOfRes; // num of output result in current buffer
} SResultRowCellInfo;
#define FUNCSTATE_SO 0x0u
#define FUNCSTATE_MO 0x1u // dynamic number of output, not multinumber of output e.g., TOP/BOTTOM
#define FUNCSTATE_STREAM 0x2u // function avail for stream
#define FUNCSTATE_STABLE 0x4u // function avail for super table
#define FUNCSTATE_NEED_TS 0x8u // timestamp is required during query processing
#define FUNCSTATE_SELECTIVITY 0x10u // selectivity functions, can exists along with tag columns
#define BASIC_FUNC_SO FUNCSTATE_SO | FUNCSTATE_STREAM | FUNCSTATE_STABLE
#define BASIC_FUNC_MO FUNCSTATE_MO | FUNCSTATE_STREAM | FUNCSTATE_STABLE
#define AVG_FUNCTION_INTER_BUFFER_SIZE 50
#define DATA_SET_FLAG ',' // to denote the output area has data, not null value
#define DATA_SET_FLAG_SIZE sizeof(DATA_SET_FLAG)
#define QUERY_ASC_FORWARD_STEP 1
#define QUERY_DESC_FORWARD_STEP -1
#define GET_FORWARD_DIRECTION_FACTOR(ord) (((ord) == TSDB_ORDER_ASC) ? QUERY_ASC_FORWARD_STEP : QUERY_DESC_FORWARD_STEP)
#define MAX_INTERVAL_TIME_WINDOW 1000000 // maximum allowed time windows in final results
#define TOP_BOTTOM_QUERY_LIMIT 100
enum {
MASTER_SCAN = 0x0u,
REVERSE_SCAN = 0x1u,
REPEAT_SCAN = 0x2u, //repeat scan belongs to the master scan
MERGE_STAGE = 0x20u,
};
#define QUERY_IS_STABLE_QUERY(type) (((type)&TSDB_QUERY_TYPE_STABLE_QUERY) != 0)
#define QUERY_IS_JOIN_QUERY(type) (TSDB_QUERY_HAS_TYPE(type, TSDB_QUERY_TYPE_JOIN_QUERY))
#define QUERY_IS_PROJECTION_QUERY(type) (((type)&TSDB_QUERY_TYPE_PROJECTION_QUERY) != 0)
#define QUERY_IS_FREE_RESOURCE(type) (((type)&TSDB_QUERY_TYPE_FREE_RESOURCE) != 0)
typedef struct SArithmeticSupport {
struct SExprInfo *pExprInfo;
int32_t numOfCols;
SColumnInfo *colList;
void *exprList; // client side used
int32_t offset;
char** data;
} SArithmeticSupport;
typedef struct SInterpInfoDetail {
TSKEY ts; // interp specified timestamp
int8_t type;
int8_t primaryCol;
} SInterpInfoDetail;
#define GET_ROWCELL_INTERBUF(_c) ((void*) ((char*)(_c) + sizeof(SResultRowCellInfo)))
#define GET_RES_INFO(ctx) ((ctx)->resultInfo)
#define IS_STREAM_QUERY_VALID(x) (((x)&TSDB_FUNCSTATE_STREAM) != 0)
#define IS_MULTIOUTPUT(x) (((x)&TSDB_FUNCSTATE_MO) != 0)
// determine the real data need to calculated the result
enum {
BLK_DATA_NO_NEEDED = 0x0,
BLK_DATA_STATIS_NEEDED = 0x1,
BLK_DATA_ALL_NEEDED = 0x3,
BLK_DATA_DISCARD = 0x4, // discard current data block since it is not qualified for filter
};
typedef struct STwaInfo {
int8_t hasResult; // flag to denote has value
double dOutput;
SPoint1 p;
STimeWindow win;
} STwaInfo;
extern int32_t functionCompatList[]; // compatible check array list
bool topbot_datablock_filter(SQLFunctionCtx *pCtx, const char *minval, const char *maxval);
/**
* the numOfRes should be kept, since it may be used later
* and allow the ResultInfo to be re initialized
*/
#define RESET_RESULT_INFO(_r) \
do { \
(_r)->initialized = false; \
} while (0)
static FORCE_INLINE void initResultInfo(SResultRowCellInfo *pResInfo, int32_t bufLen) {
pResInfo->initialized = true; // the this struct has been initialized flag
pResInfo->complete = false;
pResInfo->hasResult = false;
pResInfo->numOfRes = 0;
memset(GET_ROWCELL_INTERBUF(pResInfo), 0, bufLen);
}
#ifdef __cplusplus
}
#endif
#endif // TDENGINE_TAGGFUNCTION_H
source/libs/executor/inc/texpr.h → source/libs/function/inc/texpr.h
浏览文件 @ 41290f9a
......@@ -25,7 +25,7 @@ extern "C" {
#include "taosmsg.h"
#include "taosdef.h"
#include "tskiplist.h"
#include "tbuffer.h"
#include "function.h"
struct tExprNode;
struct SSchema;
......@@ -43,13 +43,6 @@ struct SSchema;
typedef bool (*__result_filter_fn_t)(const void *, void *);
typedef void (*__do_filter_suppl_fn_t)(void *, void *);
enum {
TSQL_NODE_DUMMY = 0x0,
TSQL_NODE_EXPR = 0x1,
TSQL_NODE_COL = 0x2,
TSQL_NODE_VALUE = 0x4,
};
/**
* this structure is used to filter data in tags, so the offset of filtered tag column in tagdata string is required
*/
......@@ -61,37 +54,16 @@ typedef struct tQueryInfo {
bool indexed; // indexed columns
} tQueryInfo;
typedef struct tExprNode {
uint8_t nodeType;
union {
struct {
uint8_t optr; // filter operator
uint8_t hasPK; // 0: do not contain primary filter, 1: contain
void *info; // support filter operation on this expression only available for leaf node
struct tExprNode *pLeft; // left child pointer
struct tExprNode *pRight; // right child pointer
} _node;
SSchema *pSchema;
struct SVariant *pVal;
};
} tExprNode;
typedef struct SExprTraverseSupp {
__result_filter_fn_t nodeFilterFn;
__do_filter_suppl_fn_t setupInfoFn;
void *pExtInfo;
} SExprTraverseSupp;
void tExprTreeDestroy(tExprNode *pNode, void (*fp)(void *));
void exprTreeToBinary(SBufferWriter* bw, tExprNode* pExprTree);
tExprNode* exprTreeFromBinary(const void* data, size_t size);
tExprNode* exprTreeFromTableName(const char* tbnameCond);
tExprNode* exprdup(tExprNode* pTree);
void exprTreeToBinary(SBufferWriter* bw, tExprNode* pExprTree);
bool exprTreeApplyFilter(tExprNode *pExpr, const void *pItem, SExprTraverseSupp *param);
void arithmeticTreeTraverse(tExprNode *pExprs, int32_t numOfRows, char *pOutput, void *param, int32_t order,
......
source/libs/function/inc/tfill.h 0 → 100644
浏览文件 @ 41290f9a
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef TDENGINE_TFILL_H
#define TDENGINE_TFILL_H
#ifdef __cplusplus
extern "C" {
#endif
#include "os.h"
#include "taosdef.h"
struct SSDataBlock;
typedef struct {
STColumn col; // column info
int16_t functionId; // sql function id
int16_t flag; // column flag: TAG COLUMN|NORMAL COLUMN
int16_t tagIndex; // index of current tag in SFillTagColInfo array list
union {int64_t i; double d;} fillVal;
} SFillColInfo;
typedef struct {
SSchema col;
char* tagVal;
} SFillTagColInfo;
typedef struct SFillInfo {
TSKEY start; // start timestamp
TSKEY end; // endKey for fill
TSKEY currentKey; // current active timestamp, the value may be changed during the fill procedure.
int32_t order; // order [TSDB_ORDER_ASC|TSDB_ORDER_DESC]
int32_t type; // fill type
int32_t numOfRows; // number of rows in the input data block
int32_t index; // active row index
int32_t numOfTotal; // number of filled rows in one round
int32_t numOfCurrent; // number of filled rows in current results
int32_t numOfTags; // number of tags
int32_t numOfCols; // number of columns, including the tags columns
int32_t rowSize; // size of each row
SInterval interval;
char * prevValues; // previous row of data, to generate the interpolation results
char * nextValues; // next row of data
char** pData; // original result data block involved in filling data
int32_t alloc; // data buffer size in rows
int8_t precision; // time resoluation
SFillColInfo* pFillCol; // column info for fill operations
SFillTagColInfo* pTags; // tags value for filling gap
void* handle; // for debug purpose
} SFillInfo;
typedef struct SPoint {
int64_t key;
void * val;
} SPoint;
SFillInfo* taosCreateFillInfo(int32_t order, TSKEY skey, int32_t numOfTags, int32_t capacity, int32_t numOfCols,
int64_t slidingTime, int8_t slidingUnit, int8_t precision, int32_t fillType,
SFillColInfo* pFillCol, void* handle);
void taosResetFillInfo(SFillInfo* pFillInfo, TSKEY startTimestamp);
void* taosDestroyFillInfo(SFillInfo *pFillInfo);
void taosFillSetStartInfo(SFillInfo* pFillInfo, int32_t numOfRows, TSKEY endKey);
void taosFillSetInputDataBlock(SFillInfo* pFillInfo, const struct SSDataBlock* pInput);
bool taosFillHasMoreResults(SFillInfo* pFillInfo);
int64_t getNumOfResultsAfterFillGap(SFillInfo* pFillInfo, int64_t ekey, int32_t maxNumOfRows);
int32_t taosGetLinearInterpolationVal(SPoint* point, int32_t outputType, SPoint* point1, SPoint* point2, int32_t inputType);
int64_t taosFillResultDataBlock(SFillInfo* pFillInfo, void** output, int32_t capacity);
#ifdef __cplusplus
}
#endif
#endif // TDENGINE_TFILL_H
source/libs/function/inc/thistogram.h 0 → 100644
浏览文件 @ 41290f9a
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef TDENGINE_HISTOGRAM_H
#define TDENGINE_HISTOGRAM_H
#ifdef __cplusplus
extern "C" {
#endif
#define USE_ARRAYLIST
#define MAX_HISTOGRAM_BIN 500
typedef struct SHistBin {
double val;
int64_t num;
#if !defined(USE_ARRAYLIST)
double delta;
int32_t index; // index in min-heap list
#endif
} SHistBin;
typedef struct SHeapEntry {
void* pData;
double val;
} SHeapEntry;
typedef struct SHistogramInfo {
int64_t numOfElems;
int32_t numOfEntries;
int32_t maxEntries;
double min;
double max;
#if defined(USE_ARRAYLIST)
SHistBin* elems;
#else
tSkipList* pList;
SLoserTreeInfo* pLoserTree;
int32_t maxIndex;
bool ordered;
#endif
} SHistogramInfo;
SHistogramInfo* tHistogramCreate(int32_t numOfBins);
SHistogramInfo* tHistogramCreateFrom(void* pBuf, int32_t numOfBins);
int32_t tHistogramAdd(SHistogramInfo** pHisto, double val);
int64_t tHistogramSum(SHistogramInfo* pHisto, double v);
double* tHistogramUniform(SHistogramInfo* pHisto, double* ratio, int32_t num);
SHistogramInfo* tHistogramMerge(SHistogramInfo* pHisto1, SHistogramInfo* pHisto2, int32_t numOfEntries);
void tHistogramDestroy(SHistogramInfo** pHisto);
void tHistogramPrint(SHistogramInfo* pHisto);
int32_t histoBinarySearch(SHistBin* pEntry, int32_t len, double val);
SHeapEntry* tHeapCreate(int32_t numOfEntries);
void tHeapSort(SHeapEntry* pEntry, int32_t len);
#ifdef __cplusplus
}
#endif
#endif // TDENGINE_HISTOGRAM_H
source/libs/function/inc/tpercentile.h 0 → 100644
浏览文件 @ 41290f9a
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef TDENGINE_TPERCENTILE_H
#define TDENGINE_TPERCENTILE_H
#ifdef __cplusplus
extern "C" {
#endif
#include "tpagedfile.h"
#include "ttszip.h"
typedef struct MinMaxEntry {
union {
double dMinVal;
int64_t i64MinVal;
uint64_t u64MinVal;
};
union {
double dMaxVal;
int64_t i64MaxVal;
int64_t u64MaxVal;
};
} MinMaxEntry;
typedef struct {
int32_t size;
int32_t pageId;
SFilePage *data;
} SSlotInfo;
typedef struct tMemBucketSlot {
SSlotInfo info;
MinMaxEntry range;
} tMemBucketSlot;
struct tMemBucket;
typedef int32_t (*__perc_hash_func_t)(struct tMemBucket *pBucket, const void *value);
typedef struct tMemBucket {
int16_t numOfSlots;
int16_t type;
int16_t bytes;
int32_t total;
int32_t elemPerPage; // number of elements for each object
int32_t maxCapacity; // maximum allowed number of elements that can be sort directly to get the result
int32_t bufPageSize; // disk page size
MinMaxEntry range; // value range
int32_t times; // count that has been checked for deciding the correct data value buckets.
__compar_fn_t comparFn;
tMemBucketSlot * pSlots;
SDiskbasedResultBuf *pBuffer;
__perc_hash_func_t hashFunc;
} tMemBucket;
tMemBucket *tMemBucketCreate(int16_t nElemSize, int16_t dataType, double minval, double maxval);
void tMemBucketDestroy(tMemBucket *pBucket);
int32_t tMemBucketPut(tMemBucket *pBucket, const void *data, size_t size);
double getPercentile(tMemBucket *pMemBucket, double percent);
#endif // TDENGINE_TPERCENTILE_H
#ifdef __cplusplus
}
#endif
\ No newline at end of file
source/libs/function/inc/tscalarfunction.h 0 → 100644
浏览文件 @ 41290f9a
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef TDENGINE_TSCALARFUNCTION_H
#define TDENGINE_TSCALARFUNCTION_H
#ifdef __cplusplus
extern "C" {
#endif
#include "function.h"
extern struct SScalarFunctionInfo scalarFunc[1];
#define FUNCTION_CEIL 38
#define FUNCTION_FLOOR 39
#define FUNCTION_ROUND 40
#define FUNCTION_MAVG 41
#define FUNCTION_CSUM 42
#ifdef __cplusplus
}
#endif
#endif // TDENGINE_TSCALARFUNCTION_H
source/libs/function/inc/ttszip.h 0 → 100644
浏览文件 @ 41290f9a
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef TDENGINE_TTSZIP_H
#define TDENGINE_TTSZIP_H
#ifdef __cplusplus
extern "C" {
#endif
#include "os.h"
#include "taosdef.h"
#include "tvariant.h"
#define MEM_BUF_SIZE (1 << 20)
#define TS_COMP_FILE_MAGIC 0x87F5EC4C
#define TS_COMP_FILE_GROUP_MAX 512
typedef struct STSList {
char* rawBuf;
int32_t allocSize;
int32_t threshold;
int32_t len;
} STSList;
typedef struct STSElem {
TSKEY ts;
SVariant* tag;
int32_t id;
} STSElem;
typedef struct STSCursor {
int32_t vgroupIndex;
int32_t blockIndex;
int32_t tsIndex;
uint32_t order;
} STSCursor;
typedef struct STSBlock {
SVariant tag; // tag value
int32_t numOfElem; // number of elements
int32_t compLen; // size after compressed
int32_t padding; // 0xFFFFFFFF by default, after the payload
char* payload; // actual data that is compressed
} STSBlock;
/*
* The size of buffer file should not be greater than 2G,
* and the offset of int32_t type is enough
*/
typedef struct STSGroupBlockInfo {
int32_t id; // group id
int32_t offset; // offset set value in file
int32_t numOfBlocks; // number of total blocks
int32_t compLen; // compressed size
} STSGroupBlockInfo;
typedef struct STSGroupBlockInfoEx {
STSGroupBlockInfo info;
int32_t len; // length before compress
} STSGroupBlockInfoEx;
typedef struct STSBuf {
FILE* f;
char path[PATH_MAX];
uint32_t fileSize;
// todo use array
STSGroupBlockInfoEx* pData;
uint32_t numOfAlloc;
uint32_t numOfGroups;
char* assistBuf;
int32_t bufSize;
STSBlock block;
STSList tsData; // uncompressed raw ts data
uint64_t numOfTotal;
bool autoDelete;
bool remainOpen;
int32_t tsOrder; // order of timestamp in ts comp buffer
STSCursor cur;
} STSBuf;
typedef struct STSBufFileHeader {
uint32_t magic; // file magic number
uint32_t numOfGroup; // number of group stored in current file
int32_t tsOrder; // timestamp order in current file
} STSBufFileHeader;
STSBuf* tsBufCreate(bool autoDelete, int32_t order);
STSBuf* tsBufCreateFromFile(const char* path, bool autoDelete);
STSBuf* tsBufCreateFromCompBlocks(const char* pData, int32_t numOfBlocks, int32_t len, int32_t tsOrder, int32_t id);
void* tsBufDestroy(STSBuf* pTSBuf);
void tsBufAppend(STSBuf* pTSBuf, int32_t id, SVariant* tag, const char* pData, int32_t len);
int32_t tsBufMerge(STSBuf* pDestBuf, const STSBuf* pSrcBuf);
STSBuf* tsBufClone(STSBuf* pTSBuf);
STSGroupBlockInfo* tsBufGetGroupBlockInfo(STSBuf* pTSBuf, int32_t id);
void tsBufFlush(STSBuf* pTSBuf);
void tsBufResetPos(STSBuf* pTSBuf);
bool tsBufNextPos(STSBuf* pTSBuf);
STSElem tsBufGetElem(STSBuf* pTSBuf);
STSElem tsBufGetElemStartPos(STSBuf* pTSBuf, int32_t id, SVariant* tag);
STSCursor tsBufGetCursor(STSBuf* pTSBuf);
void tsBufSetTraverseOrder(STSBuf* pTSBuf, int32_t order);
void tsBufSetCursor(STSBuf* pTSBuf, STSCursor* pCur);
/**
* display all data in comp block file, for debug purpose only
* @param pTSBuf
*/
void tsBufDisplay(STSBuf* pTSBuf);
int32_t tsBufGetNumOfGroup(STSBuf* pTSBuf);
void tsBufGetGroupIdList(STSBuf* pTSBuf, int32_t* num, int32_t** id);
int32_t dumpFileBlockByGroupId(STSBuf* pTSBuf, int32_t id, void* buf, int32_t* len, int32_t* numOfBlocks);
STSElem tsBufFindElemStartPosByTag(STSBuf* pTSBuf, SVariant* pTag);
bool tsBufIsValidElem(STSElem* pElem);
#ifdef __cplusplus
}
#endif
#endif // TDENGINE_TTSZIP_H
source/libs/function/inc/tudf.h 0 → 100644
浏览文件 @ 41290f9a
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef TDENGINE_TUDF_H
#define TDENGINE_TUDF_H
enum {
TSDB_UDF_FUNC_NORMAL = 0,
TSDB_UDF_FUNC_INIT,
TSDB_UDF_FUNC_FINALIZE,
TSDB_UDF_FUNC_MERGE,
TSDB_UDF_FUNC_DESTROY,
TSDB_UDF_FUNC_MAX_NUM
};
typedef struct SUdfInit {
int32_t maybe_null; /* 1 if function can return NULL */
uint32_t decimals; /* for real functions */
uint64_t length; /* For string functions */
char* ptr; /* free pointer for function data */
int32_t const_item; /* 0 if result is independent of arguments */
// script like lua/javascript
void* script_ctx;
void (*destroyCtxFunc)(void* script_ctx);
} SUdfInit;
typedef struct SUdfInfo {
int32_t functionId; // system assigned function id
int32_t funcType; // scalar function or aggregate function
int8_t resType; // result type
int16_t resBytes; // result byte
int32_t contLen; // content length
int32_t bufSize; // interbuf size
char* name; // function name
void* handle; // handle loaded in mem
void* funcs[TSDB_UDF_FUNC_MAX_NUM]; // function ptr
// for script like lua/javascript only
int isScript;
void* pScriptCtx;
SUdfInit init;
char* content;
char* path;
} SUdfInfo;
// script
typedef int32_t (*scriptInitFunc)(void* pCtx);
typedef void (*scriptNormalFunc)(void* pCtx, char* data, int16_t iType, int16_t iBytes, int32_t numOfRows,
int64_t* ptList, int64_t key, char* dataOutput, char* tsOutput, int32_t* numOfOutput,
int16_t oType, int16_t oBytes);
typedef void (*scriptFinalizeFunc)(void* pCtx, int64_t key, char* dataOutput, int32_t* numOfOutput);
typedef void (*scriptMergeFunc)(void* pCtx, char* data, int32_t numOfRows, char* dataOutput, int32_t* numOfOutput);
typedef void (*scriptDestroyFunc)(void* pCtx);
// dynamic lib
typedef void (*udfNormalFunc)(char* data, int16_t itype, int16_t iBytes, int32_t numOfRows, int64_t* ts,
char* dataOutput, char* interBuf, char* tsOutput, int32_t* numOfOutput, int16_t oType,
int16_t oBytes, SUdfInit* buf);
typedef int32_t (*udfInitFunc)(SUdfInit* data);
typedef void (*udfFinalizeFunc)(char* dataOutput, char* interBuf, int32_t* numOfOutput, SUdfInit* buf);
typedef void (*udfMergeFunc)(char* data, int32_t numOfRows, char* dataOutput, int32_t* numOfOutput, SUdfInit* buf);
typedef void (*udfDestroyFunc)(SUdfInit* buf);
#endif // TDENGINE_TUDF_H
source/libs/function/src/taggfunction.c 0 → 100644
浏览文件 @ 41290f9a
此差异已折叠。
source/libs/executor/src/texpr.c → source/libs/function/src/texpr.c
浏览文件 @ 41290f9a
......@@ -15,7 +15,6 @@
#include "os.h"
#include "texpr.h"
#include "exception.h"
#include "taosdef.h"
#include "taosmsg.h"
......@@ -26,21 +25,21 @@
#include "thash.h"
#include "tskiplist.h"
#include "texpr.h"
#include "tarithoperator.h"
//#include "tarithoperator.h"
#include "tvariant.h"
static uint8_t UNUSED_FUNC isQueryOnPrimaryKey(const char *primaryColumnName, const tExprNode *pLeft, const tExprNode *pRight) {
if (pLeft->nodeType == TSQL_NODE_COL) {
// if left node is the primary column,return true
return (strcmp(primaryColumnName, pLeft->pSchema->name) == 0) ? 1 : 0;
} else {
// if any children have query on primary key, their parents are also keep this value
return ((pLeft->nodeType == TSQL_NODE_EXPR && pLeft->_node.hasPK == 1) ||
(pRight->nodeType == TSQL_NODE_EXPR && pRight->_node.hasPK == 1)) == true
? 1
: 0;
}
}
//static uint8_t UNUSED_FUNC isQueryOnPrimaryKey(const char *primaryColumnName, const tExprNode *pLeft, const tExprNode *pRight) {
// if (pLeft->nodeType == TEXPR_COL_NODE) {
// // if left node is the primary column,return true
// return (strcmp(primaryColumnName, pLeft->pSchema->name) == 0) ? 1 : 0;
// } else {
// // if any children have query on primary key, their parents are also keep this value
// return ((pLeft->nodeType == TEXPR_BINARYEXPR_NODE && pLeft->_node.hasPK == 1) ||
// (pRight->nodeType == TEXPR_BINARYEXPR_NODE && pRight->_node.hasPK == 1)) == true
// ? 1
// : 0;
// }
//}
static void reverseCopy(char* dest, const char* src, int16_t type, int32_t numOfRows) {
switch(type) {
......@@ -114,11 +113,11 @@ void tExprTreeDestroy(tExprNode *pNode, void (*fp)(void *)) {
return;
}
if (pNode->nodeType == TSQL_NODE_EXPR) {
if (pNode->nodeType == TEXPR_BINARYEXPR_NODE || pNode->nodeType == TEXPR_UNARYEXPR_NODE) {
doExprTreeDestroy(&pNode, fp);
} else if (pNode->nodeType == TSQL_NODE_VALUE) {
} else if (pNode->nodeType == TEXPR_VALUE_NODE) {
taosVariantDestroy(pNode->pVal);
} else if (pNode->nodeType == TSQL_NODE_COL) {
} else if (pNode->nodeType == TEXPR_COL_NODE) {
tfree(pNode->pSchema);
}
......@@ -130,17 +129,17 @@ static void doExprTreeDestroy(tExprNode **pExpr, void (*fp)(void *)) {
return;
}
if ((*pExpr)->nodeType == TSQL_NODE_EXPR) {
if ((*pExpr)->nodeType == TEXPR_BINARYEXPR_NODE) {
doExprTreeDestroy(&(*pExpr)->_node.pLeft, fp);
doExprTreeDestroy(&(*pExpr)->_node.pRight, fp);
if (fp != NULL) {
fp((*pExpr)->_node.info);
}
} else if ((*pExpr)->nodeType == TSQL_NODE_VALUE) {
} else if ((*pExpr)->nodeType == TEXPR_VALUE_NODE) {
taosVariantDestroy((*pExpr)->pVal);
free((*pExpr)->pVal);
} else if ((*pExpr)->nodeType == TSQL_NODE_COL) {
} else if ((*pExpr)->nodeType == TEXPR_COL_NODE) {
free((*pExpr)->pSchema);
}
......@@ -153,7 +152,7 @@ bool exprTreeApplyFilter(tExprNode *pExpr, const void *pItem, SExprTraverseSupp
tExprNode *pRight = pExpr->_node.pRight;
//non-leaf nodes, recursively traverse the expression tree in the post-root order
if (pLeft->nodeType == TSQL_NODE_EXPR && pRight->nodeType == TSQL_NODE_EXPR) {
if (pLeft->nodeType == TEXPR_BINARYEXPR_NODE && pRight->nodeType == TEXPR_BINARYEXPR_NODE) {
if (pExpr->_node.optr == TSDB_RELATION_OR) { // or
if (exprTreeApplyFilter(pLeft, pItem, param)) {
return true;
......@@ -180,26 +179,26 @@ void arithmeticTreeTraverse(tExprNode *pExprs, int32_t numOfRows, char *pOutput,
if (pExprs == NULL) {
return;
}
#if 0
tExprNode *pLeft = pExprs->_node.pLeft;
tExprNode *pRight = pExprs->_node.pRight;
/* the left output has result from the left child syntax tree */
char *pLeftOutput = (char*)malloc(sizeof(int64_t) * numOfRows);
if (pLeft->nodeType == TSQL_NODE_EXPR) {
if (pLeft->nodeType == TEXPR_BINARYEXPR_NODE) {
arithmeticTreeTraverse(pLeft, numOfRows, pLeftOutput, param, order, getSourceDataBlock);
}
/* the right output has result from the right child syntax tree */
// the right output has result from the right child syntax tree
char *pRightOutput = malloc(sizeof(int64_t) * numOfRows);
char *pdata = malloc(sizeof(int64_t) * numOfRows);
if (pRight->nodeType == TSQL_NODE_EXPR) {
if (pRight->nodeType == TEXPR_BINARYEXPR_NODE) {
arithmeticTreeTraverse(pRight, numOfRows, pRightOutput, param, order, getSourceDataBlock);
}
if (pLeft->nodeType == TSQL_NODE_EXPR) {
if (pRight->nodeType == TSQL_NODE_EXPR) {
if (pLeft->nodeType == TEXPR_BINARYEXPR_NODE) {
if (pRight->nodeType == TEXPR_BINARYEXPR_NODE) {
/*
* exprLeft + exprRight
* the type of returned value of one expression is always double float precious
......@@ -207,7 +206,7 @@ void arithmeticTreeTraverse(tExprNode *pExprs, int32_t numOfRows, char *pOutput,
_arithmetic_operator_fn_t OperatorFn = getArithmeticOperatorFn(pExprs->_node.optr);
OperatorFn(pLeftOutput, numOfRows, TSDB_DATA_TYPE_DOUBLE, pRightOutput, numOfRows, TSDB_DATA_TYPE_DOUBLE, pOutput, TSDB_ORDER_ASC);
} else if (pRight->nodeType == TSQL_NODE_COL) { // exprLeft + columnRight
} else if (pRight->nodeType == TEXPR_COL_NODE) { // exprLeft + columnRight
_arithmetic_operator_fn_t OperatorFn = getArithmeticOperatorFn(pExprs->_node.optr);
// set input buffer
......@@ -219,14 +218,14 @@ void arithmeticTreeTraverse(tExprNode *pExprs, int32_t numOfRows, char *pOutput,
OperatorFn(pLeftOutput, numOfRows, TSDB_DATA_TYPE_DOUBLE, pInputData, numOfRows, pRight->pSchema->type, pOutput, TSDB_ORDER_ASC);
}
} else if (pRight->nodeType == TSQL_NODE_VALUE) { // exprLeft + 12
} else if (pRight->nodeType == TEXPR_VALUE_NODE) { // exprLeft + 12
_arithmetic_operator_fn_t OperatorFn = getArithmeticOperatorFn(pExprs->_node.optr);
OperatorFn(pLeftOutput, numOfRows, TSDB_DATA_TYPE_DOUBLE, &pRight->pVal->i64, 1, pRight->pVal->nType, pOutput, TSDB_ORDER_ASC);
}
} else if (pLeft->nodeType == TSQL_NODE_COL) {
} else if (pLeft->nodeType == TEXPR_COL_NODE) {
// column data specified on left-hand-side
char *pLeftInputData = getSourceDataBlock(param, pLeft->pSchema->name, pLeft->pSchema->colId);
if (pRight->nodeType == TSQL_NODE_EXPR) { // columnLeft + expr2
if (pRight->nodeType == TEXPR_BINARYEXPR_NODE) { // columnLeft + expr2
_arithmetic_operator_fn_t OperatorFn = getArithmeticOperatorFn(pExprs->_node.optr);
if (order == TSDB_ORDER_DESC) {
......@@ -236,14 +235,14 @@ void arithmeticTreeTraverse(tExprNode *pExprs, int32_t numOfRows, char *pOutput,
OperatorFn(pLeftInputData, numOfRows, pLeft->pSchema->type, pRightOutput, numOfRows, TSDB_DATA_TYPE_DOUBLE, pOutput, TSDB_ORDER_ASC);
}
} else if (pRight->nodeType == TSQL_NODE_COL) { // columnLeft + columnRight
} else if (pRight->nodeType == TEXPR_COL_NODE) { // columnLeft + columnRight
// column data specified on right-hand-side
char *pRightInputData = getSourceDataBlock(param, pRight->pSchema->name, pRight->pSchema->colId);
_arithmetic_operator_fn_t OperatorFn = getArithmeticOperatorFn(pExprs->_node.optr);
// both columns are descending order, do not reverse the source data
OperatorFn(pLeftInputData, numOfRows, pLeft->pSchema->type, pRightInputData, numOfRows, pRight->pSchema->type, pOutput, order);
} else if (pRight->nodeType == TSQL_NODE_VALUE) { // columnLeft + 12
} else if (pRight->nodeType == TEXPR_VALUE_NODE) { // columnLeft + 12
_arithmetic_operator_fn_t OperatorFn = getArithmeticOperatorFn(pExprs->_node.optr);
if (order == TSDB_ORDER_DESC) {
......@@ -255,11 +254,11 @@ void arithmeticTreeTraverse(tExprNode *pExprs, int32_t numOfRows, char *pOutput,
}
} else {
// column data specified on left-hand-side
if (pRight->nodeType == TSQL_NODE_EXPR) { // 12 + expr2
if (pRight->nodeType == TEXPR_BINARYEXPR_NODE) { // 12 + expr2
_arithmetic_operator_fn_t OperatorFn = getArithmeticOperatorFn(pExprs->_node.optr);
OperatorFn(&pLeft->pVal->i64, 1, pLeft->pVal->nType, pRightOutput, numOfRows, TSDB_DATA_TYPE_DOUBLE, pOutput, TSDB_ORDER_ASC);
} else if (pRight->nodeType == TSQL_NODE_COL) { // 12 + columnRight
} else if (pRight->nodeType == TEXPR_COL_NODE) { // 12 + columnRight
// column data specified on right-hand-side
char *pRightInputData = getSourceDataBlock(param, pRight->pSchema->name, pRight->pSchema->colId);
_arithmetic_operator_fn_t OperatorFn = getArithmeticOperatorFn(pExprs->_node.optr);
......@@ -271,7 +270,7 @@ void arithmeticTreeTraverse(tExprNode *pExprs, int32_t numOfRows, char *pOutput,
OperatorFn(&pLeft->pVal->i64, 1, pLeft->pVal->nType, pRightInputData, numOfRows, pRight->pSchema->type, pOutput, TSDB_ORDER_ASC);
}
} else if (pRight->nodeType == TSQL_NODE_VALUE) { // 12 + 12
} else if (pRight->nodeType == TEXPR_VALUE_NODE) { // 12 + 12
_arithmetic_operator_fn_t OperatorFn = getArithmeticOperatorFn(pExprs->_node.optr);
OperatorFn(&pLeft->pVal->i64, 1, pLeft->pVal->nType, &pRight->pVal->i64, 1, pRight->pVal->nType, pOutput, TSDB_ORDER_ASC);
}
......@@ -280,12 +279,14 @@ void arithmeticTreeTraverse(tExprNode *pExprs, int32_t numOfRows, char *pOutput,
tfree(pdata);
tfree(pLeftOutput);
tfree(pRightOutput);
#endif
}
static void exprTreeToBinaryImpl(SBufferWriter* bw, tExprNode* expr) {
tbufWriteUint8(bw, expr->nodeType);
if (expr->nodeType == TSQL_NODE_VALUE) {
if (expr->nodeType == TEXPR_VALUE_NODE) {
SVariant* pVal = expr->pVal;
tbufWriteUint32(bw, pVal->nType);
......@@ -296,16 +297,15 @@ static void exprTreeToBinaryImpl(SBufferWriter* bw, tExprNode* expr) {
tbufWriteInt64(bw, pVal->i64);
}
} else if (expr->nodeType == TSQL_NODE_COL) {
} else if (expr->nodeType == TEXPR_COL_NODE) {
SSchema* pSchema = expr->pSchema;
tbufWriteInt16(bw, pSchema->colId);
tbufWriteInt16(bw, pSchema->bytes);
tbufWriteUint8(bw, pSchema->type);
tbufWriteString(bw, pSchema->name);
} else if (expr->nodeType == TSQL_NODE_EXPR) {
} else if (expr->nodeType == TEXPR_BINARYEXPR_NODE) {
tbufWriteUint8(bw, expr->_node.optr);
tbufWriteUint8(bw, expr->_node.hasPK);
exprTreeToBinaryImpl(bw, expr->_node.pLeft);
exprTreeToBinaryImpl(bw, expr->_node.pRight);
}
......@@ -353,7 +353,7 @@ static tExprNode* exprTreeFromBinaryImpl(SBufferReader* br) {
CLEANUP_PUSH_VOID_PTR_PTR(true, tExprTreeDestroy, pExpr, NULL);
pExpr->nodeType = tbufReadUint8(br);
if (pExpr->nodeType == TSQL_NODE_VALUE) {
if (pExpr->nodeType == TEXPR_VALUE_NODE) {
SVariant* pVal = exception_calloc(1, sizeof(SVariant));
pExpr->pVal = pVal;
......@@ -366,7 +366,7 @@ static tExprNode* exprTreeFromBinaryImpl(SBufferReader* br) {
pVal->i64 = tbufReadInt64(br);
}
} else if (pExpr->nodeType == TSQL_NODE_COL) {
} else if (pExpr->nodeType == TEXPR_COL_NODE) {
SSchema* pSchema = exception_calloc(1, sizeof(SSchema));
pExpr->pSchema = pSchema;
......@@ -375,9 +375,8 @@ static tExprNode* exprTreeFromBinaryImpl(SBufferReader* br) {
pSchema->type = tbufReadUint8(br);
tbufReadToString(br, pSchema->name, TSDB_COL_NAME_LEN);
} else if (pExpr->nodeType == TSQL_NODE_EXPR) {
} else if (pExpr->nodeType == TEXPR_BINARYEXPR_NODE) {
pExpr->_node.optr = tbufReadUint8(br);
pExpr->_node.hasPK = tbufReadUint8(br);
pExpr->_node.pLeft = exprTreeFromBinaryImpl(br);
pExpr->_node.pRight = exprTreeFromBinaryImpl(br);
assert(pExpr->_node.pLeft != NULL && pExpr->_node.pRight != NULL);
......@@ -406,12 +405,12 @@ tExprNode* exprTreeFromTableName(const char* tbnameCond) {
tExprNode* expr = exception_calloc(1, sizeof(tExprNode));
CLEANUP_PUSH_VOID_PTR_PTR(true, tExprTreeDestroy, expr, NULL);
expr->nodeType = TSQL_NODE_EXPR;
expr->nodeType = TEXPR_BINARYEXPR_NODE;
tExprNode* left = exception_calloc(1, sizeof(tExprNode));
expr->_node.pLeft = left;
left->nodeType = TSQL_NODE_COL;
left->nodeType = TEXPR_COL_NODE;
SSchema* pSchema = exception_calloc(1, sizeof(SSchema));
left->pSchema = pSchema;
......@@ -421,7 +420,7 @@ tExprNode* exprTreeFromTableName(const char* tbnameCond) {
expr->_node.pRight = right;
if (strncmp(tbnameCond, QUERY_COND_REL_PREFIX_LIKE, QUERY_COND_REL_PREFIX_LIKE_LEN) == 0) {
right->nodeType = TSQL_NODE_VALUE;
right->nodeType = TEXPR_VALUE_NODE;
expr->_node.optr = TSDB_RELATION_LIKE;
SVariant* pVal = exception_calloc(1, sizeof(SVariant));
right->pVal = pVal;
......@@ -432,7 +431,7 @@ tExprNode* exprTreeFromTableName(const char* tbnameCond) {
pVal->nLen = (int32_t)len;
} else if (strncmp(tbnameCond, QUERY_COND_REL_PREFIX_MATCH, QUERY_COND_REL_PREFIX_MATCH_LEN) == 0) {
right->nodeType = TSQL_NODE_VALUE;
right->nodeType = TEXPR_VALUE_NODE;
expr->_node.optr = TSDB_RELATION_MATCH;
SVariant* pVal = exception_calloc(1, sizeof(SVariant));
right->pVal = pVal;
......@@ -442,7 +441,7 @@ tExprNode* exprTreeFromTableName(const char* tbnameCond) {
pVal->nType = TSDB_DATA_TYPE_BINARY;
pVal->nLen = (int32_t)len;
} else if (strncmp(tbnameCond, QUERY_COND_REL_PREFIX_NMATCH, QUERY_COND_REL_PREFIX_NMATCH_LEN) == 0) {
right->nodeType = TSQL_NODE_VALUE;
right->nodeType = TEXPR_VALUE_NODE;
expr->_node.optr = TSDB_RELATION_NMATCH;
SVariant* pVal = exception_calloc(1, sizeof(SVariant));
right->pVal = pVal;
......@@ -452,7 +451,7 @@ tExprNode* exprTreeFromTableName(const char* tbnameCond) {
pVal->nType = TSDB_DATA_TYPE_BINARY;
pVal->nLen = (int32_t)len;
} else if (strncmp(tbnameCond, QUERY_COND_REL_PREFIX_IN, QUERY_COND_REL_PREFIX_IN_LEN) == 0) {
right->nodeType = TSQL_NODE_VALUE;
right->nodeType = TEXPR_VALUE_NODE;
expr->_node.optr = TSDB_RELATION_IN;
SVariant* pVal = exception_calloc(1, sizeof(SVariant));
right->pVal = pVal;
......@@ -699,25 +698,23 @@ err_ret:
tfree(tmp);
}
tExprNode* exprdup(tExprNode* pNode) {
if (pNode == NULL) {
return NULL;
}
tExprNode* pCloned = calloc(1, sizeof(tExprNode));
if (pNode->nodeType == TSQL_NODE_EXPR) {
if (pNode->nodeType == TEXPR_BINARYEXPR_NODE) {
tExprNode* pLeft = exprdup(pNode->_node.pLeft);
tExprNode* pRight = exprdup(pNode->_node.pRight);
pCloned->_node.pLeft = pLeft;
pCloned->_node.pRight = pRight;
pCloned->_node.optr = pNode->_node.optr;
pCloned->_node.hasPK = pNode->_node.hasPK;
} else if (pNode->nodeType == TSQL_NODE_VALUE) {
} else if (pNode->nodeType == TEXPR_VALUE_NODE) {
pCloned->pVal = calloc(1, sizeof(SVariant));
taosVariantAssign(pCloned->pVal, pNode->pVal);
} else if (pNode->nodeType == TSQL_NODE_COL) {
} else if (pNode->nodeType == TEXPR_COL_NODE) {
pCloned->pSchema = calloc(1, sizeof(SSchema));
*pCloned->pSchema = *pNode->pSchema;
}
......
source/libs/function/src/tfill.c 0 → 100644
浏览文件 @ 41290f9a
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "os.h"
#include "taosdef.h"
#include "taosmsg.h"
#include "ttypes.h"
#include "tfill.h"
#include "thash.h"
#include "function.h"
#include "common.h"
#include "ttime.h"
#define FILL_IS_ASC_FILL(_f) ((_f)->order == TSDB_ORDER_ASC)
#define DO_INTERPOLATION(_v1, _v2, _k1, _k2, _k) ((_v1) + ((_v2) - (_v1)) * (((double)(_k)) - ((double)(_k1))) / (((double)(_k2)) - ((double)(_k1))))
#define GET_FORWARD_DIRECTION_FACTOR(_ord) (((_ord) == TSDB_ORDER_ASC)? 1:-1)
static void setTagsValue(SFillInfo* pFillInfo, void** data, int32_t genRows) {
for(int32_t j = 0; j < pFillInfo->numOfCols; ++j) {
SFillColInfo* pCol = &pFillInfo->pFillCol[j];
if (TSDB_COL_IS_NORMAL_COL(pCol->flag) || TSDB_COL_IS_UD_COL(pCol->flag)) {
continue;
}
char* val1 = elePtrAt(data[j], pCol->col.bytes, genRows);
assert(pCol->tagIndex >= 0 && pCol->tagIndex < pFillInfo->numOfTags);
SFillTagColInfo* pTag = &pFillInfo->pTags[pCol->tagIndex];
assert (pTag->col.colId == pCol->col.colId);
assignVal(val1, pTag->tagVal, pCol->col.bytes, pCol->col.type);
}
}
static void setNullValueForRow(SFillInfo* pFillInfo, void** data, int32_t numOfCol, int32_t rowIndex) {
// the first are always the timestamp column, so start from the second column.
for (int32_t i = 1; i < numOfCol; ++i) {
SFillColInfo* pCol = &pFillInfo->pFillCol[i];
char* output = elePtrAt(data[i], pCol->col.bytes, rowIndex);
setNull(output, pCol->col.type, pCol->col.bytes);
}
}
static void doFillOneRowResult(SFillInfo* pFillInfo, void** data, char** srcData, int64_t ts, bool outOfBound) {
char* prev = pFillInfo->prevValues;
char* next = pFillInfo->nextValues;
SPoint point1, point2, point;
int32_t step = GET_FORWARD_DIRECTION_FACTOR(pFillInfo->order);
// set the primary timestamp column value
int32_t index = pFillInfo->numOfCurrent;
char* val = elePtrAt(data[0], TSDB_KEYSIZE, index);
*(TSKEY*) val = pFillInfo->currentKey;
// set the other values
if (pFillInfo->type == TSDB_FILL_PREV) {
char* p = FILL_IS_ASC_FILL(pFillInfo) ? prev : next;
if (p != NULL) {
for (int32_t i = 1; i < pFillInfo->numOfCols; ++i) {
SFillColInfo* pCol = &pFillInfo->pFillCol[i];
if (TSDB_COL_IS_TAG(pCol->flag)) {
continue;
}
char* output = elePtrAt(data[i], pCol->col.bytes, index);
assignVal(output, p + pCol->col.offset, pCol->col.bytes, pCol->col.type);
}
} else { // no prev value yet, set the value for NULL
setNullValueForRow(pFillInfo, data, pFillInfo->numOfCols, index);
}
} else if (pFillInfo->type == TSDB_FILL_NEXT) {
char* p = FILL_IS_ASC_FILL(pFillInfo)? next : prev;
if (p != NULL) {
for (int32_t i = 1; i < pFillInfo->numOfCols; ++i) {
SFillColInfo* pCol = &pFillInfo->pFillCol[i];
if (TSDB_COL_IS_TAG(pCol->flag)) {
continue;
}
char* output = elePtrAt(data[i], pCol->col.bytes, index);
assignVal(output, p + pCol->col.offset, pCol->col.bytes, pCol->col.type);
}
} else { // no prev value yet, set the value for NULL
setNullValueForRow(pFillInfo, data, pFillInfo->numOfCols, index);
}
} else if (pFillInfo->type == TSDB_FILL_LINEAR) {
// TODO : linear interpolation supports NULL value
if (prev != NULL && !outOfBound) {
for (int32_t i = 1; i < pFillInfo->numOfCols; ++i) {
SFillColInfo* pCol = &pFillInfo->pFillCol[i];
if (TSDB_COL_IS_TAG(pCol->flag)) {
continue;
}
int16_t type = pCol->col.type;
int16_t bytes = pCol->col.bytes;
char *val1 = elePtrAt(data[i], pCol->col.bytes, index);
if (type == TSDB_DATA_TYPE_BINARY|| type == TSDB_DATA_TYPE_NCHAR || type == TSDB_DATA_TYPE_BOOL) {
setNull(val1, pCol->col.type, bytes);
continue;
}
point1 = (SPoint){.key = *(TSKEY*)(prev), .val = prev + pCol->col.offset};
point2 = (SPoint){.key = ts, .val = srcData[i] + pFillInfo->index * bytes};
point = (SPoint){.key = pFillInfo->currentKey, .val = val1};
taosGetLinearInterpolationVal(&point, type, &point1, &point2, type);
}
} else {
setNullValueForRow(pFillInfo, data, pFillInfo->numOfCols, index);
}
} else { // fill the default value */
for (int32_t i = 1; i < pFillInfo->numOfCols; ++i) {
SFillColInfo* pCol = &pFillInfo->pFillCol[i];
if (TSDB_COL_IS_TAG(pCol->flag)/* || IS_VAR_DATA_TYPE(pCol->col.type)*/) {
continue;
}
char* val1 = elePtrAt(data[i], pCol->col.bytes, index);
assignVal(val1, (char*)&pCol->fillVal.i, pCol->col.bytes, pCol->col.type);
}
}
setTagsValue(pFillInfo, data, index);
pFillInfo->currentKey = taosTimeAdd(pFillInfo->currentKey, pFillInfo->interval.sliding * step, pFillInfo->interval.slidingUnit, pFillInfo->precision);
pFillInfo->numOfCurrent++;
}
static void initBeforeAfterDataBuf(SFillInfo* pFillInfo, char** next) {
if (*next != NULL) {
return;
}
*next = calloc(1, pFillInfo->rowSize);
for (int i = 1; i < pFillInfo->numOfCols; i++) {
SFillColInfo* pCol = &pFillInfo->pFillCol[i];
setNull(*next + pCol->col.offset, pCol->col.type, pCol->col.bytes);
}
}
static void copyCurrentRowIntoBuf(SFillInfo* pFillInfo, char** srcData, char* buf) {
int32_t rowIndex = pFillInfo->index;
for (int32_t i = 0; i < pFillInfo->numOfCols; ++i) {
SFillColInfo* pCol = &pFillInfo->pFillCol[i];
memcpy(buf + pCol->col.offset, srcData[i] + rowIndex * pCol->col.bytes, pCol->col.bytes);
}
}
static int32_t fillResultImpl(SFillInfo* pFillInfo, void** data, int32_t outputRows) {
pFillInfo->numOfCurrent = 0;
char** srcData = pFillInfo->pData;
char** prev = &pFillInfo->prevValues;
char** next = &pFillInfo->nextValues;
int32_t step = GET_FORWARD_DIRECTION_FACTOR(pFillInfo->order);
if (FILL_IS_ASC_FILL(pFillInfo)) {
assert(pFillInfo->currentKey >= pFillInfo->start);
} else {
assert(pFillInfo->currentKey <= pFillInfo->start);
}
while (pFillInfo->numOfCurrent < outputRows) {
int64_t ts = ((int64_t*)pFillInfo->pData[0])[pFillInfo->index];
// set the next value for interpolation
if ((pFillInfo->currentKey < ts && FILL_IS_ASC_FILL(pFillInfo)) ||
(pFillInfo->currentKey > ts && !FILL_IS_ASC_FILL(pFillInfo))) {
initBeforeAfterDataBuf(pFillInfo, next);
copyCurrentRowIntoBuf(pFillInfo, srcData, *next);
}
if (((pFillInfo->currentKey < ts && FILL_IS_ASC_FILL(pFillInfo)) || (pFillInfo->currentKey > ts && !FILL_IS_ASC_FILL(pFillInfo))) &&
pFillInfo->numOfCurrent < outputRows) {
// fill the gap between two actual input rows
while (((pFillInfo->currentKey < ts && FILL_IS_ASC_FILL(pFillInfo)) ||
(pFillInfo->currentKey > ts && !FILL_IS_ASC_FILL(pFillInfo))) &&
pFillInfo->numOfCurrent < outputRows) {
doFillOneRowResult(pFillInfo, data, srcData, ts, false);
}
// output buffer is full, abort
if (pFillInfo->numOfCurrent == outputRows) {
pFillInfo->numOfTotal += pFillInfo->numOfCurrent;
return outputRows;
}
} else {
assert(pFillInfo->currentKey == ts);
initBeforeAfterDataBuf(pFillInfo, prev);
if (pFillInfo->type == TSDB_FILL_NEXT && (pFillInfo->index + 1) < pFillInfo->numOfRows) {
initBeforeAfterDataBuf(pFillInfo, next);
++pFillInfo->index;
copyCurrentRowIntoBuf(pFillInfo, srcData, *next);
--pFillInfo->index;
}
// assign rows to dst buffer
for (int32_t i = 0; i < pFillInfo->numOfCols; ++i) {
SFillColInfo* pCol = &pFillInfo->pFillCol[i];
if (TSDB_COL_IS_TAG(pCol->flag)/* || IS_VAR_DATA_TYPE(pCol->col.type)*/) {
continue;
}
char* output = elePtrAt(data[i], pCol->col.bytes, pFillInfo->numOfCurrent);
char* src = elePtrAt(srcData[i], pCol->col.bytes, pFillInfo->index);
if (i == 0 || (pCol->functionId != FUNCTION_COUNT && !isNull(src, pCol->col.type)) ||
(pCol->functionId == FUNCTION_COUNT && GET_INT64_VAL(src) != 0)) {
assignVal(output, src, pCol->col.bytes, pCol->col.type);
memcpy(*prev + pCol->col.offset, src, pCol->col.bytes);
} else { // i > 0 and data is null , do interpolation
if (pFillInfo->type == TSDB_FILL_PREV) {
assignVal(output, *prev + pCol->col.offset, pCol->col.bytes, pCol->col.type);
} else if (pFillInfo->type == TSDB_FILL_LINEAR) {
assignVal(output, src, pCol->col.bytes, pCol->col.type);
memcpy(*prev + pCol->col.offset, src, pCol->col.bytes);
} else if (pFillInfo->type == TSDB_FILL_NEXT) {
if (*next) {
assignVal(output, *next + pCol->col.offset, pCol->col.bytes, pCol->col.type);
} else {
setNull(output, pCol->col.type, pCol->col.bytes);
}
} else {
assignVal(output, (char*)&pCol->fillVal.i, pCol->col.bytes, pCol->col.type);
}
}
}
// set the tag value for final result
setTagsValue(pFillInfo, data, pFillInfo->numOfCurrent);
pFillInfo->currentKey = taosTimeAdd(pFillInfo->currentKey, pFillInfo->interval.sliding * step,
pFillInfo->interval.slidingUnit, pFillInfo->precision);
pFillInfo->index += 1;
pFillInfo->numOfCurrent += 1;
}
if (pFillInfo->index >= pFillInfo->numOfRows || pFillInfo->numOfCurrent >= outputRows) {
/* the raw data block is exhausted, next value does not exists */
if (pFillInfo->index >= pFillInfo->numOfRows) {
tfree(*next);
}
pFillInfo->numOfTotal += pFillInfo->numOfCurrent;
return pFillInfo->numOfCurrent;
}
}
return pFillInfo->numOfCurrent;
}
static int64_t appendFilledResult(SFillInfo* pFillInfo, void** output, int64_t resultCapacity) {
/*
* These data are generated according to fill strategy, since the current timestamp is out of the time window of
* real result set. Note that we need to keep the direct previous result rows, to generated the filled data.
*/
pFillInfo->numOfCurrent = 0;
while (pFillInfo->numOfCurrent < resultCapacity) {
doFillOneRowResult(pFillInfo, output, pFillInfo->pData, pFillInfo->start, true);
}
pFillInfo->numOfTotal += pFillInfo->numOfCurrent;
assert(pFillInfo->numOfCurrent == resultCapacity);
return resultCapacity;
}
// there are no duplicated tags in the SFillTagColInfo list
static int32_t setTagColumnInfo(SFillInfo* pFillInfo, int32_t numOfCols, int32_t capacity) {
int32_t rowsize = 0;
int32_t numOfTags = 0;
int32_t k = 0;
for (int32_t i = 0; i < numOfCols; ++i) {
SFillColInfo* pColInfo = &pFillInfo->pFillCol[i];
pFillInfo->pData[i] = NULL;
if (TSDB_COL_IS_TAG(pColInfo->flag) || pColInfo->col.type == TSDB_DATA_TYPE_BINARY) {
numOfTags += 1;
bool exists = false;
int32_t index = -1;
for (int32_t j = 0; j < k; ++j) {
if (pFillInfo->pTags[j].col.colId == pColInfo->col.colId) {
exists = true;
index = j;
break;
}
}
if (!exists) {
SSchema* pSchema = &pFillInfo->pTags[k].col;
pSchema->colId = pColInfo->col.colId;
pSchema->type = pColInfo->col.type;
pSchema->bytes = pColInfo->col.bytes;
pFillInfo->pTags[k].tagVal = calloc(1, pColInfo->col.bytes);
pColInfo->tagIndex = k;
k += 1;
} else {
pColInfo->tagIndex = index;
}
}
rowsize += pColInfo->col.bytes;
}
pFillInfo->numOfTags = numOfTags;
assert(k <= pFillInfo->numOfTags);
return rowsize;
}
static int32_t taosNumOfRemainRows(SFillInfo* pFillInfo) {
if (pFillInfo->numOfRows == 0 || (pFillInfo->numOfRows > 0 && pFillInfo->index >= pFillInfo->numOfRows)) {
return 0;
}
return pFillInfo->numOfRows - pFillInfo->index;
}
SFillInfo* taosCreateFillInfo(int32_t order, TSKEY skey, int32_t numOfTags, int32_t capacity, int32_t numOfCols,
int64_t slidingTime, int8_t slidingUnit, int8_t precision, int32_t fillType,
SFillColInfo* pCol, void* handle) {
if (fillType == TSDB_FILL_NONE) {
return NULL;
}
SFillInfo* pFillInfo = calloc(1, sizeof(SFillInfo));
taosResetFillInfo(pFillInfo, skey);
pFillInfo->order = order;
pFillInfo->type = fillType;
pFillInfo->pFillCol = pCol;
pFillInfo->numOfTags = numOfTags;
pFillInfo->numOfCols = numOfCols;
pFillInfo->precision = precision;
pFillInfo->alloc = capacity;
pFillInfo->handle = handle;
pFillInfo->interval.interval = slidingTime;
pFillInfo->interval.intervalUnit = slidingUnit;
pFillInfo->interval.sliding = slidingTime;
pFillInfo->interval.slidingUnit = slidingUnit;
pFillInfo->pData = malloc(POINTER_BYTES * numOfCols);
// if (numOfTags > 0) {
pFillInfo->pTags = calloc(numOfCols, sizeof(SFillTagColInfo));
for (int32_t i = 0; i < numOfCols; ++i) {
pFillInfo->pTags[i].col.colId = -2; // TODO
}
// }
pFillInfo->rowSize = setTagColumnInfo(pFillInfo, pFillInfo->numOfCols, pFillInfo->alloc);
assert(pFillInfo->rowSize > 0);
return pFillInfo;
}
void taosResetFillInfo(SFillInfo* pFillInfo, TSKEY startTimestamp) {
pFillInfo->start = startTimestamp;
pFillInfo->currentKey = startTimestamp;
pFillInfo->end = startTimestamp;
pFillInfo->index = -1;
pFillInfo->numOfRows = 0;
pFillInfo->numOfCurrent = 0;
pFillInfo->numOfTotal = 0;
}
void* taosDestroyFillInfo(SFillInfo* pFillInfo) {
if (pFillInfo == NULL) {
return NULL;
}
tfree(pFillInfo->prevValues);
tfree(pFillInfo->nextValues);
for(int32_t i = 0; i < pFillInfo->numOfTags; ++i) {
tfree(pFillInfo->pTags[i].tagVal);
}
tfree(pFillInfo->pTags);
tfree(pFillInfo->pData);
tfree(pFillInfo->pFillCol);
tfree(pFillInfo);
return NULL;
}
void taosFillSetStartInfo(SFillInfo* pFillInfo, int32_t numOfRows, TSKEY endKey) {
if (pFillInfo->type == TSDB_FILL_NONE) {
return;
}
pFillInfo->end = endKey;
if (!FILL_IS_ASC_FILL(pFillInfo)) {
pFillInfo->end = taosTimeTruncate(endKey, &pFillInfo->interval, pFillInfo->precision);
}
pFillInfo->index = 0;
pFillInfo->numOfRows = numOfRows;
}
void taosFillSetInputDataBlock(SFillInfo* pFillInfo, const SSDataBlock* pInput) {
for (int32_t i = 0; i < pFillInfo->numOfCols; ++i) {
SFillColInfo* pCol = &pFillInfo->pFillCol[i];
SColumnInfoData* pColData = taosArrayGet(pInput->pDataBlock, i);
pFillInfo->pData[i] = pColData->pData;
if (TSDB_COL_IS_TAG(pCol->flag)) { // copy the tag value to tag value buffer
SFillTagColInfo* pTag = &pFillInfo->pTags[pCol->tagIndex];
assert (pTag->col.colId == pCol->col.colId);
memcpy(pTag->tagVal, pColData->pData, pCol->col.bytes); // TODO not memcpy??
}
}
}
bool taosFillHasMoreResults(SFillInfo* pFillInfo) {
int32_t remain = taosNumOfRemainRows(pFillInfo);
if (remain > 0) {
return true;
}
if (pFillInfo->numOfTotal > 0 && (((pFillInfo->end > pFillInfo->start) && FILL_IS_ASC_FILL(pFillInfo)) ||
(pFillInfo->end < pFillInfo->start && !FILL_IS_ASC_FILL(pFillInfo)))) {
return getNumOfResultsAfterFillGap(pFillInfo, pFillInfo->end, 4096) > 0;
}
return false;
}
int64_t getNumOfResultsAfterFillGap(SFillInfo* pFillInfo, TSKEY ekey, int32_t maxNumOfRows) {
int64_t* tsList = (int64_t*) pFillInfo->pData[0];
int32_t numOfRows = taosNumOfRemainRows(pFillInfo);
TSKEY ekey1 = ekey;
if (!FILL_IS_ASC_FILL(pFillInfo)) {
pFillInfo->end = taosTimeTruncate(ekey, &pFillInfo->interval, pFillInfo->precision);
}
int64_t numOfRes = -1;
if (numOfRows > 0) { // still fill gap within current data block, not generating data after the result set.
TSKEY lastKey = tsList[pFillInfo->numOfRows - 1];
numOfRes = taosTimeCountInterval(
lastKey,
pFillInfo->currentKey,
pFillInfo->interval.sliding,
pFillInfo->interval.slidingUnit,
pFillInfo->precision);
numOfRes += 1;
assert(numOfRes >= numOfRows);
} else { // reach the end of data
if ((ekey1 < pFillInfo->currentKey && FILL_IS_ASC_FILL(pFillInfo)) ||
(ekey1 > pFillInfo->currentKey && !FILL_IS_ASC_FILL(pFillInfo))) {
return 0;
}
numOfRes = taosTimeCountInterval(
ekey1,
pFillInfo->currentKey,
pFillInfo->interval.sliding,
pFillInfo->interval.slidingUnit,
pFillInfo->precision);
numOfRes += 1;
}
return (numOfRes > maxNumOfRows) ? maxNumOfRows : numOfRes;
}
int32_t taosGetLinearInterpolationVal(SPoint* point, int32_t outputType, SPoint* point1, SPoint* point2, int32_t inputType) {
double v1 = -1, v2 = -1;
GET_TYPED_DATA(v1, double, inputType, point1->val);
GET_TYPED_DATA(v2, double, inputType, point2->val);
double r = DO_INTERPOLATION(v1, v2, point1->key, point2->key, point->key);
SET_TYPED_DATA(point->val, outputType, r);
return TSDB_CODE_SUCCESS;
}
int64_t taosFillResultDataBlock(SFillInfo* pFillInfo, void** output, int32_t capacity) {
int32_t remain = taosNumOfRemainRows(pFillInfo);
int64_t numOfRes = getNumOfResultsAfterFillGap(pFillInfo, pFillInfo->end, capacity);
assert(numOfRes <= capacity);
// no data existed for fill operation now, append result according to the fill strategy
if (remain == 0) {
appendFilledResult(pFillInfo, output, numOfRes);
} else {
fillResultImpl(pFillInfo, output, (int32_t) numOfRes);
assert(numOfRes == pFillInfo->numOfCurrent);
}
// qDebug("fill:%p, generated fill result, src block:%d, index:%d, brange:%"PRId64"-%"PRId64", currentKey:%"PRId64", current:%d, total:%d, %p",
// pFillInfo, pFillInfo->numOfRows, pFillInfo->index, pFillInfo->start, pFillInfo->end, pFillInfo->currentKey, pFillInfo->numOfCurrent,
// pFillInfo->numOfTotal, pFillInfo->handle);
return numOfRes;
}
source/libs/function/src/tfunction.c 0 → 100644
浏览文件 @ 41290f9a
#include "os.h"
#include "tarray.h"
#include "function.h"
#include "thash.h"
#include "taggfunction.h"
#include "tscalarfunction.h"
static SHashObj* functionHashTable = NULL;
static void doInitFunctionHashTable() {
int numOfEntries = tListLen(aggFunc);
functionHashTable = taosHashInit(numOfEntries, MurmurHash3_32, false, false);
for (int32_t i = 0; i < numOfEntries; i++) {
int32_t len = (uint32_t)strlen(aggFunc[i].name);
SAggFunctionInfo* ptr = &aggFunc[i];
taosHashPut(functionHashTable, aggFunc[i].name, len, (void*)&ptr, POINTER_BYTES);
}
numOfEntries = tListLen(scalarFunc);
for(int32_t i = 0; i < numOfEntries; ++i) {
int32_t len = (int32_t) strlen(scalarFunc[i].name);
SScalarFunctionInfo* ptr = &scalarFunc[i];
taosHashPut(functionHashTable, scalarFunc[i].name, len, (void*)&ptr, POINTER_BYTES);
}
}
static pthread_once_t functionHashTableInit = PTHREAD_ONCE_INIT;
int32_t qIsBuiltinFunction(const char* name, int32_t len) {
pthread_once(&functionHashTableInit, doInitFunctionHashTable);
SAggFunctionInfo** pInfo = taosHashGet(functionHashTable, name, len);
if (pInfo != NULL) {
return (*pInfo)->functionId;
} else {
return -1;
}
}
bool qIsValidUdf(SArray* pUdfInfo, const char* name, int32_t len, int32_t* functionId) {
return true;
}
const char* qGetFunctionName(int32_t functionId) {
}
bool isTagsQuery(SArray* pFunctionIdList) {
int32_t num = (int32_t) taosArrayGetSize(pFunctionIdList);
for (int32_t i = 0; i < num; ++i) {
int16_t f = *(int16_t*) taosArrayGet(pFunctionIdList, i);
// "select count(tbname)" query
// if (functId == FUNCTION_COUNT && pExpr->base.colInfo.colId == TSDB_TBNAME_COLUMN_INDEX) {
// continue;
// }
if (f != FUNCTION_TAGPRJ && f != FUNCTION_TID_TAG) {
return false;
}
}
return true;
}
//bool tscMultiRoundQuery(SArray* pFunctionIdList, int32_t index) {
// if (!UTIL_TABLE_IS_SUPER_TABLE(pQueryInfo->pTableMetaInfo[index])) {
// return false;
// }
//
// size_t numOfExprs = (int32_t) getNumOfExprs(pQueryInfo);
// for(int32_t i = 0; i < numOfExprs; ++i) {
// SExprInfo* pExpr = getExprInfo(pQueryInfo, i);
// if (pExpr->base.functionId == FUNCTION_STDDEV_DST) {
// return true;
// }
// }
//
// return false;
//}
bool isBlockInfoQuery(SArray* pFunctionIdList) {
int32_t num = (int32_t) taosArrayGetSize(pFunctionIdList);
for (int32_t i = 0; i < num; ++i) {
int32_t f = *(int16_t*) taosArrayGet(pFunctionIdList, i);
if (f == FUNCTION_BLKINFO) {
return true;
}
}
return false;
}
bool isProjectionQuery(SArray* pFunctionIdList) {
int32_t num = (int32_t) taosArrayGetSize(pFunctionIdList);
for (int32_t i = 0; i < num; ++i) {
int32_t f = *(int16_t*) taosArrayGet(pFunctionIdList, i);
if (f == FUNCTION_TS_DUMMY) {
continue;
}
if (f != FUNCTION_PRJ && f != FUNCTION_TAGPRJ && f != FUNCTION_TAG &&
f != FUNCTION_TS && f != FUNCTION_ARITHM && f != FUNCTION_DIFF &&
f != FUNCTION_DERIVATIVE) {
return false;
}
}
return true;
}
bool isDiffDerivQuery(SArray* pFunctionIdList) {
int32_t num = (int32_t) taosArrayGetSize(pFunctionIdList);
for (int32_t i = 0; i < num; ++i) {
int32_t f = *(int16_t*) taosArrayGet(pFunctionIdList, i);
if (f == FUNCTION_TS_DUMMY) {
continue;
}
if (f == FUNCTION_DIFF || f == FUNCTION_DERIVATIVE) {
return true;
}
}
return false;
}
bool isPointInterpQuery(SArray* pFunctionIdList) {
int32_t num = (int32_t) taosArrayGetSize(pFunctionIdList);
for (int32_t i = 0; i < num; ++i) {
int32_t f = *(int16_t*) taosArrayGet(pFunctionIdList, i);
if (f == FUNCTION_TAG || f == FUNCTION_TS) {
continue;
}
if (f != FUNCTION_INTERP) {
return false;
}
}
return true;
}
bool isArithmeticQueryOnAggResult(SArray* pFunctionIdList) {
if (isProjectionQuery(pFunctionIdList)) {
return false;
}
assert(0);
// size_t numOfOutput = getNumOfFields(pQueryInfo);
// for(int32_t i = 0; i < numOfOutput; ++i) {
// SExprInfo* pExprInfo = tscFieldInfoGetInternalField(&pQueryInfo->fieldsInfo, i)->pExpr;
// if (pExprInfo->pExpr != NULL) {
// return true;
// }
// }
return false;
}
bool isGroupbyColumn(SArray* pFunctionIdList) {
// STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0);
// int32_t numOfCols = getNumOfColumns(pTableMetaInfo->pTableMeta);
//
// SGroupbyExpr* pGroupbyExpr = &pQueryInfo->groupbyExpr;
// for (int32_t k = 0; k < pGroupbyExpr->numOfGroupCols; ++k) {
// SColIndex* pIndex = taosArrayGet(pGroupbyExpr->columnInfo, k);
// if (!TSDB_COL_IS_TAG(pIndex->flag) && pIndex->colIndex < numOfCols) { // group by normal columns
// return true;
// }
// }
return false;
}
bool isTopBotQuery(SArray* pFunctionIdList) {
int32_t num = (int32_t) taosArrayGetSize(pFunctionIdList);
for (int32_t i = 0; i < num; ++i) {
int32_t f = *(int16_t*) taosArrayGet(pFunctionIdList, i);
if (f == FUNCTION_TS) {
continue;
}
if (f == FUNCTION_TOP || f == FUNCTION_BOTTOM) {
return true;
}
}
return false;
}
bool isTsCompQuery(SArray* pFunctionIdList) {
int32_t num = (int32_t) taosArrayGetSize(pFunctionIdList);
if (num != 1) {
return false;
}
int32_t f = *(int16_t*) taosArrayGet(pFunctionIdList, 0);
return f == FUNCTION_TS_COMP;
}
bool isTWAQuery(SArray* pFunctionIdList) {
int32_t num = (int32_t) taosArrayGetSize(pFunctionIdList);
for (int32_t i = 0; i < num; ++i) {
int32_t f = *(int16_t*) taosArrayGet(pFunctionIdList, i);
if (f == FUNCTION_TWA) {
return true;
}
}
return false;
}
bool isIrateQuery(SArray* pFunctionIdList) {
int32_t num = (int32_t) taosArrayGetSize(pFunctionIdList);
for (int32_t i = 0; i < num; ++i) {
int32_t f = *(int16_t*) taosArrayGet(pFunctionIdList, i);
if (f == FUNCTION_IRATE) {
return true;
}
}
return false;
}
bool isStabledev(SArray* pFunctionIdList) {
int32_t num = (int32_t) taosArrayGetSize(pFunctionIdList);
for (int32_t i = 0; i < num; ++i) {
int32_t f = *(int16_t*) taosArrayGet(pFunctionIdList, i);
if (f == FUNCTION_STDDEV_DST) {
return true;
}
}
return false;
}
bool needReverseScan(SArray* pFunctionIdList) {
assert(0);
int32_t num = (int32_t) taosArrayGetSize(pFunctionIdList);
for (int32_t i = 0; i < num; ++i) {
int32_t f = *(int16_t*) taosArrayGet(pFunctionIdList, i);
if (f == FUNCTION_TS || f == FUNCTION_TS_DUMMY || f == FUNCTION_TAG) {
continue;
}
// if ((f == FUNCTION_FIRST || f == FUNCTION_FIRST_DST) && pQueryInfo->order.order == TSDB_ORDER_DESC) {
// return true;
// }
if (f == FUNCTION_LAST || f == FUNCTION_LAST_DST) {
// the scan order to acquire the last result of the specified column
// int32_t order = (int32_t)pExpr->base.param[0].i64;
// if (order != pQueryInfo->order.order) {
// return true;
// }
}
}
return false;
}
bool isSimpleAggregateRv(SArray* pFunctionIdList) {
assert(0);
// if (pQueryInfo->interval.interval > 0 || pQueryInfo->sessionWindow.gap > 0) {
// return false;
// }
//
// if (tscIsDiffDerivQuery(pQueryInfo)) {
// return false;
// }
//
// size_t numOfExprs = getNumOfExprs(pQueryInfo);
// for (int32_t i = 0; i < numOfExprs; ++i) {
// SExprInfo* pExpr = getExprInfo(pQueryInfo, i);
// if (pExpr == NULL) {
// continue;
// }
//
// int32_t functionId = pExpr->base.functionId;
// if (functionId < 0) {
// SUdfInfo* pUdfInfo = taosArrayGet(pQueryInfo->pUdfInfo, -1 * functionId - 1);
// if (pUdfInfo->funcType == TSDB_UDF_TYPE_AGGREGATE) {
// return true;
// }
//
// continue;
// }
//
// if (functionId == FUNCTION_TS || functionId == FUNCTION_TS_DUMMY) {
// continue;
// }
//
// if ((!IS_MULTIOUTPUT(aAggs[functionId].status)) ||
// (functionId == FUNCTION_TOP || functionId == FUNCTION_BOTTOM || functionId == FUNCTION_TS_COMP)) {
// return true;
// }
// }
return false;
}
bool isBlockDistQuery(SArray* pFunctionIdList) {
int32_t num = (int32_t) taosArrayGetSize(pFunctionIdList);
int32_t f = *(int16_t*) taosArrayGet(pFunctionIdList, 0);
return (num == 1 && f == FUNCTION_BLKINFO);
}
bool isTwoStageSTableQuery(SArray* pFunctionIdList, int32_t tableIndex) {
// if (pQueryInfo == NULL) {
// return false;
// }
//
// STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, tableIndex);
// if (pTableMetaInfo == NULL) {
// return false;
// }
//
// if ((pQueryInfo->type & TSDB_QUERY_TYPE_FREE_RESOURCE) == TSDB_QUERY_TYPE_FREE_RESOURCE) {
// return false;
// }
//
// // for ordered projection query, iterate all qualified vnodes sequentially
// if (tscNonOrderedProjectionQueryOnSTable(pQueryInfo, tableIndex)) {
// return false;
// }
//
// if (!TSDB_QUERY_HAS_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_STABLE_SUBQUERY) && pQueryInfo->command == TSDB_SQL_SELECT) {
// return UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo);
// }
return false;
}
bool isProjectionQueryOnSTable(SArray* pFunctionIdList, int32_t tableIndex) {
// STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, tableIndex);
//
// /*
// * In following cases, return false for non ordered project query on super table
// * 1. failed to get tableMeta from server; 2. not a super table; 3. limitation is 0;
// * 4. show queries, instead of a select query
// */
// size_t numOfExprs = getNumOfExprs(pQueryInfo);
// if (pTableMetaInfo == NULL || !UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo) ||
// pQueryInfo->command == TSDB_SQL_RETRIEVE_EMPTY_RESULT || numOfExprs == 0) {
// return false;
// }
//
// for (int32_t i = 0; i < numOfExprs; ++i) {
// int32_t functionId = getExprInfo(pQueryInfo, i)->base.functionId;
//
// if (functionId < 0) {
// SUdfInfo* pUdfInfo = taosArrayGet(pQueryInfo->pUdfInfo, -1 * functionId - 1);
// if (pUdfInfo->funcType == TSDB_UDF_TYPE_AGGREGATE) {
// return false;
// }
//
// continue;
// }
//
// if (functionId != FUNCTION_PRJ &&
// functionId != FUNCTION_TAGPRJ &&
// functionId != FUNCTION_TAG &&
// functionId != FUNCTION_TS &&
// functionId != FUNCTION_ARITHM &&
// functionId != FUNCTION_TS_COMP &&
// functionId != FUNCTION_DIFF &&
// functionId != FUNCTION_DERIVATIVE &&
// functionId != FUNCTION_TS_DUMMY &&
// functionId != FUNCTION_TID_TAG) {
// return false;
// }
// }
return true;
}
bool hasTagValOutput(SArray* pFunctionIdList) {
// size_t numOfExprs = getNumOfExprs(pQueryInfo);
// SExprInfo* pExpr1 = getExprInfo(pQueryInfo, 0);
//
// if (numOfExprs == 1 && pExpr1->base.functionId == FUNCTION_TS_COMP) {
// return true;
// }
//
// for (int32_t i = 0; i < numOfExprs; ++i) {
// SExprInfo* pExpr = getExprInfo(pQueryInfo, i);
// if (pExpr == NULL) {
// continue;
// }
//
// // ts_comp column required the tag value for join filter
// if (TSDB_COL_IS_TAG(pExpr->base.colInfo.flag)) {
// return true;
// }
// }
return false;
}
bool timeWindowInterpoRequired(SArray* pFunctionIdList) {
int32_t num = (int32_t) taosArrayGetSize(pFunctionIdList);
for (int32_t i = 0; i < num; ++i) {
int32_t f = *(int16_t*) taosArrayGet(pFunctionIdList, i);
if (f == FUNCTION_TWA || f == FUNCTION_INTERP) {
return true;
}
}
return false;
}
//SQueryAttrInfo setQueryType(SArray* pFunctionIdList) {
// assert(pFunctionIdList != NULL);
//
//
//}
\ No newline at end of file
source/libs/function/src/thistogram.c 0 → 100644
浏览文件 @ 41290f9a
此差异已折叠。
source/libs/function/src/tpercentile.c 0 → 100644
浏览文件 @ 41290f9a
此差异已折叠。
source/libs/function/src/tscalarfunction.c 0 → 100644
浏览文件 @ 41290f9a
#include "tscalarfunction.h"
SScalarFunctionInfo scalarFunc[1] = {
{
},
};
source/libs/function/src/ttszip.c 0 → 100644
浏览文件 @ 41290f9a
此差异已折叠。
source/libs/parser/CMakeLists.txt
浏览文件 @ 41290f9a
......@@ -8,5 +8,7 @@ target_include_directories(
target_link_libraries(
parser
PRIVATE os util common catalog transport
PRIVATE os util common catalog function transport
)
ADD_SUBDIRECTORY(test)
\ No newline at end of file
source/libs/parser/inc/astGenerator.h
浏览文件 @ 41290f9a
......@@ -42,12 +42,6 @@ enum SQL_NODE_FROM_TYPE {
SQL_NODE_FROM_TABLELIST = 2,
};
//enum SQL_EXPR_FLAG {
// EXPR_FLAG_TS_ERROR = 1,
// EXPR_FLAG_NS_TIMESTAMP = 2,
// EXPR_FLAG_TIMESTAMP_VAR = 3,
//};
extern char tTokenTypeSwitcher[13];
#define toTSDBType(x) \
......@@ -252,6 +246,7 @@ typedef struct tSqlExpr {
// used in select clause. select <SArray> from xxx
typedef struct tSqlExprItem {
tSqlExpr *pNode; // The list of expressions
int32_t functionId;
char *aliasName; // alias name, null-terminated string
bool distinct;
} tSqlExprItem;
......@@ -267,7 +262,7 @@ SRelationInfo *addSubquery(SRelationInfo *pRelationInfo, SArray *pSub, SToken *p
// sql expr leaf node
tSqlExpr *tSqlExprCreateIdValue(SToken *pToken, int32_t optrType);
tSqlExpr *tSqlExprCreateFunction(SArray *pParam, SToken *pFuncToken, SToken *endToken, int32_t optType);
SArray * tAppendFuncName(SArray *pList, SToken *pToken);
SArray * tRecordFuncName(SArray *pList, SToken *pToken);
tSqlExpr *tSqlExprCreate(tSqlExpr *pLeft, tSqlExpr *pRight, int32_t optrType);
tSqlExpr *tSqlExprClone(tSqlExpr *pSrc);
......@@ -277,6 +272,7 @@ bool tSqlExprIsParentOfLeaf(tSqlExpr *pExpr);
void tSqlExprDestroy(tSqlExpr *pExpr);
SArray * tSqlExprListAppend(SArray *pList, tSqlExpr *pNode, SToken *pDistinct, SToken *pToken);
void tSqlExprListDestroy(SArray *pList);
void tSqlExprEvaluate(tSqlExpr* pExpr);
SSqlNode *tSetQuerySqlNode(SToken *pSelectToken, SArray *pSelNodeList, SRelationInfo *pFrom, tSqlExpr *pWhere,
SArray *pGroupby, SArray *pSortOrder, SIntervalVal *pInterval, SSessionWindowVal *ps,
......@@ -299,6 +295,8 @@ SArray *appendSelectClause(SArray *pList, void *pSubclause);
void setCreatedTableName(SSqlInfo *pInfo, SToken *pTableNameToken, SToken *pIfNotExists);
void* destroyCreateTableSql(SCreateTableSql* pCreate);
void setDropFuncInfo(SSqlInfo *pInfo, int32_t type, SToken* pToken);
void setCreateFuncInfo(SSqlInfo *pInfo, int32_t type, SToken *pName, SToken *pPath, SField *output, SToken* bufSize, int32_t funcType);
void SqlInfoDestroy(SSqlInfo *pInfo);
......
source/libs/parser/inc/parserInt.h
浏览文件 @ 41290f9a
......@@ -35,6 +35,21 @@ typedef struct SInsertStmtInfo {
char *sql; // current sql statement position
} SInsertStmtInfo;
typedef struct SInternalField {
TAOS_FIELD field;
bool visible;
SExprInfo *pExpr;
} SInternalField;
typedef struct SMsgBuf {
int32_t len;
char *buf;
} SMsgBuf;
void clearTableMetaInfo(STableMetaInfo* pTableMetaInfo);
void clearAllTableMetaInfo(SQueryStmtInfo* pQueryInfo, bool removeMeta, uint64_t id);
/**
* Validate the sql info, according to the corresponding metadata info from catalog.
* @param pCatalog
......@@ -44,15 +59,33 @@ typedef struct SInsertStmtInfo {
* @param msg
* @return
*/
int32_t qParserValidateSqlNode(struct SCatalog* pCatalog, SSqlInfo* pSqlInfo, SQueryStmtInfo* pQueryInfo, int64_t id, char* msg);
int32_t qParserValidateSqlNode(struct SCatalog* pCatalog, SSqlInfo* pSqlInfo, SQueryStmtInfo* pQueryInfo, int64_t id, char* msg, int32_t msgLen);
/**
*
* @param pSqlNode
* Evaluate the numeric and timestamp arithmetic expression in the WHERE clause.
* @param pNode
* @param tsPrecision
* @param msg
* @param msgBufLen
* @return
*/
int32_t evaluateSqlNode(SSqlNode* pNode, int32_t tsPrecision, SMsgBuf* pMsgBuf);
int32_t validateSqlNode(SSqlNode* pSqlNode, SQueryStmtInfo* pQueryInfo, SMsgBuf* pMsgBuf);
void initQueryInfo(SQueryStmtInfo* pQueryInfo);
int32_t checkForInvalidExpr(SQueryStmtInfo* pQueryInfo, SMsgBuf* pMsgBuf);
/**
* Extract request meta info from the sql statement
* @param pSqlInfo
* @param pMetaInfo
* @param msg
* @param msgBufLen
* @return
*/
int32_t qParserExtractRequestedMetaInfo(const struct SSqlNode* pSqlNode, SMetaReq* pMetaInfo);
int32_t qParserExtractRequestedMetaInfo(const SSqlInfo* pSqlInfo, SMetaReq* pMetaInfo, char* msg, int32_t msgBufLen);
#ifdef __cplusplus
}
......
source/libs/parser/inc/parserUtil.h
浏览文件 @ 41290f9a
......@@ -20,6 +20,47 @@
extern "C" {
#endif
#include "os.h"
#include "ttoken.h"
#include "parserInt.h"
#define UTIL_TABLE_IS_SUPER_TABLE(metaInfo) \
(((metaInfo)->pTableMeta != NULL) && ((metaInfo)->pTableMeta->tableType == TSDB_SUPER_TABLE))
#define UTIL_TABLE_IS_CHILD_TABLE(metaInfo) \
(((metaInfo)->pTableMeta != NULL) && ((metaInfo)->pTableMeta->tableType == TSDB_CHILD_TABLE))
#define UTIL_TABLE_IS_NORMAL_TABLE(metaInfo) \
(!(UTIL_TABLE_IS_SUPER_TABLE(metaInfo) || UTIL_TABLE_IS_CHILD_TABLE(metaInfo)))
#define UTIL_TABLE_IS_TMP_TABLE(metaInfo) \
(((metaInfo)->pTableMeta != NULL) && ((metaInfo)->pTableMeta->tableType == TSDB_TEMP_TABLE))
TAOS_FIELD createField(const SSchema* pSchema);
SSchema createSchema(uint8_t type, int16_t bytes, int16_t colId, const char* name);
SInternalField* insertFieldInfo(SFieldInfo* pFieldInfo, int32_t index, SSchema* field);
int32_t getNumOfFields(SFieldInfo* pFieldInfo);
SInternalField* getInternalField(SFieldInfo* pFieldInfo, int32_t index);
int32_t parserValidateIdToken(SToken* pToken);
int32_t buildInvalidOperationMsg(SMsgBuf* pMsgBuf, const char* msg);
int32_t buildSyntaxErrMsg(char* dst, int32_t dstBufLen, const char* additionalInfo, const char* sourceStr);
int32_t createProjectionExpr(SQueryStmtInfo* pQueryInfo, STableMetaInfo* pTableMetaInfo, SExprInfo*** pExpr, int32_t* num);
STableMetaInfo* addEmptyMetaInfo(SQueryStmtInfo* pQueryInfo);
void columnListCopyAll(SArray* dst, const SArray* src);
void columnListDestroy(SArray* pColumnList);
SColumn* columnListInsert(SArray* pColumnList, int32_t columnIndex, uint64_t uid, SSchema* pSchema);
SColumn* insertPrimaryTsColumn(SArray* pColumnList, uint64_t tableUid);
void cleanupTagCond(STagCond* pTagCond);
void cleanupColumnCond(SArray** pCond);
uint32_t convertRelationalOperator(SToken *pToken);
#ifdef __cplusplus
}
......
source/libs/parser/inc/queryInfoUtil.h 0 → 100644
浏览文件 @ 41290f9a
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef TDENGINE_QUERYINFOUTIL_H
#define TDENGINE_QUERYINFOUTIL_H
#ifdef __cplusplus
extern "C" {
#endif
#include "parserInt.h"
SSchema* getTbnameColumnSchema();
int32_t getNumOfColumns(const STableMeta* pTableMeta);
int32_t getNumOfTags(const STableMeta* pTableMeta);
SSchema *getTableColumnSchema(const STableMeta *pTableMeta);
SSchema *getTableTagSchema(const STableMeta* pTableMeta);
SSchema *getOneColumnSchema(const STableMeta* pTableMeta, int32_t colIndex);
size_t getNumOfExprs(SQueryStmtInfo* pQueryInfo);
//SExprInfo* createExprInfo(STableMetaInfo* pTableMetaInfo, int16_t functionId, SColumnIndex* pColIndex, struct tExprNode* pParamExpr, SSchema* pResSchema, int16_t interSize);
SExprInfo* createBinaryExprInfo(struct tExprNode* pNode, SSchema* pResSchema);
void destroyExprInfoList();
void addExprInfo(SQueryStmtInfo* pQueryInfo, int32_t index, SExprInfo* pExprInfo);
void updateExprInfo(SExprInfo* pExprInfo, int16_t functionId, int32_t colId, int16_t srcColumnIndex, int16_t resType, int16_t resSize);
SExprInfo* getExprInfo(SQueryStmtInfo* pQueryInfo, int32_t index);
int32_t copyAllExprInfo(SArray* dst, const SArray* src, bool deepcopy);
void addExprInfoParam(SSqlExpr* pExpr, char* argument, int32_t type, int32_t bytes);
void cleanupFieldInfo(SFieldInfo* pFieldInfo);
STableComInfo getTableInfo(const STableMeta* pTableMeta);
#ifdef __cplusplus
}
#endif
#endif // TDENGINE_QUERYINFOUTIL_H
source/libs/parser/inc/sql.y
浏览文件 @ 41290f9a
......@@ -325,7 +325,7 @@ alter_db_optr(Y) ::= alter_db_optr(Z) cachelast(X). { Y = Z; Y.cachelast = str
alter_topic_optr(Y) ::= alter_db_optr(Z). { Y = Z; Y.dbType = TSDB_DB_TYPE_TOPIC; }
alter_topic_optr(Y) ::= alter_topic_optr(Z) partitions(X). { Y = Z; Y.partitions = strtol(X.z, NULL, 10); }
%type typename {TAOS_FIELD}
%type typename {SField}
typename(A) ::= ids(X). {
X.type = 0;
tSetColumnType (&A, &X);
......@@ -425,11 +425,11 @@ create_table_args(A) ::= ifnotexists(U) ids(V) cpxName(Z) AS select(S). {
setCreatedTableName(pInfo, &V, &U);
}
%type column{TAOS_FIELD}
%type column{SField}
%type columnlist{SArray*}
%destructor columnlist {taosArrayDestroy($$);}
columnlist(A) ::= columnlist(X) COMMA column(Y). {taosArrayPush(X, &Y); A = X; }
columnlist(A) ::= column(X). {A = taosArrayInit(4, sizeof(TAOS_FIELD)); taosArrayPush(A, &X);}
columnlist(A) ::= column(X). {A = taosArrayInit(4, sizeof(SField)); taosArrayPush(A, &X);}
// The information used for a column is the name and type of column:
// tinyint smallint int bigint float double bool timestamp binary(x) nchar(x)
......@@ -601,7 +601,7 @@ fill_opt(N) ::= FILL LP ID(Y) COMMA tagitemlist(X) RP. {
toTSDBType(Y.type);
taosVariantCreate(&A, Y.z, Y.n, Y.type);
tVariantListInsert(X, &A, -1, 0);
tListItemInsert(X, &A, -1, 0);
N = X;
}
......@@ -719,10 +719,10 @@ expr(A) ::= BOOL(X). { A = tSqlExprCreateIdValue(&X, TK_BOOL);}
expr(A) ::= NULL(X). { A = tSqlExprCreateIdValue(&X, TK_NULL);}
// ordinary functions: min(x), max(x), top(k, 20)
expr(A) ::= ID(X) LP exprlist(Y) RP(E). { tAppendFuncName(pInfo->funcs, &X); A = tSqlExprCreateFunction(Y, &X, &E, X.type); }
expr(A) ::= ID(X) LP exprlist(Y) RP(E). { tRecordFuncName(pInfo->funcs, &X); A = tSqlExprCreateFunction(Y, &X, &E, X.type); }
// for parsing sql functions with wildcard for parameters. e.g., count(*)/first(*)/last(*) operation
expr(A) ::= ID(X) LP STAR RP(Y). { tAppendFuncName(pInfo->funcs, &X); A = tSqlExprCreateFunction(NULL, &X, &Y, X.type); }
expr(A) ::= ID(X) LP STAR RP(Y). { tRecordFuncName(pInfo->funcs, &X); A = tSqlExprCreateFunction(NULL, &X, &Y, X.type); }
// is (not) null expression
expr(A) ::= expr(X) IS NULL. {A = tSqlExprCreate(X, NULL, TK_ISNULL);}
......
source/libs/parser/src/parser.c
浏览文件 @ 41290f9a
此差异已折叠。
source/libs/parser/src/parserUtil.c 0 → 100644
浏览文件 @ 41290f9a
此差异已折叠。
source/libs/parser/src/queryInfoUtil.c 0 → 100644
浏览文件 @ 41290f9a
此差异已折叠。
source/libs/parser/src/sql.c
浏览文件 @ 41290f9a
此差异已折叠。
source/libs/parser/src/ttokenizer.c
浏览文件 @ 41290f9a
......@@ -411,6 +411,7 @@ uint32_t tGetToken(char* z, uint32_t* tokenId) {
*tokenId = TK_QUESTION;
return 1;
}
case '`':
case '\'':
case '"': {
int delim = z[0];
......@@ -434,7 +435,7 @@ uint32_t tGetToken(char* z, uint32_t* tokenId) {
if (z[i]) i++;
if (strEnd) {
*tokenId = TK_STRING;
*tokenId = (delim == '`')? TK_ID:TK_STRING;
return i;
}
......
source/libs/parser/test/CMakeLists.txt 0 → 100644
浏览文件 @ 41290f9a
MESSAGE(STATUS "build parser unit test")
# GoogleTest requires at least C++11
SET(CMAKE_CXX_STANDARD 11)
AUX_SOURCE_DIRECTORY(${CMAKE_CURRENT_SOURCE_DIR} SOURCE_LIST)
ADD_EXECUTABLE(astTest ${SOURCE_LIST})
TARGET_LINK_LIBRARIES(
astTest
PUBLIC os util common parser catalog transport gtest
)
TARGET_INCLUDE_DIRECTORIES(
astTest
PUBLIC "${CMAKE_SOURCE_DIR}/include/libs/parser/"
PRIVATE "${CMAKE_SOURCE_DIR}/source/libs/parser/inc"
)
source/libs/parser/test/tokenizerTest.cpp 0 → 100644
浏览文件 @ 41290f9a
此差异已折叠。
source/libs/planner/CMakeLists.txt
浏览文件 @ 41290f9a
......@@ -8,5 +8,5 @@ target_include_directories(
target_link_libraries(
planner
PRIVATE os util common catalog parser transport
PRIVATE os util common catalog parser transport function
)
\ No newline at end of file
source/libs/planner/inc/plannerInt.h
浏览文件 @ 41290f9a
......@@ -33,26 +33,25 @@ typedef struct SQueryNodeBasicInfo {
typedef struct SQueryTableInfo {
char *tableName;
uint64_t uid;
int32_t tid;
} SQueryTableInfo;
typedef struct SQueryNode {
typedef struct SQueryPlanNode {
SQueryNodeBasicInfo info;
SQueryTableInfo tableInfo;
SSchema *pSchema; // the schema of the input SSDatablock
int32_t numOfCols; // number of input columns
struct SExprInfo *pExpr; // the query functions or sql aggregations
SArray *pExpr; // the query functions or sql aggregations
int32_t numOfOutput; // number of result columns, which is also the number of pExprs
void *pExtInfo; // additional information
// previous operator to generated result for current node to process
// in case of join, multiple prev nodes exist.
SArray *pPrevNodes; // upstream nodes
struct SQueryNode *nextNode;
} SQueryNode;
struct SQueryPlanNode *nextNode;
} SQueryPlanNode;
typedef struct SQueryPhyNode {
typedef struct SQueryDistPlanNode {
} SQueryPhyNode;
} SQueryDistPlanNode;
#ifdef __cplusplus
}
......
source/libs/planner/src/planner.c
浏览文件 @ 41290f9a
此差异已折叠。
source/server/vnode/src/vnodeReadMsg.c
浏览文件 @ 41290f9a
......@@ -221,16 +221,16 @@ int32_t vnodeProcessConsumeMsg(SVnode *pVnode, SReadMsg *pRead) {
tmqMsgHead msgHead = pConsumeMsg->head;
//extract head
STQ *pTq = pVnode->pTQ;
tqGroupHandle *pHandle = tqFindGHandleBycId(pTq, msgHead.clientId);
/*tqBufferHandle *pHandle = tqGetHandle(pTq, msgHead.clientId);*/
//return msg if offset not moved
if(pConsumeMsg->commitOffset == pHandle->consumeOffset) {
/*if(pConsumeMsg->commitOffset == pHandle->consumeOffset) {*/
//return msg
return 0;
}
/*return 0;*/
/*}*/
//or move offset
tqMoveOffsetToNext(pHandle);
/*tqMoveOffsetToNext(pHandle);*/
//fetch or register context
tqFetchMsg(pHandle, pRead);
/*tqFetchMsg(pHandle, pRead);*/
//judge mode, tail read or catch up read
/*int64_t lastVer = walLastVer(pVnode->wal);*/
//launch new query
......
source/server/vnode/tq/inc/tqInt.h
浏览文件 @ 41290f9a
此差异已折叠。
source/server/vnode/tq/src/tq.c
浏览文件 @ 41290f9a
此差异已折叠。
source/util/src/tpagedfile.c 0 → 100644
浏览文件 @ 41290f9a
此差异已折叠。
source/util/src/tutil.c
浏览文件 @ 41290f9a
此差异已折叠。
src/client/src/tscUtil.c
浏览文件 @ 41290f9a
此差异已折叠。
src/query/tests/astTest.cpp
浏览文件 @ 41290f9a
此差异已折叠。
Markdown is supported
0% 或 .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑！
想要评论请 注册 或