Skip to content

T
TDengine

慢慢CG / TDengine
与 Fork 源项目一致

Fork自 taosdata / TDengine

通知 1
Star 0
Fork 0
T
TDengine
提交 438174a6 编写于 作者: T Tao Liu
浏览文件
操作

merge from master

上级 d597bc97 96caf3d9
显示空白变更内容
内联 并排
Showing with 7076 addition and 149 deletion
CMakeLists.txt
浏览文件 @ 438174a6
......@@ -53,6 +53,7 @@ IF (NOT DEFINED TD_CLUSTER)
SET(TD_MIPS_32 FALSE)
SET(TD_DARWIN_64 FALSE)
SET(TD_WINDOWS_64 FALSE)
SET(TD_WINDOWS_32 FALSE)
SET(TD_PAGMODE_LITE FALSE)
IF (${PAGMODE} MATCHES "lite")
......@@ -127,6 +128,11 @@ IF (NOT DEFINED TD_CLUSTER)
SET(TD_OS_DIR ${TD_COMMUNITY_DIR}/src/os/windows)
ADD_DEFINITIONS(-D_M_X64)
MESSAGE(STATUS "The current platform is Windows 64-bit")
ELSEIF(${CMAKE_SIZEOF_VOID_P} MATCHES 4)
SET(TD_WINDOWS_32 TRUE)
SET(TD_OS_DIR ${TD_COMMUNITY_DIR}/src/os/windows)
#ADD_DEFINITIONS(-D_M_X64)
MESSAGE(STATUS "The current platform is Windows 32-bit")
ELSE ()
MESSAGE(FATAL_ERROR "The current platform is Windows 32-bit, not supported yet")
EXIT ()
......@@ -194,7 +200,7 @@ IF (NOT DEFINED TD_CLUSTER)
link_library(/usr/lib/libargp.a)
ADD_DEFINITIONS(-D_ALPINE)
ENDIF ()
ELSEIF (TD_WINDOWS_64)
ELSEIF (TD_WINDOWS_64 OR TD_WINDOWS_32)
SET(CMAKE_GENERATOR "NMake Makefiles" CACHE INTERNAL "" FORCE)
IF (NOT TD_GODLL)
SET(COMMON_FLAGS "/nologo /WX- /Oi /Oy- /Gm- /EHsc /MT /GS /Gy /fp:precise /Zc:wchar_t /Zc:forScope /Gd /errorReport:prompt /analyze-")
......@@ -235,7 +241,7 @@ IF (NOT DEFINED TD_CLUSTER)
ELSEIF (${CMAKE_BUILD_TYPE} MATCHES "Release")
MESSAGE(STATUS "Build Release Version")
ELSE ()
IF (TD_WINDOWS_64)
IF (TD_WINDOWS_64 OR TD_WINDOWS_32)
SET(CMAKE_BUILD_TYPE "Release")
MESSAGE(STATUS "Build Release Version in Windows as default")
ELSE ()
......@@ -268,7 +274,7 @@ IF (NOT DEFINED TD_CLUSTER)
INSTALL(CODE "MESSAGE(\"make install script: ${TD_MAKE_INSTALL_SH}\")")
INSTALL(CODE "execute_process(COMMAND chmod 777 ${TD_MAKE_INSTALL_SH})")
INSTALL(CODE "execute_process(COMMAND ${TD_MAKE_INSTALL_SH} ${TD_COMMUNITY_DIR} ${PROJECT_BINARY_DIR})")
ELSEIF (TD_WINDOWS_64)
ELSEIF (TD_WINDOWS_64 OR TD_WINDOWS_32)
SET(CMAKE_INSTALL_PREFIX C:/TDengine)
IF (NOT TD_GODLL)
INSTALL(DIRECTORY ${TD_COMMUNITY_DIR}/src/connector/go DESTINATION connector)
......@@ -277,6 +283,7 @@ IF (NOT DEFINED TD_CLUSTER)
INSTALL(DIRECTORY ${TD_COMMUNITY_DIR}/tests/examples DESTINATION .)
INSTALL(DIRECTORY ${TD_COMMUNITY_DIR}/packaging/cfg DESTINATION .)
INSTALL(FILES ${TD_COMMUNITY_DIR}/src/inc/taos.h DESTINATION include)
INSTALL(FILES ${TD_COMMUNITY_DIR}/src/inc/taoserror.h DESTINATION include)
INSTALL(FILES ${LIBRARY_OUTPUT_PATH}/taos.lib DESTINATION driver)
INSTALL(FILES ${LIBRARY_OUTPUT_PATH}/taos.exp DESTINATION driver)
INSTALL(FILES ${LIBRARY_OUTPUT_PATH}/taos.dll DESTINATION driver)
......
bailongma/Dockerfile 0 → 100644
浏览文件 @ 438174a6
## Builder image
#FROM tdengine/tdengine:dev as builder1
FROM golang:latest as builder1
WORKDIR /root
#COPY --from=builder1 /usr/include/taos.h /usr/include/
#COPY --from=builder1 /usr/lib/libtaos.so /usr/lib/libtaos.so
#COPY --from=builder1 /usr/include/taos.h /usr/include/
#COPY --from=builder1 /usr/lib/libtaos.so.1 /usr/lib/
COPY blm_telegraf /root
COPY blm_prometheus /root
COPY TDengine-server-1.6.5.9-Linux-x64.tar.gz /root
COPY go.mod /root
COPY go.sum /root
RUN tar -zxf TDengine-server-1.6.5.9-Linux-x64.tar.gz .
WORKDIR /root/TDengine-server
RUN sh install.sh
#RUN mkdir /usr/lib/ld
WORKDIR /root/blm_telegraf
RUN go build
WORKDIR /root/blm_prometheus
RUN go build
#RUN ln -s /usr/lib/libtaos.so.1 /usr/lib/libtaos.so
FROM centos:7
WORKDIR /root
COPY --from=builder /usr/include/taos.h /usr/include/
COPY --from=builder /usr/lib/libtaos.so /usr/lib/libtaos.so
COPY --from=builder /usr/include/taos.h /usr/include/
COPY --from=builder /usr/lib/libtaos.so.1 /usr/lib/
COPY --from=builder /root/blm_telegraf/blm_telegraf /root/
COPY --from=builder /root/blm_prometheus/blm_prometheus /root/
#RUN git config --global http.sslVerify false
#RUN git config --global http.postbuffer 524288000
#RUN go get -v -u -insecure github.com/taosdata/TDengine/src/connector/go/src/taosSql
#RUN go get -v -u -insecure github.com/gogo/protobuf/proto
#RUN go get -v -u -insecure github.com/golang/snappy
#RUN go get -v -u -insecure github.com/prometheus/common/model
#RUN go get -v -u -insecure github.com/prometheus/prometheus/prompb
#RUN go get github.com/taosdata/driver-go/taosSql
bailongma/Dockerfile.prm 0 → 100644
浏览文件 @ 438174a6
## Builder image
FROM tdengine/tdengine:1.6.5.9 as builder1
FROM golang:latest as builder
WORKDIR /root
COPY --from=builder1 /usr/include/taos.h /usr/include/
COPY --from=builder1 /usr/lib/libtaos.so /usr/lib/libtaos.so
COPY --from=builder1 /usr/include/taos.h /usr/include/
COPY --from=builder1 /usr/lib/libtaos.so.1 /usr/lib/
COPY blm_prometheus /root/blm_prometheus
COPY go.mod /root
COPY go.sum /root
WORKDIR /root/blm_prometheus
RUN go build
FROM centos:7
WORKDIR /root
COPY --from=builder1 /usr/include/taos.h /usr/include/
COPY --from=builder1 /usr/lib/libtaos.so.1 /usr/lib/
RUN ln -s /usr/lib/libtaos.so.1 /usr/lib/libtaos.so
COPY --from=builder /root/blm_prometheus /root
ENV LD_LIBRARY_PATH="$LD_LIBRARY_PATH:/usr/lib"
ENV LANG=en_US.UTF-8
ENV LANGUAGE=en_US:en
ENV LC_ALL=en_US.UTF-8
#RUN git config --global http.sslVerify false
#RUN git config --global http.postbuffer 524288000
VOLUME [ "/var/lib/taos", "/var/log/taos","/etc/taos/" ]
ENTRYPOINT [ "/root/blm_prometheus" ]
bailongma/Dockerfile.tlg 0 → 100644
浏览文件 @ 438174a6
## Builder image
FROM tdengine/tdengine:1.6.5.9 as builder1
FROM golang:latest as builder
WORKDIR /root
COPY --from=builder1 /usr/include/taos.h /usr/include/
COPY --from=builder1 /usr/lib/libtaos.so /usr/lib/libtaos.so
COPY --from=builder1 /usr/include/taos.h /usr/include/
COPY --from=builder1 /usr/lib/libtaos.so.1 /usr/lib/
COPY blm_telegraf /root/blm_telegraf
COPY go.mod /root
COPY go.sum /root
WORKDIR /root/blm_telegraf
RUN ls
RUN go build
FROM centos:7
WORKDIR /root
COPY --from=builder1 /usr/include/taos.h /usr/include/
COPY --from=builder1 /usr/lib/libtaos.so.1 /usr/lib/
RUN ln -s /usr/lib/libtaos.so.1 /usr/lib/libtaos.so
COPY --from=builder /root/blm_telegraf /root
ENV LD_LIBRARY_PATH="$LD_LIBRARY_PATH:/usr/lib"
ENV LANG=en_US.UTF-8
ENV LANGUAGE=en_US:en
ENV LC_ALL=en_US.UTF-8
VOLUME [ "/var/lib/taos", "/var/log/taos","/etc/taos/" ]
ENTRYPOINT [ "/root/blm_telegraf"]
bailongma/blm_prometheus/Dockerfile 0 → 100644
浏览文件 @ 438174a6
## Builder image
FROM tdengine/tdengine:dev as builder1
FROM tdengine/bailongma:dev as builder
COPY server.go /root/blm_prometheus/
WORKDIR /root/blm_prometheus
# build enterprise version
RUN go build
# # build community version
# RUN cmake .. -DVERSION=lite && cmake --build .
## Target image
FROM centos:7
WORKDIR /root/
# COPY --from=builder /root/build/build/lib/libtaos.so /usr/lib/libtaos.so.1
# RUN ln -s /usr/lib/libtaos.so.1 /usr/lib/libtaos.so
COPY --from=builder1 /usr/include/taos.h /usr/include/
COPY --from=builder1 /usr/lib/libtaos.so.1 /usr/lib/libtaos.so.1
RUN ln -s /usr/lib/libtaos.so.1 /usr/lib/libtaos.so
COPY --from=builder /root/blm_prometheus/blm_prometheus .
#COPY community/packaging/cfg/taos.cfg /etc/taos/
ENV LD_LIBRARY_PATH="$LD_LIBRARY_PATH:/usr/lib"
ENV LANG=en_US.UTF-8
ENV LANGUAGE=en_US:en
ENV LC_ALL=en_US.UTF-8
EXPOSE 10203
VOLUME [ "/var/lib/taos", "/var/log/taos","/etc/taos" ]
ENTRYPOINT [ "/root/blm_prometheus" ]
\ No newline at end of file
bailongma/blm_prometheus/README.md 0 → 100644
浏览文件 @ 438174a6
# Prometheus Remote Write Adapter for TDengine
This is an adapter to support Prometheus remote write into TDengine.
## Prerequisite
before running the software, you need to install the `golang-1.10` or later version in your environment and install [TDengine][] so the program can use the lib of TDengine.
To use it:
```
go build
```
During the go build process, there maybe some errors arised because of lacking some needed packages. You can use `go get` the package to solve it
```
go get github.com/gogo/protobuf/proto
go get github.com/golang/snappy
go get github.com/prometheus/common/model
go get github.com/taosdata/TDengine/src/connector/go/src/taosSql
go get github.com/prometheus/prometheus/prompb
```
After successful build, there will be a blm_prometheus in the same directory.
## Running in background
Using following command to run the program in background
```
nohup ./blm_prometheus --tdengine-ip 112.102.3.69 --batch-size 80 --http-workers 2 --sql-workers 2 --dbname prometheus --port 1234 > /dev/null 2>&1 &
```
There are several options can be set:
```sh
--tdengine-ip
set the IP of TDengine for example "192.168.0.1"
--tdengine-name
set the domain name of TDengine, then blm-prometheus can lookup the ip address of TDengine.
--tdengine-api-port
set the restful API port of TDengine. blm-prometheus will query the table schema info from TDengine to keep meta info synchronized. Default is 6020
--batch-size
set the size of how many records in one SQL cmd line writing into TDengine. There is a limitation that TDengine could only accept SQL line small than 64000 bytes, so usually the batch size should not exceed 200. Default is 100.
--http-workers
set the number of workers who process the HTTP request. default is 10
--sql-workers
set the number of workers who process the database request. default is 10
--dbname
set the database name in TDengine, if not exists, a database will be created after this dbname. default is "prometheus".
--dbuser
set the user name that have the right to access the TDengine. default is "root"
--dbpassword
set the password of dbuser. default is "taosdata"
--port
set the port that prometheus configuration remote_write. as showed above, in the prometheus.yaml. default is 10203
```
## Start prometheus
Add the following to your prometheus's configuration `prometheus.yml` :
```yaml
remote_write:
- url: "http://localhost:1234/receive"
```
Then start Prometheus:
```
prometheus
```
Then you can check the TDengine if there is super table and tables.
## Check the TDengine tables and datas
Use the taos client shell to query the result.
```
Welcome to the TDengine shell from linux, client version:1.6.4.0 server version:1.6.4.0
Copyright (c) 2017 by TAOS Data, Inc. All rights reserved.
This is the trial version and will expire at 2019-12-11 14:25:31.
taos> use prometheus;
Database changed.
taos> show stables;
name | created_time |columns| tags | tables |
====================================================================================================================
prometheus_sd_kubernetes_cache_watch_events_sum | 19-11-15 17:45:07.594| 2| 3| 1|
prometheus_sd_kubernetes_cache_watch_events_count | 19-11-15 17:45:07.596| 2| 3| 1|
prometheus_sd_kubernetes_cache_watches_total | 19-11-15 17:45:07.598| 2| 3| 1|
prometheus_sd_kubernetes_events_total | 19-11-15 17:45:07.600| 2| 5| 15|
prometheus_target_scrape_pool_reloads_total | 19-11-15 17:45:07.672| 2| 3| 1|
prometheus_sd_received_updates_total | 19-11-15 17:45:07.674| 2| 4| 1|
prometheus_target_scrape_pool_reloads_failed_total | 19-11-15 17:45:07.730| 2| 3| 1|
prometheus_sd_updates_total | 19-11-15 17:45:07.732| 2| 4| 1|
prometheus_target_scrape_pool_sync_total | 19-11-15 17:45:07.734| 2| 4| 1|
......
go_memstats_gc_cpu_fraction | 19-11-15 17:45:06.599| 2| 3| 1|
Query OK, 211 row(s) in set (0.004891s)
taos> select * from prometheus_sd_updates_total;
......
19-11-16 14:24:00.271| 1.000000000|localhost:9090 |prometheus |codelab-monitor |scrape |
19-11-16 14:24:05.271| 1.000000000|localhost:9090 |prometheus |codelab-monitor |scrape |
Query OK, 3029 row(s) in set (0.060828s)
```
## Support Kubernates liveness probe
The blm_prometheus support the liveness probe.
When the service is running, GET the url`http://ip:port/health` will return 200 OK response which means the service is running healthy. If no response, means the service is dead and need to restart it.
## Limitations
The TDengine limits the length of super table name, so if the name of prometheus metric exceeds 60 byte, it will be truncated to first 60 bytes. And the length of label name is limited within 50 byte.
[TDengine]:https://www.github.com/Taosdata/TDengine
\ No newline at end of file
bailongma/blm_prometheus/server.go 0 → 100644
浏览文件 @ 438174a6
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package main
import (
"bufio"
"container/list"
"crypto/md5"
"database/sql"
"encoding/json"
"flag"
"fmt"
"io"
"io/ioutil"
"log"
"net"
"net/http"
_ "net/http/pprof"
"os"
"path/filepath"
"sort"
"strconv"
"strings"
"sync"
"time"
"github.com/gogo/protobuf/proto"
"github.com/golang/snappy"
"github.com/prometheus/common/model"
_ "github.com/taosdata/driver-go/taosSql"
"github.com/prometheus/prometheus/prompb"
)
type Bailongma struct {
}
type nametag struct {
tagmap map[string]string
taglist *list.List
annotlen int
}
var (
daemonIP string
daemonName string
httpworkers int
sqlworkers int
batchSize int
buffersize int
dbname string
dbuser string
dbpassword string
rwport string
apiport string
debugprt int
taglen int
taglimit int = 1024
tagnumlimit int
tablepervnode int
)
// Global vars
var (
bufPool sync.Pool
batchChans []chan string //multi table one chan
nodeChans []chan prompb.WriteRequest //multi node one chan
inputDone chan struct{}
workersGroup sync.WaitGroup
reportTags [][2]string
reportHostname string
taosDriverName string = "taosSql"
IsSTableCreated sync.Map
IsTableCreated sync.Map
tagstr string
blmLog *log.Logger
tdurl string
logNameDefault string = "/var/log/taos/blm_prometheus.log"
)
var scratchBufPool = &sync.Pool{
New: func() interface{} {
return make([]byte, 0, 1024)
},
}
type FieldDescriptiion struct {
fname string
ftype string
flength int
fnote string
}
type tableStruct struct {
status string
head []string
data []FieldDescriptiion
rows int64
}
// Parse args:
func init() {
flag.StringVar(&daemonIP, "tdengine-ip", "127.0.0.1", "TDengine host IP.")
flag.StringVar(&daemonName, "tdengine-name", "", "TDengine host Name. in K8S, could be used to lookup TDengine's IP")
flag.StringVar(&apiport, "tdengine-api-port", "6020", "TDengine restful API port")
flag.IntVar(&batchSize, "batch-size", 100, "Batch size (input items).")
flag.IntVar(&httpworkers, "http-workers", 1, "Number of parallel http requests handler .")
flag.IntVar(&sqlworkers, "sql-workers", 1, "Number of parallel sql handler.")
flag.StringVar(&dbname, "dbname", "prometheus", "Database name where to store metrics")
flag.StringVar(&dbuser, "dbuser", "root", "User for host to send result metrics")
flag.StringVar(&dbpassword, "dbpassword", "taosdata", "User password for Host to send result metrics")
flag.StringVar(&rwport, "port", "10203", "remote write port")
flag.IntVar(&debugprt, "debugprt", 0, "if 0 not print, if 1 print the sql")
flag.IntVar(&taglen, "tag-length", 50, "the max length of tag string,default is 30")
flag.IntVar(&buffersize, "buffersize", 100, "the buffer size of metrics received")
flag.IntVar(&tagnumlimit, "tag-num", 16, "the number of tags in a super table, default is 8")
flag.IntVar(&tablepervnode, "table-num", 10000, "the number of tables per TDengine Vnode can create, default 10000")
flag.Parse()
if daemonName != "" {
s, _ := net.LookupIP(daemonName)
daemonIP = fmt.Sprintf("%s", s[0])
fmt.Println(daemonIP)
fmt.Println(s[0])
daemonIP = daemonIP + ":0"
tdurl = daemonName
} else {
tdurl = daemonIP
daemonIP = daemonIP + ":0"
}
tagstr = fmt.Sprintf(" binary(%d)", taglen)
logFile, err := os.OpenFile(logNameDefault, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644)
if err != nil {
fmt.Println(err)
os.Exit(1)
}
blmLog = log.New(logFile, "", log.LstdFlags)
blmLog.SetPrefix("BLM_PRM")
blmLog.SetFlags(log.LstdFlags | log.Lshortfile)
blmLog.Printf("host: ip")
blmLog.Printf(daemonIP)
blmLog.Printf(" port: ")
blmLog.Printf(rwport)
blmLog.Printf(" database: ")
blmLog.Println(dbname)
}
func main() {
for i := 0; i < httpworkers; i++ {
nodeChans = append(nodeChans, make(chan prompb.WriteRequest, buffersize))
}
createDatabase(dbname)
for i := 0; i < httpworkers; i++ {
workersGroup.Add(1)
go NodeProcess(i)
}
for i := 0; i < sqlworkers; i++ {
batchChans = append(batchChans, make(chan string, batchSize))
}
for i := 0; i < sqlworkers; i++ {
workersGroup.Add(1)
go processBatches(i)
}
http.HandleFunc("/receive", func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusAccepted)
addr := strings.Split(r.RemoteAddr, ":")
idx := TAOShashID([]byte(addr[0]))
compressed, err := ioutil.ReadAll(r.Body)
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
r.Body.Close()
reqBuf, err := snappy.Decode(nil, compressed)
if err != nil {
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
var req prompb.WriteRequest
if err := proto.Unmarshal(reqBuf, &req); err != nil {
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
nodeChans[idx%httpworkers] <- req
})
http.HandleFunc("/check", func(w http.ResponseWriter, r *http.Request) {
compressed, err := ioutil.ReadAll(r.Body)
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
//blmLog.Println(string(compressed))
var output string = ""
schema, ok := IsSTableCreated.Load(string(compressed))
if !ok {
output = "the stable is not created!"
} else {
ntag := schema.(nametag)
tbtaglist := ntag.taglist
tbtagmap := ntag.tagmap
//annotlen := ntag.annotlen
output = "tags: "
for e := tbtaglist.Front(); e != nil; e = e.Next() {
output = output + e.Value.(string) + " | "
}
output = output + "\ntagmap: "
s := fmt.Sprintln(tbtagmap)
output = output + s
}
res := queryTableStruct(string(compressed))
output = output + "\nTable structure:\n" + res
s := fmt.Sprintf("query result:\n %s\n", output)
//blmLog.Println(s)
w.Write([]byte(s))
w.WriteHeader(http.StatusOK)
})
http.HandleFunc("/health", func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusNoContent)
})
if debugprt == 5 {
TestSerialization()
}
blmLog.Fatal(http.ListenAndServe(":"+rwport, nil))
}
func queryTableStruct(tbname string) string {
client := new(http.Client)
s := fmt.Sprintf("describe %s.%s", dbname, tbname)
body := strings.NewReader(s)
req, _ := http.NewRequest("GET", "http://"+tdurl+":"+apiport+"/rest/sql", body)
//fmt.Println("http://" + tdurl + ":" + apiport + "/rest/sql" + s)
req.SetBasicAuth(dbuser, dbpassword)
resp, err := client.Do(req)
if err != nil {
blmLog.Println(err)
fmt.Println(err)
return ""
} else {
compressed, _ := ioutil.ReadAll(resp.Body)
defer resp.Body.Close()
return string(compressed)
}
}
func TAOShashID(ba []byte) int {
var sum int = 0
for i := 0; i < len(ba); i++ {
sum += int(ba[i] - '0')
}
return sum
}
func NodeProcess(workerid int) error {
for req := range nodeChans[workerid] {
ProcessReq(req)
}
return nil
}
func ProcessReq(req prompb.WriteRequest) error {
db, err := sql.Open(taosDriverName, dbuser+":"+dbpassword+"@/tcp("+daemonIP+")/"+dbname)
if err != nil {
blmLog.Fatalf("Open database error: %s\n", err)
}
defer db.Close()
for _, ts := range req.Timeseries {
err = HandleStable(ts, db)
}
return err
}
func HandleStable(ts *prompb.TimeSeries, db *sql.DB) error {
taglist := list.New()
tbtaglist := list.New()
tagmap := make(map[string]string)
tbtagmap := make(map[string]string)
m := make(model.Metric, len(ts.Labels))
tagnum := tagnumlimit
var hasName bool = false
var metricsName string
var tbn string = ""
var nt nametag
//var annotlen int
//fmt.Println(ts)
j := 0
for _, l := range ts.Labels {
m[model.LabelName(l.Name)] = model.LabelValue(l.Value)
if string(l.Name) == "__name__" {
metricsName = string(l.Value)
tbn += metricsName
hasName = true
continue
}
j++
ln := strings.ToLower(string(l.Name))
OrderInsertS(ln, taglist)
//taglist.PushBack(ln)
s := string(l.Value)
tbn += s
//tagHash += s
if j <= tagnum {
OrderInsertS(ln, tbtaglist)
//tbtaglist.PushBack(ln)
if len(s) > taglen {
s = s[:taglen]
}
tbtagmap[ln] = "y"
}
tagmap[ln] = s
}
if debugprt == 2 {
t := ts.Samples[0].Timestamp
var ns int64 = 0
if t/1000000000 > 10 {
tm := t / 1000
ns = t - tm*1000
}
blmLog.Printf(" Ts: %s, value: %f, ", time.Unix(t/1000, ns), ts.Samples[0].Value)
blmLog.Println(ts)
}
if !hasName {
info := fmt.Sprintf("no name metric")
panic(info)
}
stbname := tablenameEscape(metricsName)
var ok bool
schema, ok := IsSTableCreated.Load(stbname)
if !ok { // no local record of super table structure
nt.taglist = tbtaglist
nt.tagmap = tbtagmap
stbdescription := queryTableStruct(stbname) //query the super table from TDengine
var stbst map[string]interface{}
err := json.Unmarshal([]byte(stbdescription), &stbst)
if err == nil { //query tdengine table success!
status := stbst["status"].(string)
if status == "succ" { //yes, the super table was already created in TDengine
taostaglist := list.New()
taostagmap := make(map[string]string)
dt := stbst["data"]
for _, fd := range dt.([]interface{}) {
fdc := fd.([]interface{})
if fdc[3].(string) == "tag" && fdc[0].(string) != "taghash" {
tmpstr := fdc[0].(string)
taostaglist.PushBack(tmpstr[2:])
taostagmap[tmpstr[2:]] = "y"
}
}
nt.taglist = taostaglist
nt.tagmap = taostagmap
tbtaglist = nt.taglist
tbtagmap = nt.tagmap
// annotlen = ntag.annotlen
var sqlcmd string
i := 0
for _, l := range ts.Labels {
k := strings.ToLower(string(l.Name))
if k == "__name__" {
continue
}
i++
if i < tagnumlimit {
_, ok := tbtagmap[k]
if !ok {
sqlcmd = "alter table " + stbname + " add tag t_" + k + tagstr + "\n"
_, err := execSql(dbname, sqlcmd, db)
if err != nil {
blmLog.Println(err)
errorcode := fmt.Sprintf("%s", err)
if strings.Contains(errorcode, "duplicated column names") {
tbtaglist.PushBack(k)
//OrderInsertS(k, tbtaglist)
tbtagmap[k] = "y"
}
} else {
tbtaglist.PushBack(k)
//OrderInsertS(k, tbtaglist)
tbtagmap[k] = "y"
}
}
}
}
IsSTableCreated.Store(stbname, nt)
} else { // no, the super table haven't been created in TDengine, create it.
var sqlcmd string
sqlcmd = "create table if not exists " + stbname + " (ts timestamp, value double) tags(taghash binary(34)"
for e := tbtaglist.Front(); e != nil; e = e.Next() {
sqlcmd = sqlcmd + ",t_" + e.Value.(string) + tagstr
}
//annotlen = taglimit - i*taglen
//nt.annotlen = annotlen
//annotationstr := fmt.Sprintf(" binary(%d)", annotlen)
//sqlcmd = sqlcmd + ", annotation " + annotationstr + ")\n"
sqlcmd = sqlcmd + ")\n"
_, err := execSql(dbname, sqlcmd, db)
if err == nil {
IsSTableCreated.Store(stbname, nt)
} else {
blmLog.Println(err)
}
}
} else { //query TDengine table error
blmLog.Println(err)
}
} else {
ntag := schema.(nametag)
tbtaglist = ntag.taglist
tbtagmap = ntag.tagmap
// annotlen = ntag.annotlen
var sqlcmd string
i := 0
for _, l := range ts.Labels {
k := strings.ToLower(string(l.Name))
if k == "__name__" {
continue
}
i++
if i < tagnumlimit {
_, ok := tbtagmap[k]
if !ok {
sqlcmd = "alter table " + stbname + " add tag t_" + k + tagstr + "\n"
_, err := execSql(dbname, sqlcmd, db)
if err != nil {
blmLog.Println(err)
errorcode := fmt.Sprintf("%s", err)
if strings.Contains(errorcode, "duplicated column names") {
tbtaglist.PushBack(k)
//OrderInsertS(k, tbtaglist)
tbtagmap[k] = "y"
}
} else {
tbtaglist.PushBack(k)
//OrderInsertS(k, tbtaglist)
tbtagmap[k] = "y"
}
}
}
}
}
tbnhash := "MD5_" + md5V2(tbn)
_, tbcreated := IsTableCreated.Load(tbnhash)
if !tbcreated {
var sqlcmdhead, sqlcmd string
sqlcmdhead = "create table if not exists " + tbnhash + " using " + stbname + " tags(\""
sqlcmd = ""
i := 0
for e := tbtaglist.Front(); e != nil; e = e.Next() {
tagvalue, has := tagmap[e.Value.(string)]
if len(tagvalue) > taglen {
tagvalue = tagvalue[:taglen]
}
if i == 0 {
if has {
sqlcmd = sqlcmd + "\"" + tagvalue + "\""
} else {
sqlcmd = sqlcmd + "null"
}
i++
} else {
if has {
sqlcmd = sqlcmd + ",\"" + tagvalue + "\""
} else {
sqlcmd = sqlcmd + ",null"
}
}
}
/*
var annotation string = ""
for t := taglist.Front(); t != nil; t = t.Next() {
_, has := tbtagmap[t.Value.(string)]
if !has {
tagvalue, _ := tagmap[t.Value.(string)]
annotation += t.Value.(string) + "=" + tagvalue + ","
}
}
if len(annotation) > annotlen-2 {
annotation = annotation[:annotlen-2]
}*/
//sqlcmd = sqlcmd + ",\"" + annotation + "\")\n"
var keys []string
var tagHash = ""
for t := range tagmap {
keys = append(keys, t)
}
sort.Strings(keys)
for _, k := range keys {
tagHash += tagmap[k]
}
sqlcmd = sqlcmd + ")\n"
sqlcmd = sqlcmdhead + md5V2(tagHash) + "\"," + sqlcmd
_, err := execSql(dbname, sqlcmd, db)
if err == nil {
IsTableCreated.Store(tbnhash, true)
}
}
serilizeTDengine(ts, tbnhash, db)
return nil
}
func tablenameEscape(mn string) string {
stbb := strings.ReplaceAll(mn, ":", "_") // replace : in the metrics name to adapt the TDengine
stbc := strings.ReplaceAll(stbb, ".", "_")
stbname := strings.ReplaceAll(stbc, "-", "_")
if len(stbname) > 60 {
stbname = stbname[:60]
}
return stbname
}
func serilizeTDengine(m *prompb.TimeSeries, tbn string, db *sql.DB) error {
idx := TAOShashID([]byte(tbn))
sqlcmd := " " + tbn + " values("
vl := m.Samples[0].GetValue()
vls := strconv.FormatFloat(vl, 'E', -1, 64)
if vls == "NaN" {
vls = "null"
}
tl := m.Samples[0].GetTimestamp()
tls := strconv.FormatInt(tl, 10)
sqlcmd = sqlcmd + tls + "," + vls + ")\n"
batchChans[idx%sqlworkers] <- sqlcmd
return nil
}
func createDatabase(dbname string) {
db, err := sql.Open(taosDriverName, dbuser+":"+dbpassword+"@/tcp("+daemonIP+")/")
if err != nil {
log.Fatalf("Open database error: %s\n", err)
}
defer db.Close()
sqlcmd := fmt.Sprintf("create database if not exists %s tables %d", dbname, tablepervnode)
_, err = db.Exec(sqlcmd)
sqlcmd = fmt.Sprintf("use %s", dbname)
_, err = db.Exec(sqlcmd)
checkErr(err)
return
}
func execSql(dbname string, sqlcmd string, db *sql.DB) (sql.Result, error) {
if len(sqlcmd) < 1 {
return nil, nil
}
if debugprt == 2 {
blmLog.Println(sqlcmd)
}
res, err := db.Exec(sqlcmd)
if err != nil {
blmLog.Printf("execSql Error: %s sqlcmd: %s\n", err, sqlcmd)
}
return res, err
}
func checkErr(err error) {
if err != nil {
blmLog.Println(err)
}
}
func md5V2(str string) string {
data := []byte(str)
has := md5.Sum(data)
md5str := fmt.Sprintf("%x", has)
return md5str
}
func processBatches(iworker int) {
var i int
db, err := sql.Open(taosDriverName, dbuser+":"+dbpassword+"@/tcp("+daemonIP+")/"+dbname)
if err != nil {
blmLog.Printf("processBatches Open database error: %s\n", err)
var count int = 5
for {
if err != nil && count > 0 {
<-time.After(time.Second * 1)
_, err = sql.Open(taosDriverName, dbuser+":"+dbpassword+"@/tcp("+daemonIP+")/"+dbname)
count--
} else {
if err != nil {
blmLog.Printf("processBatches Error: %s open database\n", err)
return
}
break
}
}
}
defer db.Close()
sqlcmd := make([]string, batchSize+1)
i = 0
sqlcmd[i] = "Import into"
i++
//blmLog.Printf("processBatches")
for onepoint := range batchChans[iworker] {
sqlcmd[i] = onepoint
i++
if i > batchSize {
i = 1
//blmLog.Printf(strings.Join(sqlcmd, ""))
_, err := db.Exec(strings.Join(sqlcmd, ""))
if err != nil {
blmLog.Printf("processBatches error %s", err)
var count int = 2
for {
if err != nil && count > 0 {
<-time.After(time.Second * 1)
_, err = db.Exec(strings.Join(sqlcmd, ""))
count--
} else {
if err != nil {
blmLog.Printf("Error: %s sqlcmd: %s\n", err, strings.Join(sqlcmd, ""))
}
break
}
}
}
}
}
if i > 1 {
i = 1
// blmLog.Printf(strings.Join(sqlcmd, ""))
_, err := db.Exec(strings.Join(sqlcmd, ""))
if err != nil {
var count int = 2
for {
if err != nil && count > 0 {
<-time.After(time.Second * 1)
_, err = db.Exec(strings.Join(sqlcmd, ""))
count--
} else {
if err != nil {
blmLog.Printf("Error: %s sqlcmd: %s\n", err, strings.Join(sqlcmd, ""))
}
break
}
}
}
}
workersGroup.Done()
}
func TestSerialization() {
var req prompb.WriteRequest
var ts []*prompb.TimeSeries
var tse prompb.TimeSeries
var sample *prompb.Sample
var label prompb.Label
var lbs []*prompb.Label
promPath, err := os.Getwd()
if err != nil {
fmt.Printf("can't get current dir :%s \n", err)
os.Exit(1)
}
promPath = filepath.Join(promPath, "testData/blm_prometheus.log")
testfile, err := os.OpenFile(promPath, os.O_RDWR, 0666)
if err != nil {
fmt.Println("Open file error!", err)
return
}
defer testfile.Close()
fmt.Println(promPath)
buf := bufio.NewReader(testfile)
i := 0
lasttime := "20:40:20"
for {
line, err := buf.ReadString('\n')
if err != nil {
if err == io.EOF {
fmt.Println("File read ok! line:", i)
break
} else {
fmt.Println("Read file error!", err)
return
}
}
sa := strings.Split(line, " ")
if strings.Contains(line, "server.go:201:") {
if sa[3] != lasttime {
nodeChans[0] <- req
lasttime = sa[3]
req.Timeseries = req.Timeseries[:0]
ts = ts[:0]
}
tse.Samples = make([]prompb.Sample, 0)
T, _ := strconv.ParseInt(sa[7][:(len(sa[7])-1)], 10, 64)
V, _ := strconv.ParseFloat(sa[9][:(len(sa[9])-1)], 64)
sample = &prompb.Sample{
Value: V,
Timestamp: T,
}
tse.Samples = append(tse.Samples, *sample)
} else if strings.Contains(line, "server.go:202:") {
lbs = make([]*prompb.Label, 0)
lb := strings.Split(line[45:], "{")
label.Name = "__name__"
label.Value = lb[0]
lbs = append(lbs, &label)
lbc := strings.Split(lb[1][:len(lb[1])-1], ", ")
for i = 0; i < len(lbc); i++ {
content := strings.Split(lbc[i], "=\"")
label.Name = content[0]
if i == len(lbc)-1 {
label.Value = content[1][:len(content[1])-2]
} else {
label.Value = content[1][:len(content[1])-1]
}
lbs = append(lbs, &label)
}
tse.Labels = lbs
ts = append(ts, &tse)
req.Timeseries = ts
}
}
}
func TAOSstrCmp(a string, b string) bool {
//return if a locates before b in a dictrionary.
for i := 0; i < len(a) && i < len(b); i++ {
if int(a[i]-'0') > int(b[i]-'0') {
return false
} else if int(a[i]-'0') < int(b[i]-'0') {
return true
}
}
if len(a) > len(b) {
return false
} else {
return true
}
}
func OrderInsertS(s string, l *list.List) {
e := l.Front()
if e == nil {
l.PushFront(s)
return
}
for e = l.Front(); e != nil; e = e.Next() {
str := e.Value.(string)
if TAOSstrCmp(str, s) {
continue
} else {
l.InsertBefore(s, e)
return
}
}
l.PushBack(s)
return
}
bailongma/blm_telegraf/Dockerfile 0 → 100644
浏览文件 @ 438174a6
## Builder image
FROM tdengine/tdengine:dev as builder1
FROM tdengine/bailongma:dev as builder
COPY server.go /root/blm_telegraf/
WORKDIR /root/blm_telegraf
RUN go build
# # build community version
# RUN cmake .. -DVERSION=lite && cmake --build .
## Target image
FROM centos:7
WORKDIR /root
# COPY --from=builder /root/build/build/lib/libtaos.so /usr/lib/libtaos.so.1
# RUN ln -s /usr/lib/libtaos.so.1 /usr/lib/libtaos.so
COPY --from=builder1 /usr/include/taos.h /usr/include/
COPY --from=builder1 /usr/lib/libtaos.so /usr/lib/libtaos.so
RUN ln -s /usr/lib/libtaos.so /usr/lib/libtaos.so.1
COPY --from=builder /root/blm_telegraf/blm_telegraf .
#COPY community/packaging/cfg/taos.cfg /etc/taos/
ENV LD_LIBRARY_PATH="$LD_LIBRARY_PATH:/usr/lib"
ENV LANG=en_US.UTF-8
ENV LANGUAGE=en_US:en
ENV LC_ALL=en_US.UTF-8
EXPOSE 10202
VOLUME [ "/var/lib/taos", "/var/log/taos" ]
ENTRYPOINT [ "/root/blm_telegraf","--host" ]
\ No newline at end of file
bailongma/blm_telegraf/README.md 0 → 100644
浏览文件 @ 438174a6
# API for Telegraf
This is an API to support Telegraf writing data into TDengine.
## prerequisite
before running the software, you need to install the `golang-1.10` or later version in your environment and install [TDengine][] so the program can use the lib of TDengine.
To use it:
```
go build
```
During the go build process, there maybe some errors arised because of lacking some needed packages. You can use `go get` the package to solve it
```
go get github.com/taosdata/TDengine/src/connector/go/src/taosSql
```
After successful build, there will be a blm_telegraf in the same directory.
## Running in background
Using following command to run the program in background
```
nohup ./blm_telegraf --host 112.102.3.69:0 --batch-size 200 --http-workers 2 --sql-workers 2 --dbname telegraf --port 1234 > /dev/null 2>&1 &
```
The API url is `http://ipaddress:port/telegraf`
There are several options can be set:
```
--host
set the host of TDengine, IP:port, for example "192.168.0.1:0"
--batch-size
set the size of how many records in one SQL cmd line writing into TDengine. There is a limitation that TDengine could only accept SQL line small than 64000 bytes, so usually the batch size should not exceed 800. Default is 10.
--http-workers
set the number of workers who process the HTTP request. default is 10
--sql-workers
set the number of workers who process the database request. default is 10
--dbname
set the database name in TDengine, if not exists, a database will be created after this dbname. default is "telegraf".
--dbuser
set the user name that have the right to access the TDengine. default is "root"
--dbpassword
set the password of dbuser. default is "taosdata"
--port
set the port that prometheus configuration remote_write. as showed above, in the prometheus.yaml
```
## Configure the Telegraf
To write into blm_telegraf API, you should configure the telegraf as below
In the telegraf configuration file, output plugin part:
1. telegraf out put plugin setup:
Set the url to the blm_telegraf API
Set the data format as "json"
Set the json timstamp units as "1ms"
```toml
[[outputs.http]]
# ## URL is the address to send metrics to
url = "http://114.116.124.178:8081/telegraf"
data_format = "json"
json_timestamp_units = "1ms"
```
In the Agent part, the hostname should be unique among all the telegraf which report to the TDengine.
```toml
# Configuration for telegraf agent
[agent]
## Default data collection interval for all inputs
interval = "5s"
...
## Override default hostname, if empty use os.Hostname()
hostname = "testhost1"
```
Then start telegraf:
```sh
telegraf --config xxx.conf
```
Then you can check the TDengine if there is super table and tables.
## Check the TDengine tables and datas
Use the taos client shell to query the result.
```
Welcome to the TDengine shell from linux, client version:1.6.4.0 server version:1.6.4.0
Copyright (c) 2017 by TAOS Data, Inc. All rights reserved.
This is the trial version and will expire at 2019-12-11 14:25:31.
taos> use prometheus;
Database changed.
taos> show stables;
name | created_time |columns| tags | tables |
====================================================================================================================
system | 19-11-22 21:48:10.205| 2| 3| 12|
system_str | 19-11-22 21:48:10.205| 2| 3| 2|
cpu | 19-11-22 21:48:10.225| 2| 4| 200|
cpu_str | 19-11-22 21:48:10.226| 2| 4| 0|
processes | 19-11-22 21:48:10.230| 2| 3| 16|
processes_str | 19-11-22 21:48:10.230| 2| 3| 0|
disk | 19-11-22 21:48:10.233| 2| 7| 357|
disk_str | 19-11-22 21:48:10.234| 2| 7| 0|
diskio | 19-11-22 21:48:10.247| 2| 4| 72|
diskio_str | 19-11-22 21:48:10.248| 2| 4| 0|
swap | 19-11-22 21:48:10.254| 2| 3| 7|
swap_str | 19-11-22 21:48:10.255| 2| 3| 0|
mem | 19-11-22 21:48:10.272| 2| 3| 61|
mem_str | 19-11-22 21:48:10.272| 2| 3| 0|
Query OK, 14 row(s) in set (0.000733s)
taos> select * from mem;
......
19-11-23 14:19:11.000| 0.000000000|testhost1 |1.202.240.226 |huge_pages_free |
19-11-23 14:19:16.000| 0.000000000|testhost1 |1.202.240.226 |huge_pages_free |
19-11-23 14:19:21.000| 0.000000000|testhost1 |1.202.240.226 |huge_pages_free |
19-11-23 14:19:26.000| 0.000000000|testhost1 |1.202.240.226 |huge_pages_free |Query OK, 3029 row(s) in set (0.060828s)
```
## Support Kubernates liveness probe
The blm_telegraf support the liveness probe.
When the service is running, GET the url`http://ip:port/health` will return 200 OK response which means the service is running healthy. If no response, means the service is dead and need to restart it.
## Limitations
The TDengine limits the length of super table name, so if the name of Telegraf measurement name exceeds 60 byte, it will be truncated to first 60 bytes. And the length of tags name is limited within 50 byte.
[TDengine]:https://www.github.com/Taosdata/TDengine
\ No newline at end of file
bailongma/blm_telegraf/Telegraf写入TDengine的API.md 0 → 100644
浏览文件 @ 438174a6
# Telegraf写入TDengine的API程序
刚刚开发完Telegraf写入TDengine的APi程序，本文总结一下程序的设计思路，并在此基础上提出一种schemaless的写入TDengine的适配方法和对应的查询思路。
## Telegraf数据分析
Telegraf采集节点的数据后，按照数据的格式为measurement加上一系列的tags，再加上一系列的fields和timestamp，组成一条记录发出。
```
cpu,cpu=cpu-total,host=liutaodeMacBook-Pro.local usage_irq=0,usage_guest=0,usage_guest_nice=0,usage_iowait=0,usage_softirq=0,usage_steal=0,usage_user=10.55527763881941,usage_system=3.5767883941970986,usage_idle=85.86793396698349,usage_nice=0 1571663200000000000
```
上面是一条按照influxdb格式输出的记录，第一个字段是measurement, 然后接着两个tags, tags后面的空格来作为tags和fields的分隔；fields和timestamp之间也是用空格分隔。
```json
{
"fields":{
"usage_guest":0,
"usage_guest_nice":0,
"usage_idle":87.73726273726274,
"usage_iowait":0,
"usage_irq":0,
"usage_nice":0,
"usage_softirq":0,
"usage_steal":0,
"usage_system":2.6973026973026974,
"usage_user":9.565434565434565
},
"name":"cpu",
"tags":{
"cpu":"cpu-total",
"host":"liutaodeMacBook-Pro.local"
},
"timestamp":1571665100
}
```
如果按json格式输出，则一条json格式的记录如上。
上面的数据个看上去跟TDengine的记录格式十分类似，很自然的我们可以把name作为超级表名，tags作为tags，fields作为values，timestamp作为timestamp，对应上TDengine的数据格式，照理说应该非常好写入TDengine。但实际过程中发现telegraf在输出数据时，经常会遇到一个问题，就是name，tags格式一样的情况下，fields的格式不一样，fields里的名字，数量，都可能变化。这种变化不是随意变化，可能是两三种组合的变化。比如如下情况
```
swap,host=testhost1 out=0i,in=0i 1574663615000000000
swap,host=testhost1 total=4294967296i,used=3473670144i,free=821297152i,used_percent=80.877685546875 1574663615000000000
```
同一个时间点来的两条记录，name都是swap，tag都是host，但fields却完全不相同。
再比如
```
system,host=testhost1 uptime_format="5 days, 1:07" 1574663615000000000
system,host=testhost1 uptime=436070i 1574663615000000000
system,host=testhost1 load15=5.9521484375,n_cpus=4i,n_users=6i,load1=3.17138671875,load5=6.462890625 1574663615000000000
```
同一时间点来的三条记录，name都是system，tags都是host，但fields完全不同。
如果以name作为TDengine的超级表名，就会面临到表格的结构发生变化。并且，由于influxdb是schemaless的设计，他们能够很好的处理这种变化，不管fields如何变化，都能顺利写入。因此，很难保证telegraf后续产生的数据，fields发生会怎么变化。如何设计TDengine的存储表结构，是一个问题。
## TDengine的表结构设计思路
面对这种数据来源，我们一般可以用两种设计思路：
### 提前设计好超级表，包含所有fields, 这种就是schema方式
一种，是在对数据的行为有充分的了解后，提前设计好TDengine的表格式，将所有可能变化的fields都放到values中，提前创建好超级表；然后在插入数据时，把同一时间戳的所有fields都收集齐后，再组装成一条TDengine的SQL记录，写入TDengine。这种，是我们当前通常用到的方法，可以成为schema方式的写入。这种方法的优点是，比较符合我们TDengine的设计假设，values有多列，写入性能相对高一些。但也有明显的缺点，需要提前对数据做大量的分析，确定每个测量的格式，手动来写schema配置文件或手动在TDengine客户端创建超级表，对于节点很多的监控，这种设计会带来较大的工作量。
### 不提前设计超级表，设计一种建表规则，根据来的数据自动写入，类似influxdb的schemaless的方式
另外一种，就是根据收到的数据自动建表，只要符合name，tags，fields，timestamp的格式，都能顺利写的创建TDengine的表，并写入TDengine。本程序就是采用这种思路，将每一个fields单独拆开，和name，tags组合起来，形成一个单列的表。这样的超级表符合任何fields，对于任意fields都可以顺利写入。下面将沿着这个设计思路继续展开。
## 超级表
本程序以收到的原始数据name作为超级表名，原始数据中的tags作为tags，同时，额外增加两个tag，一个是发来请求的源IP，用来区分设备；另一个是field，这个tag的值是原始数据中fields的名称，用来表明这个超级表存的是哪个指标。以上面的system这个原始数据为例，则超级表结构为
```toml
stablename = system
tags = ['host','srcip','field']
values = ['timestamp','value']
```
其中，tags的类型都为binary(50),长度超过50的标签值都截断为50；field这个标签则的可能值则为
```toml
field : ['uptime_format','uptime','load15','n_cpus','n_users','load1'，'load5']
```
## value的类型
由于无法预知数据的类型，以及简化程序实现，我们将value的类型分成两类，一类是数值型，统一用double来存储；一类是字符串，统一用binary(256)的类型来存。由于所有field都要用同一个超级表来存，因此我一开始就为每个name创建了两个超级表，一个是数值型的超级表，表名就是name；另一个是字符串型的超级表，表名是name加上_str后缀。然后根据field的数据类型，如果是数值型，就用数值型的超级表来创建表；如果是字符串型的，就用name_str的超级表来创建表。
因此，超级表创建的时候会创建两倍的数据量
```
name | created_time |columns| tags | tables |
====================================================================================================================
system | 19-11-22 21:48:10.205| 2| 3| 12|
system_str | 19-11-22 21:48:10.205| 2| 3| 2|
cpu | 19-11-22 21:48:10.225| 2| 4| 200|
cpu_str | 19-11-22 21:48:10.226| 2| 4| 0|
processes | 19-11-22 21:48:10.230| 2| 3| 16|
processes_str | 19-11-22 21:48:10.230| 2| 3| 0|
disk | 19-11-22 21:48:10.233| 2| 7| 357|
disk_str | 19-11-22 21:48:10.234| 2| 7| 0|
diskio | 19-11-22 21:48:10.247| 2| 4| 72|
diskio_str | 19-11-22 21:48:10.248| 2| 4| 0|
swap | 19-11-22 21:48:10.254| 2| 3| 7|
swap_str | 19-11-22 21:48:10.255| 2| 3| 0|
mem | 19-11-22 21:48:10.272| 2| 3| 61|
mem_str | 19-11-22 21:48:10.272| 2| 3| 0|
Query OK, 14 row(s) in set (0.000588s)
```
因此查询的时候，需要根据查询值的类型，选择不同的超级表来查询
比如，对于数值类型，查询n_cpus值的语句为
```sql
Select * from system where field = "n_cpus";
```
对于字符串类型，查询uptime_format的值的语句为
```sql
Select * from system_str where field = "uptime_format";
```
## 表的创建
对于每个field，程序为它创建了一个表，表名规则如下：
将原始数据的所有tags值加上源ip加上field的名称，组成一个长的字符串，然后进行MD5计算，输出的结果加上MD5_作为前缀，形成表名。
这种规则，确保了只要数据的tags等特征不变，表就不会发生变化。
```
...
md5_b26d30c2e07529ac309d836b3b222f15 | 19-11-24 21:34:35.830| 2|processes |
md5_08147d718d4961368155f90432eab536 | 19-11-22 21:48:10.748| 2|disk |
md5_105158abfca0bbf0d932cc74bfc7e136 | 19-11-24 21:34:35.846| 2|mem |
md5_e6842b5c6b9744b7d5ce3510a4d54c98 | 19-11-24 21:34:35.874| 2|disk |
md5_285fd02686e0bfee76d22505dd29f14c | 19-11-22 21:48:11.509| 2|disk |
md5_9317870bb00109353f6aef58ee2ee9e9 | 19-11-24 21:34:35.919| 2|cpu |
Query OK, 727 row(s) in set (0.020405s)
```
因此在数据插入时，只要根据tags值和IP和field的名称，就能计算出表名，直接插入该表。
查询时，可以用超级表加上tag值和field值来查询，也很清晰便利。
因此，数值写入自动生成的sql语句如下：
```sql
insert into md5_285fd02686e0bfee76d22505dd29f14c values(1574663615000,375.2023040);
```
其中表名md5_285fd02686e0bfee76d22505dd29f14c是自动根据数据特征计算出来的，无需人工输入。
而查询则可以通过超级表来查询
## Schemaless写入方法
基于上面的实现，后续我们可以确定一个写入的语法，就可以不用提前设定schema，而根据接收到的数据，自动创建超级表，表，方便的写入TDengine了。
沿用现有的json格式，或者参考influxdb的语法，确定一个写入的语法，就可以实现influxdb的schemaless写入的能力。
bailongma/blm_telegraf/server.go 0 → 100644
浏览文件 @ 438174a6
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package main
import (
"container/list"
"crypto/md5"
"database/sql"
"encoding/json"
"flag"
"fmt"
"io/ioutil"
"log"
"net/http"
"os"
"sort"
"strconv"
"strings"
"sync"
"time"
_ "github.com/taosdata/driver-go/taosSql"
)
type metric struct {
Fields map[string]interface{}
Name string
Tags map[string]string
TimeStamp int64
}
type Metrics struct {
Metrics []metric
HostIP string
}
var (
daemonUrl string
httpworkers int
sqlworkers int
batchSize int
buffersize int
dbname string
dbuser string
dbpassword string
rwport string
debugprt int
taglen int
)
type nametag struct {
tagmap map[string]string
taglist *list.List
}
// Global vars
var (
bufPool sync.Pool
batchChans []chan string //multi table one chan
nodeChans []chan Metrics //multi node one chan
inputDone chan struct{}
//workersGroup sync.WaitGroup
reportTags [][2]string
reportHostname string
taosDriverName string = "taosSql"
IsSTableCreated sync.Map
IsTableCreated sync.Map
taglist *list.List
nametagmap map[string]nametag
tagstr string
blmLog *log.Logger
logNameDefault string = "/var/log/taos/blm_telegraf.log"
)
var scratchBufPool = &sync.Pool{
New: func() interface{} {
return make([]byte, 0, 1024)
},
}
// Parse args:
func init() {
flag.StringVar(&daemonUrl, "host", "", "TDengine host.")
flag.IntVar(&batchSize, "batch-size", 10, "Batch size (input items).")
flag.IntVar(&httpworkers, "http-workers", 10, "Number of parallel http requests handler .")
flag.IntVar(&sqlworkers, "sql-workers", 10, "Number of parallel sql handler.")
flag.StringVar(&dbname, "dbname", "telegraf", "Database name where to store metrics")
flag.StringVar(&dbuser, "dbuser", "root", "User for host to send result metrics")
flag.StringVar(&dbpassword, "dbpassword", "taosdata", "User password for Host to send result metrics")
flag.StringVar(&rwport, "port", "10202", "remote write port")
flag.IntVar(&debugprt, "debugprt", 0, "if 0 not print, if 1 print the sql")
flag.IntVar(&taglen, "tag-length", 30, "the max length of tag string")
flag.IntVar(&buffersize, "buffersize", 100, "the buffer size of metrics received")
flag.Parse()
daemonUrl = daemonUrl + ":0"
nametagmap = make(map[string]nametag)
fmt.Print("host: ")
fmt.Print(daemonUrl)
fmt.Print(" port: ")
fmt.Print(rwport)
fmt.Print(" database: ")
fmt.Print(dbname)
tagstr = fmt.Sprintf(" binary(%d)", taglen)
logFile, err := os.OpenFile(logNameDefault, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644)
if err != nil {
fmt.Println(err)
os.Exit(1)
}
blmLog = log.New(logFile, "", log.LstdFlags)
blmLog.SetPrefix("BLM_TLG")
blmLog.SetFlags(log.LstdFlags | log.Lshortfile)
}
func main() {
for i := 0; i < httpworkers; i++ {
nodeChans = append(nodeChans, make(chan Metrics, buffersize))
}
createDatabase(dbname)
for i := 0; i < httpworkers; i++ {
//workersGroup.Add(1)
go NodeProcess(i)
}
for i := 0; i < sqlworkers; i++ {
batchChans = append(batchChans, make(chan string, batchSize))
}
for i := 0; i < sqlworkers; i++ {
//workersGroup.Add(1)
go processBatches(i)
}
http.HandleFunc("/telegraf", func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusAccepted)
addr := strings.Split(r.RemoteAddr, ":")
idx := TAOShashID([]byte(addr[0]))
reqBuf, err := ioutil.ReadAll(r.Body)
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
r.Body.Close()
var req Metrics
if err := json.Unmarshal(reqBuf, &req); err != nil {
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
req.HostIP = addr[0]
nodeChans[idx%httpworkers] <- req
r.Body.Close()
})
http.HandleFunc("/health", func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusNoContent)
})
blmLog.Fatal(http.ListenAndServe(":"+rwport, nil))
}
func TAOShashID(ba []byte) int {
var sum int = 0
for i := 0; i < len(ba); i++ {
sum += int(ba[i] - '0')
}
return sum
}
func TAOSstrCmp(a string, b string) bool {
//return if a locates before b in a dictrionary.
for i := 0; i < len(a) && i < len(b); i++ {
if int(a[i]-'0') > int(b[i]-'0') {
return false
} else if int(a[i]-'0') < int(b[i]-'0') {
return true
}
}
if len(a) > len(b) {
return false
} else {
return true
}
}
func NodeProcess(workerid int) error {
for req := range nodeChans[workerid] {
ProcessReq(req)
}
return nil
}
func OrderInsert(ts int64, l *list.List) {
e := l.Front()
if e == nil {
l.PushFront(ts)
return
}
for e = l.Front(); e != nil; e = e.Next() {
if e.Value.(int64) < ts {
continue
} else {
l.InsertBefore(ts, e)
return
}
}
}
func OrderInsertS(s string, l *list.List) {
e := l.Front()
if e == nil {
l.PushFront(s)
return
}
for e = l.Front(); e != nil; e = e.Next() {
str := e.Value.(string)
if TAOSstrCmp(str, s) {
continue
} else {
l.InsertBefore(s, e)
return
}
}
l.PushBack(s)
return
}
func ProcessReq(req Metrics) error {
tsmap := make(map[int64]map[string][]metric)
tslist := list.New()
addr := req.HostIP
var lastTs int64 = 0
for i := 0; i < len(req.Metrics); i++ {
m := req.Metrics[i]
if tsmap[m.TimeStamp] == nil {
tsmap[m.TimeStamp] = make(map[string][]metric)
}
mp := tsmap[m.TimeStamp]
mp[m.Name] = append(mp[m.Name], m)
if lastTs != m.TimeStamp { //there is still some case that will make mistake, when the timestamp is totally out of order. but right now just forget it.
OrderInsert(m.TimeStamp, tslist)
}
lastTs = m.TimeStamp
}
for e := tslist.Front(); e != nil; e = e.Next() {
namemap, ok := tsmap[e.Value.(int64)]
if ok {
for _, v := range namemap {
ProcessData(v, dbname, addr)
}
} else {
info := fmt.Sprintf("ProcessReq: cannot retrieve map")
panic(info)
}
}
return nil
}
func SerilizeTDengine(m metric, dbn string, hostip string, taglist *list.List, db *sql.DB) error {
var tbna []string
for _, v := range m.Tags {
tbna = append(tbna, v)
}
sort.Strings(tbna)
tbn := strings.Join(tbna, "") // Go map 遍历结果是随机的，必须排下序
for k, v := range m.Fields {
s := m.Name + tbn + hostip + k
//fmt.Print(s)
s = "MD5_" + md5V2(s)
_, ok := IsTableCreated.Load(s)
if !ok {
var sqlcmd string
switch v.(type) {
case string:
sqlcmd = "create table if not exists " + s + " using " + m.Name + "_str tags("
default:
sqlcmd = "create table if not exists " + s + " using " + m.Name + " tags("
}
for e := taglist.Front(); e != nil; e = e.Next() {
tagvalue, has := m.Tags[e.Value.(string)]
if len(tagvalue) >= 60 {
tagvalue = tagvalue[:59]
}
if has {
sqlcmd = sqlcmd + "\"" + tagvalue + "\","
} else {
sqlcmd = sqlcmd + "null,"
}
}
sqlcmd = sqlcmd + "\"" + hostip + "\"," + "\"" + k + "\")\n"
execSql(dbn, sqlcmd, db)
IsTableCreated.Store(s, true)
}
idx := TAOShashID([]byte(s))
sqlcmd := " " + s + " values("
tls := strconv.FormatInt(m.TimeStamp, 10)
switch v.(type) {
case string:
sqlcmd = sqlcmd + tls + ",\"" + v.(string) + "\")"
case int64:
sqlcmd = sqlcmd + tls + "," + strconv.FormatInt(v.(int64), 10) + ")"
case float64:
sqlcmd = sqlcmd + tls + "," + strconv.FormatFloat(v.(float64), 'E', -1, 64) + ")"
default:
panic("Checktable error value type")
}
batchChans[idx%sqlworkers] <- sqlcmd
//execSql(dbn,sqlcmd)
}
return nil
}
func ProcessData(ts []metric, dbn string, hostip string) error {
db, err := sql.Open(taosDriverName, dbuser+":"+dbpassword+"@/tcp("+daemonUrl+")/"+dbname)
if err != nil {
blmLog.Fatalf("Open database error: %s\n", err)
}
defer db.Close()
schema, ok := IsSTableCreated.Load(ts[0].Name)
if !ok {
var nt nametag
nt.taglist = list.New()
nt.tagmap = make(map[string]string)
IsSTableCreated.Store(ts[0].Name, nt)
tagmap := nt.tagmap
taglist := nt.taglist
for i := 0; i < len(ts); i++ {
for k, _ := range ts[i].Tags {
_, ok := tagmap[k]
if !ok {
taglist.PushBack(k)
tagmap[k] = "y"
}
}
}
var sqlcmd string
sqlcmd = "create table if not exists " + ts[0].Name + " (ts timestamp, value double) tags("
sqlcmd1 := "create table if not exists " + ts[0].Name + "_str (ts timestamp, value binary(256)) tags("
for e := taglist.Front(); e != nil; e = e.Next() {
sqlcmd = sqlcmd + e.Value.(string) + tagstr + ","
sqlcmd1 = sqlcmd1 + e.Value.(string) + tagstr + ","
}
sqlcmd = sqlcmd + "srcip binary(20), field binary(40))\n"
sqlcmd1 = sqlcmd1 + "srcip binary(20), field binary(40))\n"
execSql(dbn, sqlcmd, db)
execSql(dbn, sqlcmd1, db)
for i := 0; i < len(ts); i++ {
SerilizeTDengine(ts[i], dbn, hostip, taglist, db)
}
return nil
}
nt := schema.(nametag)
tagmap := nt.tagmap
taglist := nt.taglist
var sqlcmd, sqlcmd1 string
for i := 0; i < len(ts); i++ {
for k, _ := range ts[i].Tags {
_, ok := tagmap[k]
if !ok {
sqlcmd = sqlcmd + "alter table " + ts[0].Name + " add tag " + k + tagstr + "\n"
sqlcmd1 = sqlcmd1 + "alter table " + ts[0].Name + "_str add tag " + k + tagstr + "\n"
taglist.PushBack(k)
tagmap[k] = "y"
}
}
}
execSql(dbn, sqlcmd, db)
execSql(dbn, sqlcmd1, db)
for i := 0; i < len(ts); i++ {
SerilizeTDengine(ts[i], dbn, hostip, taglist, db)
}
return nil
}
func createDatabase(dbname string) {
db, err := sql.Open(taosDriverName, dbuser+":"+dbpassword+"@/tcp("+daemonUrl+")/")
if err != nil {
log.Fatalf("Open database error: %s\n", err)
}
defer db.Close()
sqlcmd := fmt.Sprintf("create database if not exists %s", dbname)
_, err = db.Exec(sqlcmd)
sqlcmd = fmt.Sprintf("use %s", dbname)
_, err = db.Exec(sqlcmd)
checkErr(err)
return
}
func execSql(dbname string, sqlcmd string, db *sql.DB) {
if len(sqlcmd) < 1 {
return
}
_, err := db.Exec(sqlcmd)
if err != nil {
var count int = 2
for {
if err != nil && count > 0 {
<-time.After(time.Second * 1)
_, err = db.Exec(sqlcmd)
count--
} else {
if err != nil {
blmLog.Printf("execSql Error: %s sqlcmd: %s\n", err, sqlcmd)
return
}
break
}
}
}
return
}
func checkErr(err error) {
if err != nil {
blmLog.Println(err)
}
}
func md5V2(str string) string {
data := []byte(str)
has := md5.Sum(data)
md5str := fmt.Sprintf("%x", has)
return md5str
}
func processBatches(iworker int) {
var i int
db, err := sql.Open(taosDriverName, dbuser+":"+dbpassword+"@/tcp("+daemonUrl+")/"+dbname)
if err != nil {
blmLog.Printf("processBatches Open database error: %s\n", err)
var count int = 5
for {
if err != nil && count > 0 {
<-time.After(time.Second * 1)
_, err = sql.Open(taosDriverName, dbuser+":"+dbpassword+"@/tcp("+daemonUrl+")/"+dbname)
count--
} else {
if err != nil {
blmLog.Printf("processBatches Error: %s open database\n", err)
return
}
break
}
}
}
defer db.Close()
sqlcmd := make([]string, batchSize+1)
i = 0
sqlcmd[i] = "Insert into"
i++
for onepoint := range batchChans[iworker] {
sqlcmd[i] = onepoint
i++
if i > batchSize {
i = 1
_, err := db.Exec(strings.Join(sqlcmd, ""))
if err != nil {
var count int = 2
for {
if err != nil && count > 0 {
<-time.After(time.Second * 1)
_, err = db.Exec(strings.Join(sqlcmd, ""))
count--
} else {
if err != nil {
blmLog.Printf("Error: %s sqlcmd: %s\n", err, strings.Join(sqlcmd, ""))
}
break
}
}
}
}
}
if i > 1 {
i = 1
_, err := db.Exec(strings.Join(sqlcmd, ""))
if err != nil {
var count int = 2
for {
if err != nil && count > 0 {
<-time.After(time.Second * 1)
_, err = db.Exec(strings.Join(sqlcmd, ""))
count--
} else {
if err != nil {
blmLog.Printf("Error: %s sqlcmd: %s\n", err, strings.Join(sqlcmd, ""))
}
break
}
}
}
}
//workersGroup.Done()
}
bailongma/go.mod 0 → 100644
浏览文件 @ 438174a6
module github.com/taosdata/TDengine/bailongma
go 1.14
require (
github.com/gogo/protobuf v1.3.1
github.com/golang/snappy v0.0.1
github.com/grpc-ecosystem/grpc-gateway v1.14.5 // indirect
github.com/prometheus/common v0.9.1
github.com/prometheus/prometheus v2.5.0+incompatible
github.com/taosdata/driver-go v0.0.0-20200311072652-8c58c512b6ac
google.golang.org/genproto v0.0.0-20200507105951-43844f6eee31 // indirect
google.golang.org/grpc v1.29.1 // indirect
)
bailongma/go.sum 0 → 100644
浏览文件 @ 438174a6
cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
github.com/antihax/optional v0.0.0-20180407024304-ca021399b1a6/go.mod h1:V8iCPQYkqmusNa815XgQio277wI47sdRh1dUOLdyC6Q=
github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98=
github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE=
github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk=
github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
github.com/gogo/protobuf v1.3.1 h1:DqDEcV5aeaTmdFBePNpYsp3FlcVH/2ISVVM9Qf8PSls=
github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o=
github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
github.com/golang/protobuf v1.3.3 h1:gyjaxf+svBWX08ZjK86iN9geUJF0H6gp2IRKX6Nf6/I=
github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw=
github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4=
github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
github.com/grpc-ecosystem/grpc-gateway v1.14.5 h1:aiLxiiVzAXb7wb3lAmubA69IokWOoUNe+E7TdGKh8yw=
github.com/grpc-ecosystem/grpc-gateway v1.14.5/go.mod h1:UJ0EZAp832vCd54Wev9N1BMKEyvcZ5+IM0AwDrnlkEc=
github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00=
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo=
github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
github.com/prometheus/common v0.9.1 h1:KOMtN28tlbam3/7ZKEYKHhKoJZYYj3gMH4uc62x7X7U=
github.com/prometheus/common v0.9.1/go.mod h1:yhUN8i9wzaXS3w1O07YhxHEBxD+W35wd8bs7vj7HSQ4=
github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
github.com/prometheus/prometheus v1.8.2 h1:PAL466mnJw1VolZPm1OarpdUpqukUy/eX4tagia17DM=
github.com/prometheus/prometheus v2.5.0+incompatible h1:7QPitgO2kOFG8ecuRn9O/4L9+10He72rVRJvMXrE9Hg=
github.com/prometheus/prometheus v2.5.0+incompatible/go.mod h1:oAIUtOny2rjMX0OWN5vPR5/q/twIROJvdqnQKDdil/s=
github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ=
github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
github.com/taosdata/TDengine v0.0.0-20200511045913-fead5b90383d h1:er7tnj+w7vOo1mLnOL6imwYJdKsvuR2d6/GS8Dyt6oc=
github.com/taosdata/driver-go v0.0.0-20200311072652-8c58c512b6ac h1:uZplMwObJj8mfgI4ZvYPNHRn+fNz2leiMPqShsjtEEc=
github.com/taosdata/driver-go v0.0.0-20200311072652-8c58c512b6ac/go.mod h1:TuMZDpnBrjNO07rneM2C5qMYFqIro4aupL2cUOGGo/I=
golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20190613194153-d28f0bde5980 h1:dfGZHvZk057jK2MCeWus/TowKpJ8y4AmooUzdBSR9GU=
golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20191002035440-2ec189313ef0 h1:2mqDk8w/o6UmeUCu5Qiq2y7iMf6anbx+YA8d1JFoFrs=
golang.org/x/net v0.0.0-20191002035440-2ec189313ef0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190422165155-953cdadca894 h1:Cz4ceDQGXuKRnVBDTS23GTn/pU5OE2C0WrNTOYK1Uuc=
golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
google.golang.org/genproto v0.0.0-20190927181202-20e1ac93f88c/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8=
google.golang.org/genproto v0.0.0-20200507105951-43844f6eee31 h1:Bz1qTn2YRWV+9OKJtxHJiQKCiXIdf+kwuKXdt9cBxyU=
google.golang.org/genproto v0.0.0-20200507105951-43844f6eee31/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
google.golang.org/grpc v1.24.0/go.mod h1:XDChyiUovWa60DnaeDeZmSW86xtLtjtZbwvSiRnRtcA=
google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY=
google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
google.golang.org/grpc v1.29.1 h1:EC2SB8S04d2r73uptxphDSUG+kTKVgjRPF+N3xpxRB4=
google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk=
gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
deps/dirent/CMakeLists.txt 0 → 100644
浏览文件 @ 438174a6
cmake_minimum_required (VERSION 2.8.11)
project (dirent LANGUAGES C CXX)
# Initialize C and C++ compilers
enable_language (C CXX)
# Compile in debug mode by default
if (NOT CMAKE_BUILD_TYPE)
set (CMAKE_BUILD_TYPE Debug
CACHE STRING
"Type of build: None Debug Release RelWithDebInfo MinSizeRel."
FORCE
)
endif (NOT CMAKE_BUILD_TYPE)
# Use the include directory only on Windows
if (WIN32)
include_directories (${CMAKE_SOURCE_DIR}/include)
endif (WIN32)
# Install dirent.h
if (WIN32)
install (FILES include/dirent.h DESTINATION include)
endif (WIN32)
# Add distclean target
add_custom_target (distclean
COMMAND ${CMAKE_BUILD_TOOL} clean
COMMAND ${CMAKE_COMMAND} -P ${CMAKE_SOURCE_DIR}/distclean.cmake
)
# Build example programs
add_executable (find examples/find.c)
add_executable (ls examples/ls.c)
add_executable (locate examples/locate.c)
add_executable (updatedb examples/updatedb.c)
add_executable (scandir examples/scandir.c)
add_executable (cat examples/cat.c)
# Build test programs
include (CTest)
add_custom_target (check COMMAND ${CMAKE_CTEST_COMMAND} --output-on-failure -C ${CMAKE_CFG_INTDIR})
function (add_test_executable TEST_NAME)
add_executable (${TEST_NAME} EXCLUDE_FROM_ALL ${ARGN})
add_test (NAME ${TEST_NAME} WORKING_DIRECTORY ${CMAKE_SOURCE_DIR} COMMAND $<TARGET_FILE:${TEST_NAME}>)
add_dependencies (check ${TEST_NAME})
endfunction (add_test_executable)
add_test_executable (t-compile tests/t-compile.c)
add_test_executable (t-dirent tests/t-dirent.c)
add_test_executable (t-scandir tests/t-scandir.c)
add_test_executable (t-unicode tests/t-unicode.c)
add_test_executable (t-cplusplus tests/t-cplusplus.cpp)
deps/dirent/ChangeLog 0 → 100644
浏览文件 @ 438174a6
2018-05-08 Toni Rönkkö
* Version 1.23.2: fixes bad scandir prototype.
2017-08-27 Toni Rönkkö
* Version 1.23: support readdir_r and scandir functions.
2017-07-18 Toni Rönkkö
* Created release branches v1.22 and v1.21 to Git. Published version
1.22 at softagalleria.net.
2016-09-11 Toni Rönkkö
* Version 1.22: added support for CMake. Thanks to Paul Fultz II.
2014-09-25 Toni Rönkkö
* Version 1.21: compiles correctly under Open Watcom. Thanks to
Virgil Banowetz for a patch!
2014-04-07 Toni Rönkkö
* Version 1.20.1: the zip file from the previous version did not open
correctly with Microsoft's compressed folders. Thanks to Alexandre
for info!
2014-03-17 Toni Ronkko
* Version 1.20: dirent.h compiles correctly in 64-bit architecture.
Thanks to Aaron Simmons!
2014-03-03 Toni Ronkko
* Version 1.13.2: define DT_LNK for compatibility with Unix
programs. Thanks to Joel Bruick for suggestion!
2013-01-27 Toni Ronkko
* Version 1.13.1: patch from Edward Berner fixes set_errno() on
Windows NT 4.0.
* Revised wcstombs() and mbstowcs() wrappers to make sure that they do
not write past their target string.
* PATH_MAX from windows.h includes zero terminator so there is no
need to add one extra byte to variables and structures.
2012-12-12 Toni Ronkko
* Version 1.13: use the traditional 8+3 file naming scheme if a file
name cannot be represented in the default ANSI code page. Now
compiles again with MSVC 6.0. Thanks to Konstantin Khomoutov for
testing.
2012-10-01 Toni Ronkko
* Version 1.12.1: renamed wide-character DIR structure _wDIR to
_WDIR (with capital W) in order to maintain compatibility with MingW.
2012-09-30 Toni Ronkko
* Version 1.12: define PATH_MAX and NAME_MAX. Added wide-character
variants _wDIR, _wdirent, _wopendir(), _wreaddir(), _wclosedir() and
_wrewinddir(). Thanks to Edgar Buerkle and Jan Nijtmans for ideas
and code.
* Now avoiding windows.h. This allows dirent.h to be integrated
more easily into programs using winsock. Thanks to Fernando
Azaldegui.
2011-03-15 Toni Ronkko
* Version 1.11: defined FILE_ATTRIBUTE_DEVICE for MSVC 6.0.
2010-08-11 Toni Ronkko
* Version 1.10: added d_type and d_namlen fields to dirent structure.
The former is especially useful for determining whether directory
entry represents a file or a directory. For more information, see
http://www.delorie.com/gnu/docs/glibc/libc_270.html
* Improved conformance to the standards. For example, errno is now
set properly on failure and assert() is never used. Thanks to Peter
Brockam for suggestions.
* Fixed a bug in rewinddir(): when using relative directory names,
change of working directory no longer causes rewinddir() to fail.
2009-12-15 John Cunningham
* Version 1.9: added rewinddir member function
2008-01-18 Toni Ronkko
* Version 1.8: Using FindFirstFileA and WIN32_FIND_DATAA to avoid
converting string between multi-byte and unicode representations.
This makes the code simpler and also allows the code to be compiled
under MingW. Thanks to Azriel Fasten for the suggestion.
2007-03-04 Toni Ronkko
* Bug fix: due to the strncpy_s() function this file only compiled in
Visual Studio 2005. Using the new string functions only when the
compiler version allows.
2006-11-02 Toni Ronkko
* Major update: removed support for Watcom C, MS-DOS and Turbo C to
simplify the file, updated the code to compile cleanly on Visual
Studio 2005 with both unicode and multi-byte character strings,
removed rewinddir() as it had a bug.
2006-08-20 Toni Ronkko
* Removed all remarks about MSVC 1.0, which is antiqued now.
Simplified comments by removing SGML tags.
2002-05-14 Toni Ronkko
* Embedded the function definitions directly to the header so that no
source modules need to be included in the Visual Studio project.
Removed all the dependencies to other projects so that this header
file can be used independently.
1998-05-28 Toni Ronkko
* First version.
deps/dirent/LICENSE 0 → 100644
浏览文件 @ 438174a6
The MIT License (MIT)
Copyright (c) 1998-2019 Toni Ronkko
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
deps/dirent/README.md 0 → 100644
浏览文件 @ 438174a6
# Dirent
Dirent is a C/C++ programming interface that allows programmers to retrieve
information about files and directories under Linux/UNIX. This project
provides Linux compatible Dirent interface for Microsoft Windows.
# Installation
Download the latest Dirent installation package from
[GitHub](https://github.com/tronkko/dirent/releases) and
unpack the installation file with 7-zip, for example. The installation
package contains dirent.h file as well as a few example programs and
tests.
## Install Dirent for All Programs
To make dirent.h available for all C/C++ programs, simply copy the
``include/dirent.h`` file to the system include directory. System include
directory contains header files such as assert.h and windows.h. In Visual
Studio 2008, for example, the system include may be found at
``C:\Program Files\Microsoft Visual Studio 9.0\VC\include``.
Everything you need is included in the single dirent.h file, and you can
start using Dirent immediately -- there is no need to add files to your
Visual Studio project.
## Embed Dirent into Your Own Project
If you wish to distribute dirent.h alongside with your own source code, then
copy ``include/dirent.h`` file to a new sub-directory within your project and
add that directory to include path on Windows while omitting the directory
under Linux/UNIX. This allows your project to be compiled against native
dirent.h on Linux/UNIX while substituting the functionality on Microsoft
Windows.
## Examples
The installation package contains four example programs:
Program | Purpose
-------- | -----------------------------------------------------------------
ls | List files in a directory, e.g. ls "c:\Program Files"
find | Find files in subdirectories, e.g. find "c:\Program Files\CMake"
updatedb | Build database of files in a drive, e.g. updatedb c:\
locate | Locate a file from database, e.g. locate notepad.exe
To build the example programs, first install [CMake](https://cmake.org/).
Then, with CMake installed, open command prompt and create a temporary
directory ``c:\temp\dirent`` for the build files as
```
c:\
mkdir temp
mkdir temp\dirent
cd temp\dirent
```
Generate build files as
```
cmake d:\dirent
```
where ``d:\dirent`` is the root directory of the Dirent package (containing
this README.md and LICENSE file).
Once CMake is finished, open Visual Studio, load the generated dirent.sln file
from the build directory and build the solution. Once the build completes, run
the example programs from the command prompt as
```
cd Debug
ls .
find .
updatedb c:\
locate cmd.exe
```
# Copying
Dirent may be freely distributed under the MIT license. See the
[LICENSE](LICENSE) file for details.
# Alternatives to Dirent
I ported Dirent to Microsoft Windows in 1998 when only a few alternatives
were available. However, the situation has changed since then and nowadays
both [Cygwin](http://www.cygwin.com) and [MingW](http://www.mingw.org)
allow you to compile a great number of UNIX programs in Microsoft Windows.
They both provide a full dirent API as well as many other UNIX APIs. MingW
can even be used for commercial applications!
deps/dirent/distclean.cmake 0 → 100644
浏览文件 @ 438174a6
# Remove CMake generated temporary files
set (cmake_generated
${CMAKE_BINARY_DIR}/ALL_BUILD.vcxproj
${CMAKE_BINARY_DIR}/ALL_BUILD.vcxproj.filters
${CMAKE_BINARY_DIR}/CMakeCache.txt
${CMAKE_BINARY_DIR}/CMakeFiles
${CMAKE_BINARY_DIR}/CTestTestfile.cmake
${CMAKE_BINARY_DIR}/Continuous.vcxproj
${CMAKE_BINARY_DIR}/Continuous.vcxproj.filters
${CMAKE_BINARY_DIR}/DartConfiguration.tcl
${CMAKE_BINARY_DIR}/Debug
${CMAKE_BINARY_DIR}/Experimental.vcxproj
${CMAKE_BINARY_DIR}/Experimental.vcxproj.filters
${CMAKE_BINARY_DIR}/INSTALL.vcxproj
${CMAKE_BINARY_DIR}/INSTALL.vcxproj.filters
${CMAKE_BINARY_DIR}/Makefile
${CMAKE_BINARY_DIR}/Nightly.vcxproj
${CMAKE_BINARY_DIR}/Nightly.vcxproj.filters
${CMAKE_BINARY_DIR}/NightlyMemoryCheck.vcxproj
${CMAKE_BINARY_DIR}/NightlyMemoryCheck.vcxproj.filters
${CMAKE_BINARY_DIR}/RUN_TESTS.vcxproj
${CMAKE_BINARY_DIR}/RUN_TESTS.vcxproj.filters
${CMAKE_BINARY_DIR}/Testing
${CMAKE_BINARY_DIR}/Win32
${CMAKE_BINARY_DIR}/ZERO_CHECK.vcxproj
${CMAKE_BINARY_DIR}/ZERO_CHECK.vcxproj.filters
${CMAKE_BINARY_DIR}/check.vcxproj
${CMAKE_BINARY_DIR}/check.vcxproj.filters
${CMAKE_BINARY_DIR}/cmake_install.cmake
${CMAKE_BINARY_DIR}/dirent.sln
${CMAKE_BINARY_DIR}/distclean.vcxproj
${CMAKE_BINARY_DIR}/distclean.vcxproj.filters
${CMAKE_BINARY_DIR}/find
${CMAKE_BINARY_DIR}/find.dir
${CMAKE_BINARY_DIR}/find.vcxproj
${CMAKE_BINARY_DIR}/find.vcxproj.filters
${CMAKE_BINARY_DIR}/locate
${CMAKE_BINARY_DIR}/locate.dir
${CMAKE_BINARY_DIR}/locate.vcxproj
${CMAKE_BINARY_DIR}/locate.vcxproj.filters
${CMAKE_BINARY_DIR}/ls
${CMAKE_BINARY_DIR}/ls.dir
${CMAKE_BINARY_DIR}/ls.vcxproj
${CMAKE_BINARY_DIR}/ls.vcxproj.filters
${CMAKE_BINARY_DIR}/t-compile
${CMAKE_BINARY_DIR}/t-compile.dir
${CMAKE_BINARY_DIR}/t-compile.vcxproj
${CMAKE_BINARY_DIR}/t-compile.vcxproj.filters
${CMAKE_BINARY_DIR}/t-dirent
${CMAKE_BINARY_DIR}/t-dirent.dir
${CMAKE_BINARY_DIR}/t-dirent.vcxproj
${CMAKE_BINARY_DIR}/t-dirent.vcxproj.filters
${CMAKE_BINARY_DIR}/updatedb
${CMAKE_BINARY_DIR}/updatedb.dir
${CMAKE_BINARY_DIR}/updatedb.vcxproj
${CMAKE_BINARY_DIR}/updatedb.vcxproj.filters
)
foreach (file ${cmake_generated})
if (EXISTS ${file})
file (REMOVE_RECURSE ${file})
endif()
endforeach (file)
deps/dirent/examples/cat.c 0 → 100644
浏览文件 @ 438174a6
/*
* Output contents of a file.
*
* Compile this file with Visual Studio and run the produced command in
* console with a file name argument. For example, command
*
* cat include\dirent.h
*
* will output the dirent.h to screen.
*
* Copyright (C) 1998-2019 Toni Ronkko
* This file is part of dirent. Dirent may be freely distributed
* under the MIT license. For all details and documentation, see
* https://github.com/tronkko/dirent
*/
#define _CRT_SECURE_NO_WARNINGS
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <dirent.h>
#include <errno.h>
#include <locale.h>
static void output_file (const char *fn);
int
main(
int argc, char *argv[])
{
int i;
/* Select default locale */
setlocale (LC_ALL, "");
/* Require at least one file */
if (argc == 1) {
fprintf (stderr, "Usage: cat filename\n");
return EXIT_FAILURE;
}
/* For each file name argument in command line */
i = 1;
while (i < argc) {
output_file (argv[i]);
i++;
}
return EXIT_SUCCESS;
}
/*
* Output file to screen
*/
static void
output_file(
const char *fn)
{
FILE *fp;
/* Open file */
fp = fopen (fn, "r");
if (fp != NULL) {
size_t n;
char buffer[4096];
/* Output file to screen */
do {
/* Read some bytes from file */
n = fread (buffer, 1, 4096, fp);
/* Output bytes to screen */
fwrite (buffer, 1, n, stdout);
} while (n != 0);
/* Close file */
fclose (fp);
} else {
/* Could not open directory */
fprintf (stderr, "Cannot open %s (%s)\n", fn, strerror (errno));
exit (EXIT_FAILURE);
}
}
deps/dirent/examples/find.c 0 → 100644
浏览文件 @ 438174a6
/*
* An example demonstrating recursive directory traversal.
*
* Compile this file with Visual Studio and run the produced command in
* console with a directory name argument. For example, command
*
* find "C:\Program Files"
*
* will output thousands of file names such as
*
* c:\Program Files/7-Zip/7-zip.chm
* c:\Program Files/7-Zip/7-zip.dll
* c:\Program Files/7-Zip/7z.dll
* c:\Program Files/Adobe/Reader 10.0/Reader/logsession.dll
* c:\Program Files/Adobe/Reader 10.0/Reader/LogTransport2.exe
* c:\Program Files/Windows NT/Accessories/wordpad.exe
* c:\Program Files/Windows NT/Accessories/write.wpc
*
* The find command provided by this file is only an example: the command does
* not provide options to restrict the output to certain files as the Linux
* version does.
*
* Copyright (C) 1998-2019 Toni Ronkko
* This file is part of dirent. Dirent may be freely distributed
* under the MIT license. For all details and documentation, see
* https://github.com/tronkko/dirent
*/
#define _CRT_SECURE_NO_WARNINGS
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <dirent.h>
#include <errno.h>
#include <locale.h>
static int find_directory (const char *dirname);
int
main(
int argc, char *argv[])
{
int i;
int ok;
/* Select default locale */
setlocale (LC_ALL, "");
/* For each directory in command line */
i = 1;
while (i < argc) {
ok = find_directory (argv[i]);
if (!ok) {
exit (EXIT_FAILURE);
}
i++;
}
/* List current working directory if no arguments on command line */
if (argc == 1) {
find_directory (".");
}
return EXIT_SUCCESS;
}
/* Find files and subdirectories recursively */
static int
find_directory(
const char *dirname)
{
DIR *dir;
char buffer[PATH_MAX + 2];
char *p = buffer;
const char *src;
char *end = &buffer[PATH_MAX];
int ok;
/* Copy directory name to buffer */
src = dirname;
while (p < end && *src != '\0') {
*p++ = *src++;
}
*p = '\0';
/* Open directory stream */
dir = opendir (dirname);
if (dir != NULL) {
struct dirent *ent;
/* Print all files and directories within the directory */
while ((ent = readdir (dir)) != NULL) {
char *q = p;
char c;
/* Get final character of directory name */
if (buffer < q) {
c = q[-1];
} else {
c = ':';
}
/* Append directory separator if not already there */
if (c != ':' && c != '/' && c != '\\') {
*q++ = '/';
}
/* Append file name */
src = ent->d_name;
while (q < end && *src != '\0') {
*q++ = *src++;
}
*q = '\0';
/* Decide what to do with the directory entry */
switch (ent->d_type) {
case DT_LNK:
case DT_REG:
/* Output file name with directory */
printf ("%s\n", buffer);
break;
case DT_DIR:
/* Scan sub-directory recursively */
if (strcmp (ent->d_name, ".") != 0
&& strcmp (ent->d_name, "..") != 0) {
find_directory (buffer);
}
break;
default:
/* Ignore device entries */
/*NOP*/;
}
}
closedir (dir);
ok = 1;
} else {
/* Could not open directory */
fprintf (stderr, "Cannot open %s (%s)\n", dirname, strerror (errno));
ok = 0;
}
return ok;
}
deps/dirent/examples/locate.c 0 → 100644
浏览文件 @ 438174a6
/*
* A file look-up utility to complement updatedb
*
* Compile and run updatedb command first to create locate.db file. Then,
* compile this program with Visual Studio and run the program in console with
* a file name argument. For example, the command
*
* locate autoexec
*
* might output something like
*
* c:/AUTOEXEC.BAT
* c:/WINDOWS/repair/autoexec.nt
* c:/WINDOWS/system32/AUTOEXEC.NT
*
* Be ware that this file uses wide-character API which is not compatible
* with Linux or other major Unixes.
*
* Copyright (C) 1998-2019 Toni Ronkko
* This file is part of dirent. Dirent may be freely distributed
* under the MIT license. For all details and documentation, see
* https://github.com/tronkko/dirent
*/
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <wchar.h>
#ifdef WIN32
# include <io.h>
# include <fcntl.h>
#endif
#include <dirent.h>
/* File name and location of database file */
#define DB_LOCATION L"locate.db"
/* Forward-decl */
static int db_locate (const wchar_t *pattern);
static int db_match (const wchar_t *fn, const wchar_t *pattern);
static void db_open (void);
static void db_close (void);
static int db_read (wchar_t *buffer, size_t max);
/* Module local variables */
static FILE *db = NULL;
int
main(
int argc, char *argv[])
{
#ifdef WIN32
int i;
/* Prepare for unicode output */
_setmode (_fileno (stdout), _O_U16TEXT);
/* For each pattern in command line */
i = 1;
while (i < argc) {
wchar_t buffer[PATH_MAX + 1];
errno_t error;
size_t n;
int count = 0;
/* Convert ith argument to wide-character string */
error = mbstowcs_s (&n, buffer, PATH_MAX, argv[i], _TRUNCATE);
if (!error) {
/* Find files matching pattern */
count = db_locate (buffer);
/* Output warning if string is not found */
if (count == 0) {
wprintf (L"%s not found\n", buffer);
}
}
i++;
}
if (argc < 2) {
wprintf (L"Usage: locate pattern\n");
exit (EXIT_FAILURE);
}
#else
printf ("locate only works on Microsoft Windows\n");
#endif
return EXIT_SUCCESS;
}
/* Match pattern against files in locate.db file */
static int
db_locate(
const wchar_t *pattern)
{
int count = 0;
#ifdef WIN32
wchar_t buffer[PATH_MAX + 1];
/* Open locate.db for read */
db_open ();
/* Read one directory and file name at a time from database file */
while (db_read (buffer, PATH_MAX)) {
/* See if file name in buffer matches the search pattern */
if (db_match (buffer, pattern)) {
/* Match found => output file name and path */
wprintf (L"%s\n", buffer);
count++;
}
}
db_close ();
#endif
return count;
}
/* Match pattern against file name */
static int
db_match(
const wchar_t *fn, const wchar_t *pattern)
{
int found = 0;
#ifdef WIN32
wchar_t *p;
wchar_t base[PATH_MAX + 1];
wchar_t patt[PATH_MAX + 1];
int i;
int done = 0;
/* Locate zero-terminator from fn */
p = wcschr (fn, '\0');
/* Find base name from buffer */
while (fn < p && !done) {
switch (p[-1]) {
case ':':
case '/':
case '\\':
/* Final path separator found */
done = 1;
break;
default:
/* No path separator yet */
p--;
}
}
/* Convert base name to lower case */
i = 0;
while (i < PATH_MAX && p[i] != '\0') {
base[i] = towlower (p[i]);
i++;
}
base[i] = '\0';
/* Convert search pattern to lower case */
i = 0;
while (i < PATH_MAX && pattern[i] != '\0') {
patt[i] = towlower (pattern[i]);
i++;
}
patt[i] = '\0';
/* See if file name matches pattern */
if (wcsstr (base, patt) != NULL) {
found = 1;
} else {
found = 0;
}
#endif
return found;
}
/*
* Read line from locate.db. This function is same as fgetws() except
* that new-line at the end of line is not included.
*/
static int
db_read(
wchar_t *buffer, size_t max)
{
int ok = 0;
#ifdef WIN32
size_t i = 0;
wchar_t c;
int done = 0;
do {
/* Read wide-character from stream */
if (db) {
c = fgetwc (db);
} else {
wprintf (L"Database not open\n");
exit (EXIT_SUCCESS);
}
/* Determine how to process character */
switch (c) {
case '\r':
/* Ignore, should be handled by run-time libraries */
/*NOP*/;
break;
case '\n':
/* End of string => return file name and true */
done = 1;
ok = 1;
break;
case /*EOF*/WEOF:
/* End of file */
done = 1;
if (i == 0) {
/* No data in buffer => return false to indicate EOF */
ok = 0;
} else {
/* Data in buffer => return true */
ok = 1;
}
break;
default:
/* Store character */
if (i < max - 1) {
buffer[i++] = c;
} else {
wprintf (L"Buffer too small");
exit (EXIT_FAILURE);
}
}
} while (!done);
/* Zero-terminate buffer */
buffer[i] = '\0';
#endif
return ok;
}
/* Open database file locate.db */
static void
db_open(
void)
{
#ifdef WIN32
if (db == NULL) {
errno_t error;
/* Open file for writing */
error = _wfopen_s (&db, DB_LOCATION, L"rt, ccs=UNICODE");
if (error) {
wprintf (L"Cannot open %s\n", DB_LOCATION);
exit (EXIT_FAILURE);
}
}
#endif
}
/* Close database file */
static void
db_close(
void)
{
if (db) {
fclose (db);
db = NULL;
}
}
deps/dirent/examples/ls.c 0 → 100644
浏览文件 @ 438174a6
/*
* An example demonstrating basic directory listing.
*
* Compile this file with Visual Studio and run the produced command in
* console with a directory name argument. For example, command
*
* ls "c:\Program Files"
*
* might output something like
*
* ./
* ../
* 7-Zip/
* Internet Explorer/
* Microsoft Visual Studio 9.0/
* Microsoft.NET/
* Mozilla Firefox/
*
* The ls command provided by this file is only an example: the command does
* not have any fancy options like "ls -al" in Linux and the command does not
* support file name matching like "ls *.c".
*
* Copyright (C) 1998-2019 Toni Ronkko
* This file is part of dirent. Dirent may be freely distributed
* under the MIT license. For all details and documentation, see
* https://github.com/tronkko/dirent
*/
#define _CRT_SECURE_NO_WARNINGS
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <dirent.h>
#include <errno.h>
#include <locale.h>
static void list_directory (const char *dirname);
int
main(
int argc, char *argv[])
{
int i;
/* Select default locale */
setlocale (LC_ALL, "");
/* For each directory in command line */
i = 1;
while (i < argc) {
list_directory (argv[i]);
i++;
}
/* List current working directory if no arguments on command line */
if (argc == 1) {
list_directory (".");
}
return EXIT_SUCCESS;
}
/*
* List files and directories within a directory.
*/
static void
list_directory(
const char *dirname)
{
DIR *dir;
struct dirent *ent;
/* Open directory stream */
dir = opendir (dirname);
if (dir != NULL) {
/* Print all files and directories within the directory */
while ((ent = readdir (dir)) != NULL) {
switch (ent->d_type) {
case DT_REG:
printf ("%s\n", ent->d_name);
break;
case DT_DIR:
printf ("%s/\n", ent->d_name);
break;
case DT_LNK:
printf ("%s@\n", ent->d_name);
break;
default:
printf ("%s*\n", ent->d_name);
}
}
closedir (dir);
} else {
/* Could not open directory */
fprintf (stderr, "Cannot open %s (%s)\n", dirname, strerror (errno));
exit (EXIT_FAILURE);
}
}
deps/dirent/examples/scandir.c 0 → 100644
浏览文件 @ 438174a6
/*
* Example program demonstrating the use of scandir function.
*
* Compile this file with Visual Studio and run the produced command in
* console with a directory name argument. For example, command
*
* scandir "c:\Program Files"
*
* might output something like
*
* ./
* ../
* 7-Zip/
* Internet Explorer/
* Microsoft Visual Studio 9.0/
* Microsoft.NET/
* Mozilla Firefox/
*
* Copyright (C) 1998-2019 Toni Ronkko
* This file is part of dirent. Dirent may be freely distributed
* under the MIT license. For all details and documentation, see
* https://github.com/tronkko/dirent
*/
#define _CRT_SECURE_NO_WARNINGS
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <dirent.h>
#include <errno.h>
#include <locale.h>
static void list_directory (const char *dirname);
int
main(
int argc, char *argv[])
{
int i;
/* Select default locale */
setlocale (LC_ALL, "");
/* For each directory in command line */
i = 1;
while (i < argc) {
list_directory (argv[i]);
i++;
}
/* List current working directory if no arguments on command line */
if (argc == 1) {
list_directory (".");
}
return EXIT_SUCCESS;
}
/*
* List files and directories within a directory.
*/
static void
list_directory(
const char *dirname)
{
struct dirent **files;
int i;
int n;
/* Scan files in directory */
n = scandir (dirname, &files, NULL, alphasort);
if (n >= 0) {
/* Loop through file names */
for (i = 0; i < n; i++) {
struct dirent *ent;
/* Get pointer to file entry */
ent = files[i];
/* Output file name */
switch (ent->d_type) {
case DT_REG:
printf ("%s\n", ent->d_name);
break;
case DT_DIR:
printf ("%s/\n", ent->d_name);
break;
case DT_LNK:
printf ("%s@\n", ent->d_name);
break;
default:
printf ("%s*\n", ent->d_name);
}
}
/* Release file names */
for (i = 0; i < n; i++) {
free (files[i]);
}
free (files);
} else {
fprintf (stderr, "Cannot open %s (%s)\n", dirname, strerror (errno));
exit (EXIT_FAILURE);
}
}
deps/dirent/examples/updatedb.c 0 → 100644
浏览文件 @ 438174a6
/*
* An example demonstrating wide-character functions
*
* Compile this file with Visual Studio and run the produced command in
* console with a directory name argument. For example, command
*
* updatedb C:\
*
* will produce the file locate.db with one file name per line such as
*
* c:\Program Files/7-Zip/7-zip.chm
* c:\Program Files/7-Zip/7-zip.dll
* c:\Program Files/7-Zip/7z.dll
* c:\Program Files/Adobe/Reader 10.0/Reader/logsession.dll
* c:\Program Files/Adobe/Reader 10.0/Reader/LogTransport2.exe
* c:\Program Files/Windows NT/Accessories/wordpad.exe
* c:\Program Files/Windows NT/Accessories/write.wpc
*
* Be ware that this file uses wide-character API which is not compatible
* with Linux or other major Unixes.
*
* Copyright (C) 1998-2019 Toni Ronkko
* This file is part of dirent. Dirent may be freely distributed
* under the MIT license. For all details and documentation, see
* https://github.com/tronkko/dirent
*/
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <wchar.h>
#ifdef WIN32
# include <io.h>
# include <fcntl.h>
#endif
#include <dirent.h>
/* File name and location of database file */
#define DB_LOCATION L"locate.db"
/* Forward-decl */
static int update_directory (const wchar_t *dirname);
static void db_open (void);
static void db_close (void);
static void db_store (const wchar_t *dirname);
/* Module local variables */
static FILE *db = NULL;
int
main(
int argc, char *argv[])
{
#ifdef WIN32
int i;
int ok;
/* Prepare for unicode output */
_setmode (_fileno (stdout), _O_U16TEXT);
/* Open locate.db */
db_open ();
/* For each directory in command line */
i = 1;
while (i < argc) {
wchar_t buffer[PATH_MAX + 1];
errno_t error;
size_t n;
/* Convert ith argument to wide-character string */
error = mbstowcs_s (&n, buffer, PATH_MAX, argv[i], _TRUNCATE);
if (!error) {
/* Scan directory for files */
ok = update_directory (buffer);
if (!ok) {
wprintf (L"Cannot open directory %s\n", buffer);
exit (EXIT_FAILURE);
}
}
i++;
}
/* Use current working directory if no arguments on command line */
if (argc == 1) {
update_directory (L".");
}
db_close ();
#else
printf ("updatedb only works on Microsoft Windows\n");
#endif
return EXIT_SUCCESS;
}
/* Find files recursively */
static int
update_directory(
const wchar_t *dirname)
{
int ok = 0;
#ifdef WIN32
_WDIR *dir;
wchar_t buffer[PATH_MAX + 2];
wchar_t *p = buffer;
const wchar_t *src;
wchar_t *end = &buffer[PATH_MAX];
/* Copy directory name to buffer */
src = dirname;
while (p < end && *src != '\0') {
*p++ = *src++;
}
*p = '\0';
/* Open directory stream */
dir = _wopendir (dirname);
if (dir != NULL) {
struct _wdirent *ent;
/* Print all files and directories within the directory */
while ((ent = _wreaddir (dir)) != NULL) {
wchar_t *q = p;
wchar_t c;
/* Get final character of directory name */
if (buffer < q) {
c = q[-1];
} else {
c = ':';
}
/* Append directory separator if not already there */
if (c != ':' && c != '/' && c != '\\') {
*q++ = '/';
}
/* Append file name */
src = ent->d_name;
while (q < end && *src != '\0') {
*q++ = *src++;
}
*q = '\0';
/* Decide what to do with the directory entry */
switch (ent->d_type) {
case DT_REG:
/* Store file name */
db_store (buffer);
break;
case DT_DIR:
/* Scan sub-directory recursively */
if (wcscmp (ent->d_name, L".") != 0
&& wcscmp (ent->d_name, L"..") != 0) {
update_directory (buffer);
}
break;
default:
/* Do not device entries */
/*NOP*/;
}
}
wclosedir (dir);
ok = 1;
} else {
/* Cannot open directory */
ok = 0;
}
#endif
return ok;
}
/* Store file name to locate.db */
static void
db_store(
const wchar_t *dirname)
{
#ifdef WIN32
if (db) {
/* Output line to file */
fwprintf (db, L"%s\n", dirname);
} else {
wprintf (L"Database not open\n");
exit (EXIT_FAILURE);
}
#endif
}
/* Open database file locate.db */
static void
db_open(
void)
{
#ifdef WIN32
if (db == NULL) {
errno_t error;
/* Open file for writing */
error = _wfopen_s (&db, DB_LOCATION, L"wt, ccs=UNICODE");
if (error) {
wprintf (L"Cannot open %s\n", DB_LOCATION);
exit (EXIT_FAILURE);
}
}
#endif
}
/* Close database file */
static void
db_close(
void)
{
if (db) {
fclose (db);
db = NULL;
}
}
deps/dirent/include/dirent.h 0 → 100644
浏览文件 @ 438174a6
/*
* Dirent interface for Microsoft Visual Studio
*
* Copyright (C) 1998-2019 Toni Ronkko
* This file is part of dirent. Dirent may be freely distributed
* under the MIT license. For all details and documentation, see
* https://github.com/tronkko/dirent
*/
#ifndef DIRENT_H
#define DIRENT_H
/* Hide warnings about unreferenced local functions */
#if defined(__clang__)
# pragma clang diagnostic ignored "-Wunused-function"
#elif defined(_MSC_VER)
# pragma warning(disable:4505)
#elif defined(__GNUC__)
# pragma GCC diagnostic ignored "-Wunused-function"
#endif
/*
* Include windows.h without Windows Sockets 1.1 to prevent conflicts with
* Windows Sockets 2.0.
*/
#ifndef WIN32_LEAN_AND_MEAN
# define WIN32_LEAN_AND_MEAN
#endif
#include <windows.h>
#include <stdio.h>
#include <stdarg.h>
#include <wchar.h>
#include <string.h>
#include <stdlib.h>
#include <malloc.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <errno.h>
/* Indicates that d_type field is available in dirent structure */
#define _DIRENT_HAVE_D_TYPE
/* Indicates that d_namlen field is available in dirent structure */
#define _DIRENT_HAVE_D_NAMLEN
/* Entries missing from MSVC 6.0 */
#if !defined(FILE_ATTRIBUTE_DEVICE)
# define FILE_ATTRIBUTE_DEVICE 0x40
#endif
/* File type and permission flags for stat(), general mask */
#if !defined(S_IFMT)
# define S_IFMT _S_IFMT
#endif
/* Directory bit */
#if !defined(S_IFDIR)
# define S_IFDIR _S_IFDIR
#endif
/* Character device bit */
#if !defined(S_IFCHR)
# define S_IFCHR _S_IFCHR
#endif
/* Pipe bit */
#if !defined(S_IFFIFO)
# define S_IFFIFO _S_IFFIFO
#endif
/* Regular file bit */
#if !defined(S_IFREG)
# define S_IFREG _S_IFREG
#endif
/* Read permission */
#if !defined(S_IREAD)
# define S_IREAD _S_IREAD
#endif
/* Write permission */
#if !defined(S_IWRITE)
# define S_IWRITE _S_IWRITE
#endif
/* Execute permission */
#if !defined(S_IEXEC)
# define S_IEXEC _S_IEXEC
#endif
/* Pipe */
#if !defined(S_IFIFO)
# define S_IFIFO _S_IFIFO
#endif
/* Block device */
#if !defined(S_IFBLK)
# define S_IFBLK 0
#endif
/* Link */
#if !defined(S_IFLNK)
# define S_IFLNK 0
#endif
/* Socket */
#if !defined(S_IFSOCK)
# define S_IFSOCK 0
#endif
/* Read user permission */
#if !defined(S_IRUSR)
# define S_IRUSR S_IREAD
#endif
/* Write user permission */
#if !defined(S_IWUSR)
# define S_IWUSR S_IWRITE
#endif
/* Execute user permission */
#if !defined(S_IXUSR)
# define S_IXUSR 0
#endif
/* Read group permission */
#if !defined(S_IRGRP)
# define S_IRGRP 0
#endif
/* Write group permission */
#if !defined(S_IWGRP)
# define S_IWGRP 0
#endif
/* Execute group permission */
#if !defined(S_IXGRP)
# define S_IXGRP 0
#endif
/* Read others permission */
#if !defined(S_IROTH)
# define S_IROTH 0
#endif
/* Write others permission */
#if !defined(S_IWOTH)
# define S_IWOTH 0
#endif
/* Execute others permission */
#if !defined(S_IXOTH)
# define S_IXOTH 0
#endif
/* Maximum length of file name */
#if !defined(PATH_MAX)
# define PATH_MAX MAX_PATH
#endif
#if !defined(FILENAME_MAX)
# define FILENAME_MAX MAX_PATH
#endif
#if !defined(NAME_MAX)
# define NAME_MAX FILENAME_MAX
#endif
/* File type flags for d_type */
#define DT_UNKNOWN 0
#define DT_REG S_IFREG
#define DT_DIR S_IFDIR
#define DT_FIFO S_IFIFO
#define DT_SOCK S_IFSOCK
#define DT_CHR S_IFCHR
#define DT_BLK S_IFBLK
#define DT_LNK S_IFLNK
/* Macros for converting between st_mode and d_type */
#define IFTODT(mode) ((mode) & S_IFMT)
#define DTTOIF(type) (type)
/*
* File type macros. Note that block devices, sockets and links cannot be
* distinguished on Windows and the macros S_ISBLK, S_ISSOCK and S_ISLNK are
* only defined for compatibility. These macros should always return false
* on Windows.
*/
#if !defined(S_ISFIFO)
# define S_ISFIFO(mode) (((mode) & S_IFMT) == S_IFIFO)
#endif
#if !defined(S_ISDIR)
# define S_ISDIR(mode) (((mode) & S_IFMT) == S_IFDIR)
#endif
#if !defined(S_ISREG)
# define S_ISREG(mode) (((mode) & S_IFMT) == S_IFREG)
#endif
#if !defined(S_ISLNK)
# define S_ISLNK(mode) (((mode) & S_IFMT) == S_IFLNK)
#endif
#if !defined(S_ISSOCK)
# define S_ISSOCK(mode) (((mode) & S_IFMT) == S_IFSOCK)
#endif
#if !defined(S_ISCHR)
# define S_ISCHR(mode) (((mode) & S_IFMT) == S_IFCHR)
#endif
#if !defined(S_ISBLK)
# define S_ISBLK(mode) (((mode) & S_IFMT) == S_IFBLK)
#endif
/* Return the exact length of the file name without zero terminator */
#define _D_EXACT_NAMLEN(p) ((p)->d_namlen)
/* Return the maximum size of a file name */
#define _D_ALLOC_NAMLEN(p) ((PATH_MAX)+1)
#ifdef __cplusplus
extern "C" {
#endif
/* Wide-character version */
struct _wdirent {
/* Always zero */
long d_ino;
/* File position within stream */
long d_off;
/* Structure size */
unsigned short d_reclen;
/* Length of name without \0 */
size_t d_namlen;
/* File type */
int d_type;
/* File name */
wchar_t d_name[PATH_MAX+1];
};
typedef struct _wdirent _wdirent;
struct _WDIR {
/* Current directory entry */
struct _wdirent ent;
/* Private file data */
WIN32_FIND_DATAW data;
/* True if data is valid */
int cached;
/* Win32 search handle */
HANDLE handle;
/* Initial directory name */
wchar_t *patt;
};
typedef struct _WDIR _WDIR;
/* Multi-byte character version */
struct dirent {
/* Always zero */
long d_ino;
/* File position within stream */
long d_off;
/* Structure size */
unsigned short d_reclen;
/* Length of name without \0 */
size_t d_namlen;
/* File type */
int d_type;
/* File name */
char d_name[PATH_MAX+1];
};
typedef struct dirent dirent;
struct DIR {
struct dirent ent;
struct _WDIR *wdirp;
};
typedef struct DIR DIR;
/* Dirent functions */
static DIR *opendir (const char *dirname);
static _WDIR *_wopendir (const wchar_t *dirname);
static struct dirent *readdir (DIR *dirp);
static struct _wdirent *_wreaddir (_WDIR *dirp);
static int readdir_r(
DIR *dirp, struct dirent *entry, struct dirent **result);
static int _wreaddir_r(
_WDIR *dirp, struct _wdirent *entry, struct _wdirent **result);
static int closedir (DIR *dirp);
static int _wclosedir (_WDIR *dirp);
static void rewinddir (DIR* dirp);
static void _wrewinddir (_WDIR* dirp);
static int scandir (const char *dirname, struct dirent ***namelist,
int (*filter)(const struct dirent*),
int (*compare)(const struct dirent**, const struct dirent**));
static int alphasort (const struct dirent **a, const struct dirent **b);
static int versionsort (const struct dirent **a, const struct dirent **b);
/* For compatibility with Symbian */
#define wdirent _wdirent
#define WDIR _WDIR
#define wopendir _wopendir
#define wreaddir _wreaddir
#define wclosedir _wclosedir
#define wrewinddir _wrewinddir
/* Internal utility functions */
static WIN32_FIND_DATAW *dirent_first (_WDIR *dirp);
static WIN32_FIND_DATAW *dirent_next (_WDIR *dirp);
static int dirent_mbstowcs_s(
size_t *pReturnValue,
wchar_t *wcstr,
size_t sizeInWords,
const char *mbstr,
size_t count);
static int dirent_wcstombs_s(
size_t *pReturnValue,
char *mbstr,
size_t sizeInBytes,
const wchar_t *wcstr,
size_t count);
static void dirent_set_errno (int error);
/*
* Open directory stream DIRNAME for read and return a pointer to the
* internal working area that is used to retrieve individual directory
* entries.
*/
static _WDIR*
_wopendir(
const wchar_t *dirname)
{
_WDIR *dirp;
DWORD n;
wchar_t *p;
/* Must have directory name */
if (dirname == NULL || dirname[0] == '\0') {
dirent_set_errno (ENOENT);
return NULL;
}
/* Allocate new _WDIR structure */
dirp = (_WDIR*) malloc (sizeof (struct _WDIR));
if (!dirp) {
return NULL;
}
/* Reset _WDIR structure */
dirp->handle = INVALID_HANDLE_VALUE;
dirp->patt = NULL;
dirp->cached = 0;
/*
* Compute the length of full path plus zero terminator
*
* Note that on WinRT there's no way to convert relative paths
* into absolute paths, so just assume it is an absolute path.
*/
#if WINAPI_FAMILY_PARTITION(WINAPI_PARTITION_DESKTOP)
/* Desktop */
n = GetFullPathNameW (dirname, 0, NULL, NULL);
#else
/* WinRT */
n = wcslen (dirname);
#endif
/* Allocate room for absolute directory name and search pattern */
dirp->patt = (wchar_t*) malloc (sizeof (wchar_t) * n + 16);
if (dirp->patt == NULL) {
goto exit_closedir;
}
/*
* Convert relative directory name to an absolute one. This
* allows rewinddir() to function correctly even when current
* working directory is changed between opendir() and rewinddir().
*
* Note that on WinRT there's no way to convert relative paths
* into absolute paths, so just assume it is an absolute path.
*/
#if WINAPI_FAMILY_PARTITION(WINAPI_PARTITION_DESKTOP)
/* Desktop */
n = GetFullPathNameW (dirname, n, dirp->patt, NULL);
if (n <= 0) {
goto exit_closedir;
}
#else
/* WinRT */
wcsncpy_s (dirp->patt, n+1, dirname, n);
#endif
/* Append search pattern \* to the directory name */
p = dirp->patt + n;
switch (p[-1]) {
case '\\':
case '/':
case ':':
/* Directory ends in path separator, e.g. c:\temp\ */
/*NOP*/;
break;
default:
/* Directory name doesn't end in path separator */
*p++ = '\\';
}
*p++ = '*';
*p = '\0';
/* Open directory stream and retrieve the first entry */
if (!dirent_first (dirp)) {
goto exit_closedir;
}
/* Success */
return dirp;
/* Failure */
exit_closedir:
_wclosedir (dirp);
return NULL;
}
/*
* Read next directory entry.
*
* Returns pointer to static directory entry which may be overwritten by
* subsequent calls to _wreaddir().
*/
static struct _wdirent*
_wreaddir(
_WDIR *dirp)
{
struct _wdirent *entry;
/*
* Read directory entry to buffer. We can safely ignore the return value
* as entry will be set to NULL in case of error.
*/
(void) _wreaddir_r (dirp, &dirp->ent, &entry);
/* Return pointer to statically allocated directory entry */
return entry;
}
/*
* Read next directory entry.
*
* Returns zero on success. If end of directory stream is reached, then sets
* result to NULL and returns zero.
*/
static int
_wreaddir_r(
_WDIR *dirp,
struct _wdirent *entry,
struct _wdirent **result)
{
WIN32_FIND_DATAW *datap;
/* Read next directory entry */
datap = dirent_next (dirp);
if (datap) {
size_t n;
DWORD attr;
/*
* Copy file name as wide-character string. If the file name is too
* long to fit in to the destination buffer, then truncate file name
* to PATH_MAX characters and zero-terminate the buffer.
*/
n = 0;
while (n < PATH_MAX && datap->cFileName[n] != 0) {
entry->d_name[n] = datap->cFileName[n];
n++;
}
entry->d_name[n] = 0;
/* Length of file name excluding zero terminator */
entry->d_namlen = n;
/* File type */
attr = datap->dwFileAttributes;
if ((attr & FILE_ATTRIBUTE_DEVICE) != 0) {
entry->d_type = DT_CHR;
} else if ((attr & FILE_ATTRIBUTE_DIRECTORY) != 0) {
entry->d_type = DT_DIR;
} else {
entry->d_type = DT_REG;
}
/* Reset dummy fields */
entry->d_ino = 0;
entry->d_off = 0;
entry->d_reclen = sizeof (struct _wdirent);
/* Set result address */
*result = entry;
} else {
/* Return NULL to indicate end of directory */
*result = NULL;
}
return /*OK*/0;
}
/*
* Close directory stream opened by opendir() function. This invalidates the
* DIR structure as well as any directory entry read previously by
* _wreaddir().
*/
static int
_wclosedir(
_WDIR *dirp)
{
int ok;
if (dirp) {
/* Release search handle */
if (dirp->handle != INVALID_HANDLE_VALUE) {
FindClose (dirp->handle);
}
/* Release search pattern */
free (dirp->patt);
/* Release directory structure */
free (dirp);
ok = /*success*/0;
} else {
/* Invalid directory stream */
dirent_set_errno (EBADF);
ok = /*failure*/-1;
}
return ok;
}
/*
* Rewind directory stream such that _wreaddir() returns the very first
* file name again.
*/
static void
_wrewinddir(
_WDIR* dirp)
{
if (dirp) {
/* Release existing search handle */
if (dirp->handle != INVALID_HANDLE_VALUE) {
FindClose (dirp->handle);
}
/* Open new search handle */
dirent_first (dirp);
}
}
/* Get first directory entry (internal) */
static WIN32_FIND_DATAW*
dirent_first(
_WDIR *dirp)
{
WIN32_FIND_DATAW *datap;
DWORD error;
/* Open directory and retrieve the first entry */
dirp->handle = FindFirstFileExW(
dirp->patt, FindExInfoStandard, &dirp->data,
FindExSearchNameMatch, NULL, 0);
if (dirp->handle != INVALID_HANDLE_VALUE) {
/* a directory entry is now waiting in memory */
datap = &dirp->data;
dirp->cached = 1;
} else {
/* Failed to open directory: no directory entry in memory */
dirp->cached = 0;
datap = NULL;
/* Set error code */
error = GetLastError ();
switch (error) {
case ERROR_ACCESS_DENIED:
/* No read access to directory */
dirent_set_errno (EACCES);
break;
case ERROR_DIRECTORY:
/* Directory name is invalid */
dirent_set_errno (ENOTDIR);
break;
case ERROR_PATH_NOT_FOUND:
default:
/* Cannot find the file */
dirent_set_errno (ENOENT);
}
}
return datap;
}
/*
* Get next directory entry (internal).
*
* Returns
*/
static WIN32_FIND_DATAW*
dirent_next(
_WDIR *dirp)
{
WIN32_FIND_DATAW *p;
/* Get next directory entry */
if (dirp->cached != 0) {
/* A valid directory entry already in memory */
p = &dirp->data;
dirp->cached = 0;
} else if (dirp->handle != INVALID_HANDLE_VALUE) {
/* Get the next directory entry from stream */
if (FindNextFileW (dirp->handle, &dirp->data) != FALSE) {
/* Got a file */
p = &dirp->data;
} else {
/* The very last entry has been processed or an error occurred */
FindClose (dirp->handle);
dirp->handle = INVALID_HANDLE_VALUE;
p = NULL;
}
} else {
/* End of directory stream reached */
p = NULL;
}
return p;
}
/*
* Open directory stream using plain old C-string.
*/
static DIR*
opendir(
const char *dirname)
{
struct DIR *dirp;
/* Must have directory name */
if (dirname == NULL || dirname[0] == '\0') {
dirent_set_errno (ENOENT);
return NULL;
}
/* Allocate memory for DIR structure */
dirp = (DIR*) malloc (sizeof (struct DIR));
if (!dirp) {
return NULL;
}
{
int error;
wchar_t wname[PATH_MAX + 1];
size_t n;
/* Convert directory name to wide-character string */
error = dirent_mbstowcs_s(
&n, wname, PATH_MAX + 1, dirname, PATH_MAX + 1);
if (error) {
/*
* Cannot convert file name to wide-character string. This
* occurs if the string contains invalid multi-byte sequences or
* the output buffer is too small to contain the resulting
* string.
*/
goto exit_free;
}
/* Open directory stream using wide-character name */
dirp->wdirp = _wopendir (wname);
if (!dirp->wdirp) {
goto exit_free;
}
}
/* Success */
return dirp;
/* Failure */
exit_free:
free (dirp);
return NULL;
}
/*
* Read next directory entry.
*/
static struct dirent*
readdir(
DIR *dirp)
{
struct dirent *entry;
/*
* Read directory entry to buffer. We can safely ignore the return value
* as entry will be set to NULL in case of error.
*/
(void) readdir_r (dirp, &dirp->ent, &entry);
/* Return pointer to statically allocated directory entry */
return entry;
}
/*
* Read next directory entry into called-allocated buffer.
*
* Returns zero on success. If the end of directory stream is reached, then
* sets result to NULL and returns zero.
*/
static int
readdir_r(
DIR *dirp,
struct dirent *entry,
struct dirent **result)
{
WIN32_FIND_DATAW *datap;
/* Read next directory entry */
datap = dirent_next (dirp->wdirp);
if (datap) {
size_t n;
int error;
/* Attempt to convert file name to multi-byte string */
error = dirent_wcstombs_s(
&n, entry->d_name, PATH_MAX + 1, datap->cFileName, PATH_MAX + 1);
/*
* If the file name cannot be represented by a multi-byte string,
* then attempt to use old 8+3 file name. This allows traditional
* Unix-code to access some file names despite of unicode
* characters, although file names may seem unfamiliar to the user.
*
* Be ware that the code below cannot come up with a short file
* name unless the file system provides one. At least
* VirtualBox shared folders fail to do this.
*/
if (error && datap->cAlternateFileName[0] != '\0') {
error = dirent_wcstombs_s(
&n, entry->d_name, PATH_MAX + 1,
datap->cAlternateFileName, PATH_MAX + 1);
}
if (!error) {
DWORD attr;
/* Length of file name excluding zero terminator */
entry->d_namlen = n - 1;
/* File attributes */
attr = datap->dwFileAttributes;
if ((attr & FILE_ATTRIBUTE_DEVICE) != 0) {
entry->d_type = DT_CHR;
} else if ((attr & FILE_ATTRIBUTE_DIRECTORY) != 0) {
entry->d_type = DT_DIR;
} else {
entry->d_type = DT_REG;
}
/* Reset dummy fields */
entry->d_ino = 0;
entry->d_off = 0;
entry->d_reclen = sizeof (struct dirent);
} else {
/*
* Cannot convert file name to multi-byte string so construct
* an erroneous directory entry and return that. Note that
* we cannot return NULL as that would stop the processing
* of directory entries completely.
*/
entry->d_name[0] = '?';
entry->d_name[1] = '\0';
entry->d_namlen = 1;
entry->d_type = DT_UNKNOWN;
entry->d_ino = 0;
entry->d_off = -1;
entry->d_reclen = 0;
}
/* Return pointer to directory entry */
*result = entry;
} else {
/* No more directory entries */
*result = NULL;
}
return /*OK*/0;
}
/*
* Close directory stream.
*/
static int
closedir(
DIR *dirp)
{
int ok;
if (dirp) {
/* Close wide-character directory stream */
ok = _wclosedir (dirp->wdirp);
dirp->wdirp = NULL;
/* Release multi-byte character version */
free (dirp);
} else {
/* Invalid directory stream */
dirent_set_errno (EBADF);
ok = /*failure*/-1;
}
return ok;
}
/*
* Rewind directory stream to beginning.
*/
static void
rewinddir(
DIR* dirp)
{
/* Rewind wide-character string directory stream */
_wrewinddir (dirp->wdirp);
}
/*
* Scan directory for entries.
*/
static int
scandir(
const char *dirname,
struct dirent ***namelist,
int (*filter)(const struct dirent*),
int (*compare)(const struct dirent**, const struct dirent**))
{
struct dirent **files = NULL;
size_t size = 0;
size_t allocated = 0;
const size_t init_size = 1;
DIR *dir = NULL;
struct dirent *entry;
struct dirent *tmp = NULL;
size_t i;
int result = 0;
/* Open directory stream */
dir = opendir (dirname);
if (dir) {
/* Read directory entries to memory */
while (1) {
/* Enlarge pointer table to make room for another pointer */
if (size >= allocated) {
void *p;
size_t num_entries;
/* Compute number of entries in the enlarged pointer table */
if (size < init_size) {
/* Allocate initial pointer table */
num_entries = init_size;
} else {
/* Double the size */
num_entries = size * 2;
}
/* Allocate first pointer table or enlarge existing table */
p = realloc (files, sizeof (void*) * num_entries);
if (p != NULL) {
/* Got the memory */
files = (dirent**) p;
allocated = num_entries;
} else {
/* Out of memory */
result = -1;
break;
}
}
/* Allocate room for temporary directory entry */
if (tmp == NULL) {
tmp = (struct dirent*) malloc (sizeof (struct dirent));
if (tmp == NULL) {
/* Cannot allocate temporary directory entry */
result = -1;
break;
}
}
/* Read directory entry to temporary area */
if (readdir_r (dir, tmp, &entry) == /*OK*/0) {
/* Did we get an entry? */
if (entry != NULL) {
int pass;
/* Determine whether to include the entry in result */
if (filter) {
/* Let the filter function decide */
pass = filter (tmp);
} else {
/* No filter function, include everything */
pass = 1;
}
if (pass) {
/* Store the temporary entry to pointer table */
files[size++] = tmp;
tmp = NULL;
/* Keep up with the number of files */
result++;
}
} else {
/*
* End of directory stream reached => sort entries and
* exit.
*/
qsort (files, size, sizeof (void*),
(int (*) (const void*, const void*)) compare);
break;
}
} else {
/* Error reading directory entry */
result = /*Error*/ -1;
break;
}
}
} else {
/* Cannot open directory */
result = /*Error*/ -1;
}
/* Release temporary directory entry */
free (tmp);
/* Release allocated memory on error */
if (result < 0) {
for (i = 0; i < size; i++) {
free (files[i]);
}
free (files);
files = NULL;
}
/* Close directory stream */
if (dir) {
closedir (dir);
}
/* Pass pointer table to caller */
if (namelist) {
*namelist = files;
}
return result;
}
/* Alphabetical sorting */
static int
alphasort(
const struct dirent **a, const struct dirent **b)
{
return strcoll ((*a)->d_name, (*b)->d_name);
}
/* Sort versions */
static int
versionsort(
const struct dirent **a, const struct dirent **b)
{
/* FIXME: implement strverscmp and use that */
return alphasort (a, b);
}
/* Convert multi-byte string to wide character string */
static int
dirent_mbstowcs_s(
size_t *pReturnValue,
wchar_t *wcstr,
size_t sizeInWords,
const char *mbstr,
size_t count)
{
int error;
#if defined(_MSC_VER) && _MSC_VER >= 1400
/* Microsoft Visual Studio 2005 or later */
error = mbstowcs_s (pReturnValue, wcstr, sizeInWords, mbstr, count);
#else
/* Older Visual Studio or non-Microsoft compiler */
size_t n;
/* Convert to wide-character string (or count characters) */
n = mbstowcs (wcstr, mbstr, sizeInWords);
if (!wcstr || n < count) {
/* Zero-terminate output buffer */
if (wcstr && sizeInWords) {
if (n >= sizeInWords) {
n = sizeInWords - 1;
}
wcstr[n] = 0;
}
/* Length of resulting multi-byte string WITH zero terminator */
if (pReturnValue) {
*pReturnValue = n + 1;
}
/* Success */
error = 0;
} else {
/* Could not convert string */
error = 1;
}
#endif
return error;
}
/* Convert wide-character string to multi-byte string */
static int
dirent_wcstombs_s(
size_t *pReturnValue,
char *mbstr,
size_t sizeInBytes, /* max size of mbstr */
const wchar_t *wcstr,
size_t count)
{
int error;
#if defined(_MSC_VER) && _MSC_VER >= 1400
/* Microsoft Visual Studio 2005 or later */
error = wcstombs_s (pReturnValue, mbstr, sizeInBytes, wcstr, count);
#else
/* Older Visual Studio or non-Microsoft compiler */
size_t n;
/* Convert to multi-byte string (or count the number of bytes needed) */
n = wcstombs (mbstr, wcstr, sizeInBytes);
if (!mbstr || n < count) {
/* Zero-terminate output buffer */
if (mbstr && sizeInBytes) {
if (n >= sizeInBytes) {
n = sizeInBytes - 1;
}
mbstr[n] = '\0';
}
/* Length of resulting multi-bytes string WITH zero-terminator */
if (pReturnValue) {
*pReturnValue = n + 1;
}
/* Success */
error = 0;
} else {
/* Cannot convert string */
error = 1;
}
#endif
return error;
}
/* Set errno variable */
static void
dirent_set_errno(
int error)
{
#if defined(_MSC_VER) && _MSC_VER >= 1400
/* Microsoft Visual Studio 2005 and later */
_set_errno (error);
#else
/* Non-Microsoft compiler or older Microsoft compiler */
errno = error;
#endif
}
#ifdef __cplusplus
}
#endif
#endif /*DIRENT_H*/
deps/dirent/tests/1/dir/readme.txt 0 → 100644
浏览文件 @ 438174a6
This file ensures that the directory dir will be created accordingly when
you unzip dirent to your computer. The directory itself is needed by the
test program t-dirent.
deps/dirent/tests/2/file.txt 0 → 100644
浏览文件 @ 438174a6
This dummy file is needed by the test program t-dirent.
deps/dirent/tests/3/README.txt 0 → 100644
浏览文件 @ 438174a6
This directory contains some random files for the t-scandir test program. The
files are empty and only the file names matter.
deps/dirent/tests/t-compile.c 0 → 100644
浏览文件 @ 438174a6
/*
* Test program to make sure that dirent compiles cleanly with winsock.
*
* Copyright (C) 1998-2019 Toni Ronkko
* This file is part of dirent. Dirent may be freely distributed
* under the MIT license. For all details and documentation, see
* https://github.com/tronkko/dirent
*/
#include <dirent.h>
#ifdef WIN32
# include <winsock2.h>
# include <ws2tcpip.h>
#endif
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
int
main(
int argc, char *argv[])
{
struct dirent *dirp = NULL;
(void) argc;
(void) argv;
#ifdef _DIRENT_HAVE_D_TYPE
printf ("Has d_type\n");
#endif
#ifdef _DIRENT_HAVE_D_NAMLEN
printf ("Has d_namlen\n");
#endif
#ifdef _D_EXACT_NAMLEN
printf ("Has _D_EXACT_NAMLEN\n");
#endif
#ifdef _D_ALLOC_NAMLEN
printf ("Has _D_ALLOC_NAMLEN\n");
#endif
#ifdef _D_ALLOC_NAMLEN
printf ("Has _D_ALLOC_NAMLEN\n");
#endif
printf ("Length of d_name with terminator: %d\n",
(int) sizeof (dirp->d_name));
printf ("OK\n");
return EXIT_SUCCESS;
}
deps/dirent/tests/t-cplusplus.cpp 0 → 100644
浏览文件 @ 438174a6
/*
* Test program to make sure that dirent compiles cleanly with C++
*
* Copyright (C) 1998-2019 Toni Ronkko
* This file is part of dirent. Dirent may be freely distributed
* under the MIT license. For all details and documentation, see
* https://github.com/tronkko/dirent
*/
#include <iostream>
#include <string.h>
#include <dirent.h>
#include <assert.h>
using namespace std;
/* Filter and sort functions */
static int only_readme (const struct dirent *entry);
int
main(
int argc, char *argv[])
{
(void) argc;
(void) argv;
/* Basic directory retrieval */
{
DIR *dir;
struct dirent *ent;
int found = 0;
/* Open directory */
dir = opendir ("tests/1");
if (dir == NULL) {
cerr << "Directory tests/1 not found" << endl;
abort ();
}
/* Read entries */
while ((ent = readdir (dir)) != NULL) {
/* Check each file */
if (strcmp (ent->d_name, ".") == 0) {
/* Directory itself */
#ifdef _DIRENT_HAVE_D_TYPE
assert (ent->d_type == DT_DIR);
#endif
#ifdef _DIRENT_HAVE_D_NAMLEN
assert (ent->d_namlen == 1);
#endif
#ifdef _D_EXACT_NAMLEN
assert (_D_EXACT_NAMLEN(ent) == 1);
#endif
#ifdef _D_ALLOC_NAMLEN
assert (_D_ALLOC_NAMLEN(ent) > 1);
#endif
found += 1;
} else if (strcmp (ent->d_name, "..") == 0) {
/* Parent directory */
#ifdef _DIRENT_HAVE_D_TYPE
assert (ent->d_type == DT_DIR);
#endif
#ifdef _DIRENT_HAVE_D_NAMLEN
assert (ent->d_namlen == 2);
#endif
#ifdef _D_EXACT_NAMLEN
assert (_D_EXACT_NAMLEN(ent) == 2);
#endif
#ifdef _D_ALLOC_NAMLEN
assert (_D_ALLOC_NAMLEN(ent) > 2);
#endif
found += 2;
} else if (strcmp (ent->d_name, "file") == 0) {
/* Regular file */
#ifdef _DIRENT_HAVE_D_TYPE
assert (ent->d_type == DT_REG);
#endif
#ifdef _DIRENT_HAVE_D_NAMLEN
assert (ent->d_namlen == 4);
#endif
#ifdef _D_EXACT_NAMLEN
assert (_D_EXACT_NAMLEN(ent) == 4);
#endif
#ifdef _D_ALLOC_NAMLEN
assert (_D_ALLOC_NAMLEN(ent) > 4);
#endif
found += 4;
} else if (strcmp (ent->d_name, "dir") == 0) {
/* Just a directory */
#ifdef _DIRENT_HAVE_D_TYPE
assert (ent->d_type == DT_DIR);
#endif
#ifdef _DIRENT_HAVE_D_NAMLEN
assert (ent->d_namlen == 3);
#endif
#ifdef _D_EXACT_NAMLEN
assert (_D_EXACT_NAMLEN(ent) == 3);
#endif
#ifdef _D_ALLOC_NAMLEN
assert (_D_ALLOC_NAMLEN(ent) > 3);
#endif
found += 8;
} else {
/* Other file */
cerr << "Unexpected file " << ent->d_name << endl;
abort ();
}
}
/* Make sure that all files were found */
assert (found == 0xf);
closedir (dir);
}
/* Basic scan with simple filter function */
{
struct dirent **files;
int n;
int i;
/* Read directory entries */
n = scandir ("tests/3", &files, only_readme, alphasort);
assert (n == 1);
/* Make sure that the filter works */
assert (strcmp (files[0]->d_name, "README.txt") == 0);
/* Release file names */
for (i = 0; i < n; i++) {
free (files[i]);
}
free (files);
}
cout << "OK" << endl;
return EXIT_SUCCESS;
}
/* Only pass README.txt file */
static int
only_readme (const struct dirent *entry)
{
int pass;
if (strcmp (entry->d_name, "README.txt") == 0) {
pass = 1;
} else {
pass = 0;
}
return pass;
}
deps/dirent/tests/t-dirent.c 0 → 100644
浏览文件 @ 438174a6
/*
* A test program to make sure that dirent works correctly.
*
* Copyright (C) 1998-2019 Toni Ronkko
* This file is part of dirent. Dirent may be freely distributed
* under the MIT license. For all details and documentation, see
* https://github.com/tronkko/dirent
*/
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#ifdef _MSC_VER
# include <direct.h>
# define chdir(x) _chdir(x)
#else
# include <unistd.h>
#endif
#include <sys/stat.h>
#include <dirent.h>
#include <errno.h>
#undef NDEBUG
#include <assert.h>
int
main(
int argc, char *argv[])
{
(void) argc;
(void) argv;
/* File type macros */
{
assert (DTTOIF(DT_REG) == S_IFREG);
assert (DTTOIF(DT_DIR) == S_IFDIR);
assert (DTTOIF(DT_FIFO) == S_IFIFO);
assert (DTTOIF(DT_SOCK) == S_IFSOCK);
assert (DTTOIF(DT_CHR) == S_IFCHR);
assert (DTTOIF(DT_BLK) == S_IFBLK);
assert (IFTODT(S_IFREG) == DT_REG);
assert (IFTODT(S_IFDIR) == DT_DIR);
assert (IFTODT(S_IFIFO) == DT_FIFO);
assert (IFTODT(S_IFSOCK) == DT_SOCK);
assert (IFTODT(S_IFCHR) == DT_CHR);
assert (IFTODT(S_IFBLK) == DT_BLK);
}
/* Basic directory retrieval */
{
DIR *dir;
struct dirent *ent;
int found = 0;
/* Open directory */
dir = opendir ("tests/1");
if (dir == NULL) {
fprintf (stderr, "Directory tests/1 not found\n");
abort ();
}
/* Read entries */
while ((ent = readdir (dir)) != NULL) {
/* Check each file */
if (strcmp (ent->d_name, ".") == 0) {
/* Directory itself */
#ifdef _DIRENT_HAVE_D_TYPE
assert (ent->d_type == DT_DIR);
#endif
#ifdef _DIRENT_HAVE_D_NAMLEN
assert (ent->d_namlen == 1);
#endif
#ifdef _D_EXACT_NAMLEN
assert (_D_EXACT_NAMLEN(ent) == 1);
#endif
#ifdef _D_ALLOC_NAMLEN
assert (_D_ALLOC_NAMLEN(ent) > 1);
#endif
found += 1;
} else if (strcmp (ent->d_name, "..") == 0) {
/* Parent directory */
#ifdef _DIRENT_HAVE_D_TYPE
assert (ent->d_type == DT_DIR);
#endif
#ifdef _DIRENT_HAVE_D_NAMLEN
assert (ent->d_namlen == 2);
#endif
#ifdef _D_EXACT_NAMLEN
assert (_D_EXACT_NAMLEN(ent) == 2);
#endif
#ifdef _D_ALLOC_NAMLEN
assert (_D_ALLOC_NAMLEN(ent) > 2);
#endif
found += 2;
} else if (strcmp (ent->d_name, "file") == 0) {
/* Regular file */
#ifdef _DIRENT_HAVE_D_TYPE
assert (ent->d_type == DT_REG);
#endif
#ifdef _DIRENT_HAVE_D_NAMLEN
assert (ent->d_namlen == 4);
#endif
#ifdef _D_EXACT_NAMLEN
assert (_D_EXACT_NAMLEN(ent) == 4);
#endif
#ifdef _D_ALLOC_NAMLEN
assert (_D_ALLOC_NAMLEN(ent) > 4);
#endif
found += 4;
} else if (strcmp (ent->d_name, "dir") == 0) {
/* Just a directory */
#ifdef _DIRENT_HAVE_D_TYPE
assert (ent->d_type == DT_DIR);
#endif
#ifdef _DIRENT_HAVE_D_NAMLEN
assert (ent->d_namlen == 3);
#endif
#ifdef _D_EXACT_NAMLEN
assert (_D_EXACT_NAMLEN(ent) == 3);
#endif
#ifdef _D_ALLOC_NAMLEN
assert (_D_ALLOC_NAMLEN(ent) > 3);
#endif
found += 8;
} else {
/* Other file */
fprintf (stderr, "Unexpected file %s\n", ent->d_name);
abort ();
}
}
/* Make sure that all files were found */
assert (found == 0xf);
closedir (dir);
}
/* Function opendir() fails if directory doesn't exist */
{
DIR *dir;
/* Open directory */
dir = opendir ("tests/invalid");
assert (dir == NULL);
assert (errno == ENOENT);
}
/* Function opendir() fails if pathname is really a file */
{
DIR *dir;
/* Open directory */
dir = opendir ("tests/1/file");
assert (dir == NULL);
assert (errno == ENOTDIR);
}
/* Function opendir() fails if pathname is a zero-length string */
{
DIR *dir;
/* Open directory */
dir = opendir ("");
assert (dir == NULL);
assert (errno == ENOENT);
}
/* Rewind of directory stream */
{
DIR *dir;
struct dirent *ent;
int found = 0;
/* Open directory */
dir = opendir ("tests/1");
assert (dir != NULL);
/* Read entries */
while ((ent = readdir (dir)) != NULL) {
/* Check each file */
if (strcmp (ent->d_name, ".") == 0) {
/* Directory itself */
found += 1;
} else if (strcmp (ent->d_name, "..") == 0) {
/* Parent directory */
found += 2;
} else if (strcmp (ent->d_name, "file") == 0) {
/* Regular file */
found += 4;
} else if (strcmp (ent->d_name, "dir") == 0) {
/* Just a directory */
found += 8;
} else {
/* Other file */
fprintf (stderr, "Unexpected file %s\n", ent->d_name);
abort ();
}
}
/* Make sure that all files were found */
assert (found == 0xf);
/* Rewind stream and read entries again */
rewinddir (dir);
found = 0;
/* Read entries */
while ((ent = readdir (dir)) != NULL) {
/* Check each file */
if (strcmp (ent->d_name, ".") == 0) {
/* Directory itself */
found += 1;
} else if (strcmp (ent->d_name, "..") == 0) {
/* Parent directory */
found += 2;
} else if (strcmp (ent->d_name, "file") == 0) {
/* Regular file */
found += 4;
} else if (strcmp (ent->d_name, "dir") == 0) {
/* Just a directory */
found += 8;
} else {
/* Other file */
fprintf (stderr, "Unexpected file %s\n", ent->d_name);
abort ();
}
}
/* Make sure that all files were found */
assert (found == 0xf);
closedir (dir);
}
/* Rewind with intervening change of working directory */
{
DIR *dir;
struct dirent *ent;
int found = 0;
int errorcode;
/* Open directory */
dir = opendir ("tests/1");
assert (dir != NULL);
/* Read entries */
while ((ent = readdir (dir)) != NULL) {
/* Check each file */
if (strcmp (ent->d_name, ".") == 0) {
/* Directory itself */
found += 1;
} else if (strcmp (ent->d_name, "..") == 0) {
/* Parent directory */
found += 2;
} else if (strcmp (ent->d_name, "file") == 0) {
/* Regular file */
found += 4;
} else if (strcmp (ent->d_name, "dir") == 0) {
/* Just a directory */
found += 8;
} else {
/* Other file */
fprintf (stderr, "Unexpected file %s\n", ent->d_name);
abort ();
}
}
/* Make sure that all files were found */
assert (found == 0xf);
/* Change working directory */
errorcode = chdir ("tests");
assert (errorcode == 0);
/* Rewind stream and read entries again */
rewinddir (dir);
found = 0;
/* Read entries */
while ((ent = readdir (dir)) != NULL) {
/* Check each file */
if (strcmp (ent->d_name, ".") == 0) {
/* Directory itself */
found += 1;
} else if (strcmp (ent->d_name, "..") == 0) {
/* Parent directory */
found += 2;
} else if (strcmp (ent->d_name, "file") == 0) {
/* Regular file */
found += 4;
} else if (strcmp (ent->d_name, "dir") == 0) {
/* Just a directory */
found += 8;
} else {
/* Other file */
fprintf (stderr, "Unexpected file %s\n", ent->d_name);
abort ();
}
}
/* Make sure that all files were found */
assert (found == 0xf);
/* Restore working directory */
errorcode = chdir ("..");
assert (errorcode == 0);
closedir (dir);
}
/* Long file name */
{
DIR *dir;
struct dirent *ent;
int found = 0;
/* Open directory */
dir = opendir ("tests/2");
if (dir == NULL) {
fprintf (stderr, "Directory tests/2 not found\n");
abort ();
}
/* Read entries */
while ((ent = readdir (dir)) != NULL) {
/* Check each file */
if (strcmp (ent->d_name, ".") == 0) {
/* Directory itself */
found += 1;
} else if (strcmp (ent->d_name, "..") == 0) {
/* Parent directory */
found += 2;
} else if (strcmp (ent->d_name, "file.txt") == 0) {
/* Regular 8+3 filename */
#ifdef _DIRENT_HAVE_D_TYPE
assert (ent->d_type == DT_REG);
#endif
#ifdef _DIRENT_HAVE_D_NAMLEN
assert (ent->d_namlen == 8);
#endif
#ifdef _D_EXACT_NAMLEN
assert (_D_EXACT_NAMLEN(ent) == 8);
#endif
#ifdef _D_ALLOC_NAMLEN
assert (_D_ALLOC_NAMLEN(ent) > 8);
#endif
found += 4;
} else if (strcmp (ent->d_name, "Testfile-1.2.3.dat") == 0) {
/* Long file name with multiple dots */
#ifdef _DIRENT_HAVE_D_TYPE
assert (ent->d_type == DT_REG);
#endif
#ifdef _DIRENT_HAVE_D_NAMLEN
assert (ent->d_namlen == 18);
#endif
#ifdef _D_EXACT_NAMLEN
assert (_D_EXACT_NAMLEN(ent) == 18);
#endif
#ifdef _D_ALLOC_NAMLEN
assert (_D_ALLOC_NAMLEN(ent) > 18);
#endif
found += 8;
} else {
/* Other file */
fprintf (stderr, "Unexpected file %s\n", ent->d_name);
abort ();
}
}
/* Make sure that all files were found */
assert (found == 0xf);
closedir (dir);
}
/* Basic directory retrieval with readdir_r */
{
DIR *dir;
struct dirent ent[10];
struct dirent *entry;
size_t i = 0;
size_t n = 0;
int found = 0;
/* Open directory */
dir = opendir ("tests/1");
if (dir == NULL) {
fprintf (stderr, "Directory tests/1 not found\n");
abort ();
}
/* Read entries to table */
while (readdir_r (dir, &ent[n], &entry) == /*OK*/0 && entry != 0) {
n++;
assert (n <= 4);
}
/* Make sure that we got all the files from directory */
assert (n == 4);
/* Check entries in memory */
for (i = 0; i < 4; i++) {
entry = &ent[i];
/* Check each file */
if (strcmp (entry->d_name, ".") == 0) {
/* Directory itself */
#ifdef _DIRENT_HAVE_D_TYPE
assert (entry->d_type == DT_DIR);
#endif
#ifdef _DIRENT_HAVE_D_NAMLEN
assert (entry->d_namlen == 1);
#endif
#ifdef _D_EXACT_NAMLEN
assert (_D_EXACT_NAMLEN(entry) == 1);
#endif
#ifdef _D_ALLOC_NAMLEN
assert (_D_ALLOC_NAMLEN(entry) > 1);
#endif
found += 1;
} else if (strcmp (entry->d_name, "..") == 0) {
/* Parent directory */
#ifdef _DIRENT_HAVE_D_TYPE
assert (entry->d_type == DT_DIR);
#endif
#ifdef _DIRENT_HAVE_D_NAMLEN
assert (entry->d_namlen == 2);
#endif
#ifdef _D_EXACT_NAMLEN
assert (_D_EXACT_NAMLEN(entry) == 2);
#endif
#ifdef _D_ALLOC_NAMLEN
assert (_D_ALLOC_NAMLEN(entry) > 2);
#endif
found += 2;
} else if (strcmp (entry->d_name, "file") == 0) {
/* Regular file */
#ifdef _DIRENT_HAVE_D_TYPE
assert (entry->d_type == DT_REG);
#endif
#ifdef _DIRENT_HAVE_D_NAMLEN
assert (entry->d_namlen == 4);
#endif
#ifdef _D_EXACT_NAMLEN
assert (_D_EXACT_NAMLEN(entry) == 4);
#endif
#ifdef _D_ALLOC_NAMLEN
assert (_D_ALLOC_NAMLEN(entry) > 4);
#endif
found += 4;
} else if (strcmp (entry->d_name, "dir") == 0) {
/* Just a directory */
#ifdef _DIRENT_HAVE_D_TYPE
assert (entry->d_type == DT_DIR);
#endif
#ifdef _DIRENT_HAVE_D_NAMLEN
assert (entry->d_namlen == 3);
#endif
#ifdef _D_EXACT_NAMLEN
assert (_D_EXACT_NAMLEN(entry) == 3);
#endif
#ifdef _D_ALLOC_NAMLEN
assert (_D_ALLOC_NAMLEN(entry) > 3);
#endif
found += 8;
} else {
/* Other file */
fprintf (stderr, "Unexpected file %s\n", entry->d_name);
abort ();
}
}
/* Make sure that all files were found */
assert (found == 0xf);
closedir (dir);
}
/* Basic directory retrieval with _wreaddir_r */
#ifdef WIN32
{
_WDIR *dir;
struct _wdirent ent[10];
struct _wdirent *entry;
size_t i = 0;
size_t n = 0;
int found = 0;
/* Open directory */
dir = _wopendir (L"tests/1");
if (dir == NULL) {
fprintf (stderr, "Directory tests/1 not found\n");
abort ();
}
/* Read entries to table */
while (_wreaddir_r (dir, &ent[n], &entry) == /*OK*/0 && entry != 0) {
n++;
assert (n <= 4);
}
/* Make sure that we got all the files from directory */
assert (n == 4);
/* Check entries in memory */
for (i = 0; i < 4; i++) {
entry = &ent[i];
/* Check each file */
if (wcscmp (entry->d_name, L".") == 0) {
/* Directory itself */
#ifdef _DIRENT_HAVE_D_TYPE
assert (entry->d_type == DT_DIR);
#endif
#ifdef _DIRENT_HAVE_D_NAMLEN
assert (entry->d_namlen == 1);
#endif
#ifdef _D_EXACT_NAMLEN
assert (_D_EXACT_NAMLEN(entry) == 1);
#endif
#ifdef _D_ALLOC_NAMLEN
assert (_D_ALLOC_NAMLEN(entry) > 1);
#endif
found += 1;
} else if (wcscmp (entry->d_name, L"..") == 0) {
/* Parent directory */
#ifdef _DIRENT_HAVE_D_TYPE
assert (entry->d_type == DT_DIR);
#endif
#ifdef _DIRENT_HAVE_D_NAMLEN
assert (entry->d_namlen == 2);
#endif
#ifdef _D_EXACT_NAMLEN
assert (_D_EXACT_NAMLEN(entry) == 2);
#endif
#ifdef _D_ALLOC_NAMLEN
assert (_D_ALLOC_NAMLEN(entry) > 2);
#endif
found += 2;
} else if (wcscmp (entry->d_name, L"file") == 0) {
/* Regular file */
#ifdef _DIRENT_HAVE_D_TYPE
assert (entry->d_type == DT_REG);
#endif
#ifdef _DIRENT_HAVE_D_NAMLEN
assert (entry->d_namlen == 4);
#endif
#ifdef _D_EXACT_NAMLEN
assert (_D_EXACT_NAMLEN(entry) == 4);
#endif
#ifdef _D_ALLOC_NAMLEN
assert (_D_ALLOC_NAMLEN(entry) > 4);
#endif
found += 4;
} else if (wcscmp (entry->d_name, L"dir") == 0) {
/* Just a directory */
#ifdef _DIRENT_HAVE_D_TYPE
assert (entry->d_type == DT_DIR);
#endif
#ifdef _DIRENT_HAVE_D_NAMLEN
assert (entry->d_namlen == 3);
#endif
#ifdef _D_EXACT_NAMLEN
assert (_D_EXACT_NAMLEN(entry) == 3);
#endif
#ifdef _D_ALLOC_NAMLEN
assert (_D_ALLOC_NAMLEN(entry) > 3);
#endif
found += 8;
} else {
/* Other file */
fprintf (stderr, "Unexpected file\n");
abort ();
}
}
/* Make sure that all files were found */
assert (found == 0xf);
_wclosedir (dir);
}
#endif
printf ("OK\n");
return EXIT_SUCCESS;
}
deps/dirent/tests/t-scandir.c 0 → 100644
浏览文件 @ 438174a6
/*
* Make sure that scandir function works OK.
*
* Copyright (C) 1998-2019 Toni Ronkko
* This file is part of dirent. Dirent may be freely distributed
* under the MIT license. For all details and documentation, see
* https://github.com/tronkko/dirent
*/
/* Silence warning about fopen being insecure */
#define _CRT_SECURE_NO_WARNINGS
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <sys/stat.h>
#include <dirent.h>
#include <errno.h>
#include <time.h>
#include <limits.h>
#undef NDEBUG
#include <assert.h>
/* Filter and sort functions */
static int only_readme (const struct dirent *entry);
static int no_directories (const struct dirent *entry);
static int reverse_alpha (const struct dirent **a, const struct dirent **b);
int
main(
int argc, char *argv[])
{
struct dirent **files;
int i;
int n;
(void) argc;
(void) argv;
/* Initialize random number generator */
srand ((unsigned) time (NULL));
/* Basic scan with simple filter function */
{
/* Read directory entries */
n = scandir ("tests/3", &files, only_readme, alphasort);
assert (n == 1);
/* Make sure that the filter works */
assert (strcmp (files[0]->d_name, "README.txt") == 0);
/* Release file names */
for (i = 0; i < n; i++) {
free (files[i]);
}
free (files);
}
/* Basic scan with default sorting function */
{
/* Read directory entries in alphabetic order */
n = scandir ("tests/3", &files, NULL, alphasort);
assert (n == 13);
/* Make sure that we got all the file names in the proper order */
assert (strcmp (files[0]->d_name, ".") == 0);
assert (strcmp (files[1]->d_name, "..") == 0);
assert (strcmp (files[2]->d_name, "3zero.dat") == 0);
assert (strcmp (files[3]->d_name, "666.dat") == 0);
assert (strcmp (files[4]->d_name, "Qwerty-my-aunt.dat") == 0);
assert (strcmp (files[5]->d_name, "README.txt") == 0);
assert (strcmp (files[6]->d_name, "aaa.dat") == 0);
assert (strcmp (files[7]->d_name, "dirent.dat") == 0);
assert (strcmp (files[8]->d_name, "empty.dat") == 0);
assert (strcmp (files[9]->d_name, "sane-1.12.0.dat") == 0);
assert (strcmp (files[10]->d_name, "sane-1.2.3.dat") == 0);
assert (strcmp (files[11]->d_name, "sane-1.2.4.dat") == 0);
assert (strcmp (files[12]->d_name, "zebra.dat") == 0);
/* Release file names */
for (i = 0; i < n; i++) {
free (files[i]);
}
free (files);
}
/* Custom filter AND sort function */
{
/* Read directory entries in alphabetic order */
n = scandir ("tests/3", &files, no_directories, reverse_alpha);
assert (n == 11);
/* Make sure that we got all the FILE names in the REVERSE order */
assert (strcmp (files[0]->d_name, "zebra.dat") == 0);
assert (strcmp (files[1]->d_name, "sane-1.2.4.dat") == 0);
assert (strcmp (files[2]->d_name, "sane-1.2.3.dat") == 0);
assert (strcmp (files[3]->d_name, "sane-1.12.0.dat") == 0);
assert (strcmp (files[4]->d_name, "empty.dat") == 0);
assert (strcmp (files[5]->d_name, "dirent.dat") == 0);
assert (strcmp (files[6]->d_name, "aaa.dat") == 0);
assert (strcmp (files[7]->d_name, "README.txt") == 0);
assert (strcmp (files[8]->d_name, "Qwerty-my-aunt.dat") == 0);
assert (strcmp (files[9]->d_name, "666.dat") == 0);
assert (strcmp (files[10]->d_name, "3zero.dat") == 0);
/* Release file names */
for (i = 0; i < n; i++) {
free (files[i]);
}
free (files);
}
/* Trying to read from non-existent directory leads to an error */
{
files = NULL;
n = scandir ("tests/invalid", &files, NULL, alphasort);
assert (n == -1);
assert (files == NULL);
assert (errno == ENOENT);
}
/* Trying to open file as a directory produces ENOTDIR error */
{
files = NULL;
n = scandir ("tests/3/666.dat", &files, NULL, alphasort);
assert (n == -1);
assert (files == NULL);
assert (errno == ENOTDIR);
}
/* Scan large directory */
{
char dirname[PATH_MAX+1];
int i, j;
int ok;
/* Copy name of temporary directory to variable dirname */
#ifdef WIN32
i = GetTempPathA (PATH_MAX, dirname);
assert (i > 0);
#else
strcpy (dirname, "/tmp/");
i = strlen (dirname);
#endif
/* Append random characters to dirname */
for (j = 0; j < 10; j++) {
char c;
/* Generate random character */
c = "abcdefghijklmnopqrstuvwxyz"[rand() % 26];
/* Append character to dirname */
assert (i < PATH_MAX);
dirname[i++] = c;
}
/* Terminate directory name */
assert (i < PATH_MAX);
dirname[i] = '\0';
/* Create directory */
#ifdef WIN32
ok = CreateDirectoryA (dirname, NULL);
assert (ok);
#else
ok = mkdir (dirname, 0700);
assert (ok == /*success*/0);
#endif
/* Create one thousand files */
assert (i + 5 < PATH_MAX);
for (j = 0; j < 1000; j++) {
FILE *fp;
/* Construct file name */
dirname[i] = '/';
dirname[i+1] = 'z';
dirname[i+2] = '0' + ((j / 100) % 10);
dirname[i+3] = '0' + ((j / 10) % 10);
dirname[i+4] = '0' + (j % 10);
dirname[i+5] = '\0';
/* Create file */
fp = fopen (dirname, "w");
assert (fp != NULL);
fclose (fp);
}
/* Cut out the file name part */
dirname[i] = '\0';
/* Scan directory */
n = scandir (dirname, &files, no_directories, alphasort);
assert (n == 1000);
/* Make sure that all 1000 files are read back */
for (j = 0; j < n; j++) {
char match[100];
/* Construct file name */
match[0] = 'z';
match[1] = '0' + ((j / 100) % 10);
match[2] = '0' + ((j / 10) % 10);
match[3] = '0' + (j % 10);
match[4] = '\0';
/* Make sure that file name matches that on the disk */
assert (strcmp (files[j]->d_name, match) == 0);
}
/* Release file names */
for (j = 0; j < n; j++) {
free (files[j]);
}
free (files);
}
printf ("OK\n");
return EXIT_SUCCESS;
}
/* Only pass README.txt file */
static int
only_readme (const struct dirent *entry)
{
int pass;
if (strcmp (entry->d_name, "README.txt") == 0) {
pass = 1;
} else {
pass = 0;
}
return pass;
}
/* Filter out directories */
static int
no_directories (const struct dirent *entry)
{
int pass;
if (entry->d_type != DT_DIR) {
pass = 1;
} else {
pass = 0;
}
return pass;
}
/* Sort in reverse direction */
static int
reverse_alpha(
const struct dirent **a, const struct dirent **b)
{
return strcoll ((*b)->d_name, (*a)->d_name);
}
deps/dirent/tests/t-unicode.c 0 → 100644
浏览文件 @ 438174a6
/*
* Test program to try unicode file names.
*
* Copyright (C) 1998-2019 Toni Ronkko
* This file is part of dirent. Dirent may be freely distributed
* under the MIT license. For all details and documentation, see
* https://github.com/tronkko/dirent
*/
/* Silence warning about fopen being insecure */
#define _CRT_SECURE_NO_WARNINGS
#include <dirent.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <wchar.h>
#include <time.h>
#include <locale.h>
#undef NDEBUG
#include <assert.h>
int
main(
int argc, char *argv[])
{
#ifdef WIN32
wchar_t wpath[MAX_PATH+1];
char path[MAX_PATH+1];
DWORD i, j, k, x;
BOOL ok;
HANDLE fh;
_WDIR *wdir;
struct _wdirent *wentry;
DIR *dir;
struct dirent *entry;
char buffer[100];
FILE *fp;
int counter = 0;
(void) argc;
(void) argv;
/* Initialize random number generator */
srand (((int) time (NULL)) * 257 + ((int) GetCurrentProcessId ()));
/* Set current locale */
if (argc > 1) {
printf ("Locale %s\n", argv[1]);
setlocale (LC_ALL, argv[1]);
} else {
setlocale (LC_ALL, "");
}
/****** CREATE FILE WITH UNICODE FILE NAME ******/
/* Get path to temporary directory (wide-character and ascii) */
i = GetTempPathW (MAX_PATH, wpath);
assert (i > 0);
j = GetTempPathA (MAX_PATH, path);
assert (j > 0);
/* Append random directory name */
for (k = 0; k < 10; k++) {
char c;
/* Generate random character */
c = "abcdefghijklmnopqrstuvwxyz"[rand() % 26];
/* Append character to paths */
assert (i < MAX_PATH && j < MAX_PATH);
wpath[i++] = c;
path[j++] = c;
}
/* Terminate paths */
assert (i < MAX_PATH && j < MAX_PATH);
wpath[i] = '\0';
path[j] = '\0';
/* Remember the end of directory name */
k = i;
/* Create directory using unicode */
ok = CreateDirectoryW (wpath, NULL);
if (!ok) {
DWORD e = GetLastError ();
wprintf (L"Cannot create directory %ls (code %u)\n", wpath, e);
abort ();
}
/* Overwrite zero terminator with path separator */
assert (i < MAX_PATH && j < MAX_PATH);
wpath[i++] = '\\';
/* Append a few unicode characters */
assert (i < MAX_PATH);
wpath[i++] = 0x6d4b;
assert (i < MAX_PATH);
wpath[i++] = 0x8bd5;
/* Terminate string */
assert (i < MAX_PATH);
wpath[i] = '\0';
/* Create file with unicode */
fh = CreateFileW(
wpath,
/* Access */ GENERIC_READ | GENERIC_WRITE,
/* Share mode */ 0,
/* Security attributes */ NULL,
/* Creation disposition */ CREATE_NEW,
/* Attributes */ FILE_ATTRIBUTE_NORMAL,
/* Template files */ NULL
);
assert (fh != INVALID_HANDLE_VALUE);
/* Write some data to file */
ok = WriteFile(
/* File handle */ fh,
/* Pointer to data */ "hep\n",
/* Number of bytes to write */ 4,
/* Number of bytes written */ NULL,
/* Overlapped */ NULL
);
assert (ok);
/* Close file */
ok = CloseHandle (fh);
assert (ok);
/****** MAKE SURE THAT UNICODE FILE NAME CAN BE READ BY _WREADDIR ******/
/* Zero terminate wide-character path and open directory stream */
wpath[k] = '\0';
wdir = _wopendir (wpath);
if (wdir == NULL) {
wprintf (L"Cannot open directory %ls\n", wpath);
abort ();
}
/* Read through entries */
counter = 0;
while ((wentry = _wreaddir (wdir)) != NULL) {
/* Skip pseudo directories */
if (wcscmp (wentry->d_name, L".") == 0) {
continue;
}
if (wcscmp (wentry->d_name, L"..") == 0) {
continue;
}
/* Found a file */
counter++;
assert (wentry->d_type == DT_REG);
/* Append file name to path */
i = k;
assert (i < MAX_PATH);
wpath[i++] = '\\';
x = 0;
while (wentry->d_name[x] != '\0') {
assert (i < MAX_PATH);
wpath[i++] = wentry->d_name[x++];
}
assert (i < MAX_PATH);
wpath[i] = '\0';
/* Open file for read */
fh = CreateFileW(
wpath,
/* Access */ GENERIC_READ,
/* Share mode */ 0,
/* Security attributes */ NULL,
/* Creation disposition */ OPEN_EXISTING,
/* Attributes */ FILE_ATTRIBUTE_NORMAL,
/* Template files */ NULL
);
assert (fh != INVALID_HANDLE_VALUE);
/* Read data from file */
ok = ReadFile(
/* File handle */ fh,
/* Output buffer */ buffer,
/* Maximum number of bytes to read */ sizeof (buffer) - 1,
/* Number of bytes actually read */ &x,
/* Overlapped */ NULL
);
assert (ok);
/* Make sure that we got the file contents right */
assert (x == 4);
assert (buffer[0] == 'h');
assert (buffer[1] == 'e');
assert (buffer[2] == 'p');
assert (buffer[3] == '\n');
/* Close file */
ok = CloseHandle (fh);
assert (ok);
}
assert (counter == 1);
/* Close directory */
_wclosedir (wdir);
/****** MAKE SURE THAT UNICODE FILE NAME CAN BE READ BY READDIR ******/
/* Zero terminate ascii path and open directory stream */
k = j;
path[k] = '\0';
dir = opendir (path);
if (dir == NULL) {
fprintf (stderr, "Cannot open directory %s\n", path);
abort ();
}
/* Read through entries */
counter = 0;
while ((entry = readdir (dir)) != NULL) {
/* Skip pseudo directories */
if (strcmp (entry->d_name, ".") == 0) {
continue;
}
if (strcmp (entry->d_name, "..") == 0) {
continue;
}
/* Found a file */
counter++;
assert (entry->d_type == DT_REG);
/* Append file name to path */
j = k;
assert (j < MAX_PATH);
path[j++] = '\\';
x = 0;
while (entry->d_name[x] != '\0') {
assert (j < MAX_PATH);
path[j++] = entry->d_name[x++];
}
assert (j < MAX_PATH);
path[j] = '\0';
/* Open file for read */
fp = fopen (path, "r");
if (!fp) {
fprintf (stderr, "Cannot open file %s\n", path);
abort ();
}
/* Read data from file */
if (fgets (buffer, sizeof (buffer), fp) == NULL) {
fprintf (stderr, "Cannot read file %s\n", path);
abort ();
}
/* Make sure that we got the file contents right */
assert (buffer[0] == 'h');
assert (buffer[1] == 'e');
assert (buffer[2] == 'p');
assert (buffer[3] == '\n');
assert (buffer[4] == '\0');
/* Close file */
fclose (fp);
}
assert (counter == 1);
/* Close directory */
closedir (dir);
/****** CREATE FILE WITH UTF-8 ******/
/* Append UTF-8 file name (åäö.txt) to path */
j = k;
path[j++] = '\\';
path[j++] = 0xc3;
path[j++] = 0xa5;
path[j++] = 0xc3;
path[j++] = 0xa4;
path[j++] = 0xc3;
path[j++] = 0xb6;
path[j++] = 0x2e;
path[j++] = 0x74;
path[j++] = 0x78;
path[j++] = 0x74;
assert (j < MAX_PATH);
path[j] = '\0';
/*
* Create file.
*
* Be ware that the code below creates a different file depending on the
* current locale! For example, if the current locale is
* english_us.65001, then the file name will be "åäö.txt" (7 characters).
* However, if the current locale is english_us.1252, then the file name
* will be "ÃċÃĊö.txt" (10 characters).
*/
printf ("Creating %s\n", path);
fp = fopen (path, "w");
if (!fp) {
fprintf (stderr, "Cannot open file %s\n", path);
abort ();
}
fputs ("hep\n", fp);
fclose (fp);
/* Open directory again */
path[k] = '\0';
dir = opendir (path);
if (dir == NULL) {
fprintf (stderr, "Cannot open directory %s\n", path);
abort ();
}
/* Read through entries */
counter = 0;
while ((entry = readdir (dir)) != NULL) {
/* Skip pseudo directories */
if (strcmp (entry->d_name, ".") == 0) {
continue;
}
if (strcmp (entry->d_name, "..") == 0) {
continue;
}
/* Found a file */
counter++;
assert (entry->d_type == DT_REG);
/* Append file name to path */
j = k;
assert (j < MAX_PATH);
path[j++] = '\\';
x = 0;
while (entry->d_name[x] != '\0') {
assert (j < MAX_PATH);
path[j++] = entry->d_name[x++];
}
assert (j < MAX_PATH);
path[j] = '\0';
/* Print file name for debugging */
printf ("Opening \"%s\" hex ", path + k + 1);
x = 0;
while (entry->d_name[x] != '\0') {
printf ("0x%02x ", (unsigned) (entry->d_name[x++] & 0xff));
}
printf ("\n");
/* Open file for read */
fp = fopen (path, "r");
if (!fp) {
fprintf (stderr, "Cannot open file %s\n", path);
abort ();
}
/* Read data from file */
if (fgets (buffer, sizeof (buffer), fp) == NULL) {
fprintf (stderr, "Cannot read file %s\n", path);
abort ();
}
/* Make sure that we got the file contents right */
assert (buffer[0] == 'h');
assert (buffer[1] == 'e');
assert (buffer[2] == 'p');
assert (buffer[3] == '\n');
assert (buffer[4] == '\0');
/* Close file */
fclose (fp);
}
assert (counter == 2);
/* Close directory */
closedir (dir);
#else
/* Linux */
(void) argc;
(void) argv;
#endif
return EXIT_SUCCESS;
}
deps/iconv/CMakeLists.txt
浏览文件 @ 438174a6
CMAKE_MINIMUM_REQUIRED(VERSION 2.8)
PROJECT(TDengine)
IF (TD_WINDOWS_64)
IF (TD_WINDOWS_64 OR TD_WINDOWS_32)
LIST(APPEND SRC iconv.c)
LIST(APPEND SRC localcharset.c)
INCLUDE_DIRECTORIES(.)
......
deps/pthread/CMakeLists.txt
浏览文件 @ 438174a6
CMAKE_MINIMUM_REQUIRED(VERSION 2.8)
PROJECT(TDengine)
IF (TD_WINDOWS_64)
IF (TD_WINDOWS_64 OR TD_WINDOWS_32)
INCLUDE_DIRECTORIES(.)
LIST(APPEND SRC pthread.c)
ADD_LIBRARY(pthread ${SRC})
......
deps/regex/CMakeLists.txt
浏览文件 @ 438174a6
CMAKE_MINIMUM_REQUIRED(VERSION 2.8)
PROJECT(TDengine)
IF (TD_WINDOWS_64)
IF (TD_WINDOWS_64 OR TD_WINDOWS_32)
INCLUDE_DIRECTORIES(inc .)
LIST(APPEND SRC regex.c)
ADD_LIBRARY(regex ${SRC})
......
src/client/CMakeLists.txt
浏览文件 @ 438174a6
......@@ -31,7 +31,7 @@ IF ((TD_LINUX_64) OR (TD_LINUX_32 AND TD_ARM))
MESSAGE(STATUS "build version ${VERSION_INFO}")
SET_TARGET_PROPERTIES(taos PROPERTIES VERSION ${VERSION_INFO} SOVERSION 1)
ELSEIF (TD_WINDOWS_64)
ELSEIF (TD_WINDOWS_64 OR TD_WINDOWS_32)
INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/jni/windows)
INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/jni/windows/win32)
INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/pthread)
......
src/client/jni/com_taosdata_jdbc_TSDBJNIConnector.h
浏览文件 @ 438174a6
......@@ -142,8 +142,8 @@ JNIEXPORT jlong JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_subscribeImp
* Method: consumeImp
* Signature: (J)Lcom/taosdata/jdbc/TSDBResultSetRowData;
*/
JNIEXPORT jobject JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_consumeImp
(JNIEnv *, jobject, jlong, jint);
JNIEXPORT jlong JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_consumeImp
(JNIEnv *, jobject, jlong);
/*
* Class: com_taosdata_jdbc_TSDBJNIConnector
......
src/client/src/TSDBJNIConnector.c
浏览文件 @ 438174a6
......@@ -606,7 +606,7 @@ static jobject convert_one_row(JNIEnv *env, TAOS_ROW row, TAOS_FIELD* fields, in
return rowobj;
}
JNIEXPORT jobject JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_consumeImp(JNIEnv *env, jobject jobj, jlong sub, jint timeout) {
JNIEXPORT jlong JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_consumeImp(JNIEnv *env, jobject jobj, jlong sub) {
jniTrace("jobj:%p, in TSDBJNIConnector_consumeImp, sub:%ld", jobj, sub);
jniGetGlobalMethod(env);
......@@ -616,38 +616,14 @@ JNIEXPORT jobject JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_consumeImp(JNI
int64_t start = taosGetTimestampMs();
int count = 0;
while (true) {
TAOS_RES * res = taos_consume(tsub);
TAOS_RES *res = taos_consume(tsub);
if (res == NULL) {
jniError("jobj:%p, tsub:%p, taos_consume returns NULL", jobj, tsub);
return NULL;
}
TAOS_FIELD *fields = taos_fetch_fields(res);
int num_fields = taos_num_fields(res);
while (true) {
TAOS_ROW row = taos_fetch_row(res);
if (row == NULL) {
break;
}
jobject rowobj = convert_one_row(env, row, fields, num_fields);
(*env)->CallBooleanMethod(env, rows, g_arrayListAddFp, rowobj);
count++;
}
if (count > 0) {
break;
}
if (timeout == -1) {
continue;
}
if (((int)(taosGetTimestampMs() - start)) >= timeout) {
jniTrace("jobj:%p, sub:%ld, timeout", jobj, sub);
break;
}
}
return rows;
return res;
}
JNIEXPORT void JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_unsubscribeImp(JNIEnv *env, jobject jobj, jlong sub, jboolean keepProgress) {
......
src/client/src/tscAsync.c
浏览文件 @ 438174a6
......@@ -26,6 +26,7 @@
#include "tutil.h"
#include "tnote.h"
extern void tscSetNextLaunchTimer(SSqlStream *pStream, SSqlObj *pSql);
static void tscProcessFetchRow(SSchedMsg *pMsg);
static void tscAsyncQueryRowsForNextVnode(void *param, TAOS_RES *tres, int numOfRows);
......@@ -494,6 +495,11 @@ void tscMeterMetaCallBack(void *param, TAOS_RES *res, int code) {
if ((pQueryInfo->type & TSDB_QUERY_TYPE_STABLE_SUBQUERY) == TSDB_QUERY_TYPE_STABLE_SUBQUERY) {
SMeterMetaInfo* pMeterMetaInfo = tscGetMeterMetaInfoFromQueryInfo(pQueryInfo, 0);
if(pMeterMetaInfo->pMeterMeta == NULL) {
code = tscGetMeterMeta(pSql, pMeterMetaInfo);
assert(code == TSDB_CODE_SUCCESS);
}
assert(pMeterMetaInfo->pMeterMeta->numOfTags != 0 && pMeterMetaInfo->vnodeIndex >= 0 && pSql->param != NULL);
SRetrieveSupport *trs = (SRetrieveSupport *)pSql->param;
......@@ -504,11 +510,6 @@ void tscMeterMetaCallBack(void *param, TAOS_RES *res, int code) {
tscTrace("%p get metricMeta during super table query successfully", pSql);
code = tscGetMeterMeta(pSql, pMeterMetaInfo);
pRes->code = code;
if (code == TSDB_CODE_ACTION_IN_PROGRESS) return;
code = tscGetMetricMeta(pSql, 0);
pRes->code = code;
......@@ -553,7 +554,6 @@ void tscMeterMetaCallBack(void *param, TAOS_RES *res, int code) {
}
if (pSql->pStream) {
tscTrace("%p stream:%p meta is updated, start new query, command:%d", pSql, pSql->pStream, pSql->cmd.command);
/*
* NOTE:
* transfer the sql function for super table query before get meter/metric meta,
......@@ -561,6 +561,21 @@ void tscMeterMetaCallBack(void *param, TAOS_RES *res, int code) {
*/
SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex);
SMeterMetaInfo *pMeterMetaInfo = tscGetMeterMetaInfoFromQueryInfo(pQueryInfo, 0);
if ((UTIL_METER_IS_SUPERTABLE(pMeterMetaInfo)
&& ( pMeterMetaInfo->pMeterMeta == NULL
|| pMeterMetaInfo->pMetricMeta == NULL
|| pMeterMetaInfo->pMetricMeta->numOfMeters == 0
|| pMeterMetaInfo->pMetricMeta->numOfVnodes == 0))
|| (!(UTIL_METER_IS_SUPERTABLE(pMeterMetaInfo)) && (pMeterMetaInfo->pMeterMeta == NULL))) {
tscTrace("%p stream:%p meta is updated, but no table, clear meter meta and set next launch new query, command:%d", pSql, pSql->pStream, pSql->cmd.command);
tscClearMeterMetaInfo(pMeterMetaInfo, false);
tscSetNextLaunchTimer(pSql->pStream, pSql);
return;
}
tscTrace("%p stream:%p meta is updated, start new query, command:%d", pSql, pSql->pStream, pSql->cmd.command);
tscTansformSQLFunctionForSTableQuery(pQueryInfo);
tscIncStreamExecutionCount(pSql->pStream);
} else {
......
src/client/src/tscFunctionImpl.c
浏览文件 @ 438174a6
......@@ -3669,7 +3669,7 @@ int WCSPatternMatch(const wchar_t *patterStr, const wchar_t *str, size_t size, c
wchar_t accept[3] = {towupper(c), towlower(c), 0};
while (1) {
size_t n = wcsspn(str, accept);
size_t n = wcscspn(str, accept);
str += n;
if (str[0] == 0 || (n >= size - 1)) {
......@@ -3678,7 +3678,7 @@ int WCSPatternMatch(const wchar_t *patterStr, const wchar_t *str, size_t size, c
str++;
int32_t ret = WCSPatternMatch(&patterStr[i], str, wcslen(str), pInfo);
int32_t ret = WCSPatternMatch(&patterStr[i], str, twcslen(str), pInfo);
if (ret != TSDB_PATTERN_NOMATCH) {
return ret;
}
......
src/client/src/tscLocal.c
浏览文件 @ 438174a6
......@@ -434,6 +434,19 @@ static void tscProcessServStatus(SSqlObj *pSql) {
if (pObj->pHb->res.code == TSDB_CODE_NETWORK_UNAVAIL) {
pSql->res.code = TSDB_CODE_NETWORK_UNAVAIL;
return;
} else {
int32_t* data = (int32_t*) pObj->pHb->res.data;
if (data != NULL) {
int32_t totalDnode = data[0];
int32_t onlineDnode = data[1];
assert(onlineDnode <= totalDnode);
if (onlineDnode < totalDnode) {
pSql->res.code = TSDB_CODE_NETWORK_UNAVAIL;
return;
}
}
}
} else {
if (pSql->res.code == TSDB_CODE_NETWORK_UNAVAIL) {
......
src/client/src/tscParseInsert.c
浏览文件 @ 438174a6
......@@ -1442,6 +1442,9 @@ static int tscInsertDataFromFile(SSqlObj *pSql, FILE *fp, char *tmpTokenBuf) {
pTableDataBlock->size = sizeof(SShellSubmitBlock);
pTableDataBlock->rowSize = pMeterMeta->rowSize;
code = tscAllocateMemIfNeed(pTableDataBlock, rowSize, &maxRows);
if (TSDB_CODE_SUCCESS != code) return -1;
numOfRows += pSql->res.numOfRows;
pSql->res.numOfRows = 0;
count = 0;
......
src/client/src/tscSQLParser.c
浏览文件 @ 438174a6
......@@ -282,6 +282,7 @@ int32_t tscToSQLCmd(SSqlObj* pSql, struct SSqlInfo* pInfo) {
const char* msg2 = "name too long";
SCreateDBInfo* pCreateDB = &(pInfo->pDCLInfo->dbOpt);
pCmd->existsCheck = pInfo->pDCLInfo->existsCheck;
if (tscValidateName(&pCreateDB->dbname) != TSDB_CODE_SUCCESS) {
return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1);
}
......@@ -1949,6 +1950,7 @@ static int16_t doGetColumnIndex(SQueryInfo* pQueryInfo, int32_t index, SSQLToken
if (strncasecmp(pSchema[i].name, pToken->z, pToken->n) == 0) {
columnIndex = i;
break;
}
}
......@@ -2595,7 +2597,7 @@ static int32_t doExtractColumnFilterInfo(SQueryInfo* pQueryInfo, SColumnFilterIn
tVariantDump(&pRight->val, (char*)pColumnFilter->pz, colType);
size_t len = wcslen((wchar_t*)pColumnFilter->pz);
size_t len = twcslen((wchar_t*)pColumnFilter->pz);
pColumnFilter->len = len * TSDB_NCHAR_SIZE;
} else {
tVariantDump(&pRight->val, (char*)&pColumnFilter->lowerBndd, colType);
......@@ -4659,6 +4661,7 @@ int32_t parseLimitClause(SQueryInfo* pQueryInfo, int32_t clauseIndex, SQuerySQL*
if (pMeterMetaInfo->pMeterMeta == NULL || pMetricMeta == NULL || pMetricMeta->numOfMeters == 0) {
tscTrace("%p no table in metricmeta, no output result", pSql);
pQueryInfo->command = TSDB_SQL_RETRIEVE_EMPTY_RESULT;
pSql->res.qhandle = 0x1; // to pass the qhandle check;
}
// keep original limitation value in globalLimit
......
src/client/src/tscSQLParserImpl.c
浏览文件 @ 438174a6
......@@ -48,6 +48,7 @@ int32_t tSQLParse(SSqlInfo *pSQLInfo, const char *pStr) {
Parse(pParser, 0, t0, pSQLInfo);
goto abort_parse;
}
case TK_QUESTION:
case TK_ILLEGAL: {
snprintf(pSQLInfo->pzErrMsg, tListLen(pSQLInfo->pzErrMsg), "unrecognized token: \"%s\"", t0.z);
pSQLInfo->valid = false;
......@@ -815,6 +816,7 @@ void setCreateDBSQL(SSqlInfo *pInfo, int32_t type, SSQLToken *pToken, SCreateDBI
pInfo->pDCLInfo->dbOpt.dbname = *pToken;
tTokenListAppend(pInfo->pDCLInfo, pIgExists);
pInfo->pDCLInfo->existsCheck = (pIgExists->n == 1);
}
void setCreateAcctSQL(SSqlInfo *pInfo, int32_t type, SSQLToken *pName, SSQLToken *pPwd, SCreateAcctSQL *pAcctInfo) {
......
src/client/src/tscSecondaryMerge.c
浏览文件 @ 438174a6
......@@ -231,13 +231,13 @@ void tscCreateLocalReducer(tExtMemBuffer **pMemBuffer, int32_t numOfBuffer, tOrd
idx += 1;
}
}
assert(idx >= pReducer->numOfBuffer);
if (idx == 0) {
free(pReducer);
return;
}
pReducer->numOfBuffer = idx;
pReducer->numOfBuffer = idx; // the actual entries that has result for merge
SCompareParam *param = malloc(sizeof(SCompareParam));
param->pLocalData = pReducer->pLocalDataSrc;
......
src/client/src/tscServer.c
浏览文件 @ 438174a6
......@@ -106,12 +106,12 @@ static int32_t tscGetMgmtConnMaxRetryTimes() {
return tscMgmtIpList.numOfIps * factor;
}
void tscProcessHeartBeatRsp(void *param, TAOS_RES *tres, int code) {
int32_t tscProcessHeartBeatRsp(void *param, TAOS_RES *tres, int code) {
STscObj *pObj = (STscObj *)param;
if (pObj == NULL) return;
if (pObj == NULL) return TSDB_CODE_APP_ERROR;
if (pObj != pObj->signature) {
tscError("heart beat msg, pObj:%p, signature:%p invalid", pObj, pObj->signature);
return;
return TSDB_CODE_APP_ERROR;
}
SSqlObj *pSql = pObj->pHb;
......@@ -128,11 +128,19 @@ void tscProcessHeartBeatRsp(void *param, TAOS_RES *tres, int code) {
if (pRsp->queryId) tscKillQuery(pObj, pRsp->queryId);
if (pRsp->streamId) tscKillStream(pObj, pRsp->streamId);
}
if (pRes->data == NULL) {
pRes->data = calloc(2, sizeof(int32_t));
}
((int32_t*)pRes->data)[0] = htonl(pRsp->totalDnodes);
((int32_t*)pRes->data)[1] = htonl(pRsp->onlineDnodes);
} else {
tscTrace("heart beat failed, code:%d", code);
}
taosTmrReset(tscProcessActivityTimer, tsShellActivityTimer * 500, pObj, tscTmr, &pObj->pTimer);
return code;
}
void tscProcessActivityTimer(void *handle, void *tmrId) {
......@@ -710,15 +718,21 @@ int32_t tscLaunchJoinSubquery(SSqlObj *pSql, int16_t tableIndex, SJoinSubquerySu
int doProcessSql(SSqlObj *pSql) {
SSqlCmd *pCmd = &pSql->cmd;
SSqlRes *pRes = &pSql->res;
int32_t code = TSDB_CODE_SUCCESS;
void *asyncFp = pSql->fp;
if (pCmd->command == TSDB_SQL_SELECT || pCmd->command == TSDB_SQL_FETCH || pCmd->command == TSDB_SQL_RETRIEVE ||
pCmd->command == TSDB_SQL_INSERT || pCmd->command == TSDB_SQL_CONNECT || pCmd->command == TSDB_SQL_HB ||
pCmd->command == TSDB_SQL_META || pCmd->command == TSDB_SQL_METRIC) {
tscBuildMsg[pCmd->command](pSql, NULL);
code = tscBuildMsg[pCmd->command](pSql, NULL);
}
int32_t code = tscSendMsgToServer(pSql);
if (code != TSDB_CODE_SUCCESS) {
pRes->code = code;
return code;
}
code = tscSendMsgToServer(pSql);
if (asyncFp) {
if (code != TSDB_CODE_SUCCESS) {
......@@ -1005,7 +1019,13 @@ int tscLaunchSTableSubqueries(SSqlObj *pSql) {
SRetrieveSupport* pSupport = pSub->param;
tscTrace("%p sub:%p launch subquery, orderOfSub:%d.", pSql, pSub, pSupport->subqueryIndex);
tscProcessSql(pSub);
int code = tscProcessSql(pSub);
if (code != TSDB_CODE_SUCCESS) {
tscLocalReducerEnvDestroy(pMemoryBuf, pDesc, pModel, numOfSubQueries);
doCleanupSubqueries(pSql, i, pState);
pRes->code = code;
return pRes->code;
}
}
return TSDB_CODE_SUCCESS;
......@@ -2713,7 +2733,7 @@ int tscBuildMultiMeterMetaMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
tscTrace("%p build load multi-metermeta msg completed, numOfMeters:%d, msg size:%d", pSql, pCmd->count,
pCmd->payloadLen);
return pCmd->payloadLen;
return TSDB_CODE_SUCCESS;
}
static int32_t tscEstimateMetricMetaMsgSize(SSqlCmd *pCmd) {
......@@ -2940,7 +2960,7 @@ int tscBuildHeartBeatMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
pCmd->msgType = TSDB_MSG_TYPE_HEARTBEAT;
assert(msgLen + minMsgSize() <= size);
return msgLen;
return TSDB_CODE_SUCCESS;
}
int tscProcessMeterMetaRsp(SSqlObj *pSql) {
......@@ -3700,7 +3720,8 @@ int tscGetMetricMeta(SSqlObj *pSql, int32_t clauseIndex) {
for (int32_t i = 0; i < pQueryInfo->numOfTables; ++i) {
SMeterMetaInfo *pMMInfo = tscGetMeterMetaInfoFromQueryInfo(pQueryInfo, i);
SMeterMeta *pMeterMeta = taosGetDataFromCache(tscCacheHandle, pMMInfo->name);
SMeterMeta *pMeterMeta = (SMeterMeta *)taosGetDataFromExists(tscCacheHandle, pQueryInfo->pMeterInfo[i]->pMeterMeta);
assert(pMeterMeta != NULL);
tscAddMeterMetaInfo(pNewQueryInfo, pMMInfo->name, pMeterMeta, NULL, pMMInfo->numOfTags, pMMInfo->tagColumnIndex);
}
......
src/client/src/tscSql.c
浏览文件 @ 438174a6
......@@ -321,7 +321,7 @@ TAOS_FIELD *taos_fetch_fields(TAOS_RES *res) {
if (pSql == NULL || pSql->signature != pSql) return 0;
SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(&pSql->cmd, 0);
if(NULL == pQueryInfo) {
if(NULL == pQueryInfo){
return NULL;
}
return pQueryInfo->fieldsInfo.pFields;
......@@ -756,6 +756,7 @@ TAOS_ROW taos_fetch_row(TAOS_RES *res) {
}
int taos_fetch_block(TAOS_RES *res, TAOS_ROW *rows) {
#if 0
SSqlObj *pSql = (SSqlObj *)res;
SSqlCmd *pCmd = &pSql->cmd;
SSqlRes *pRes = &pSql->res;
......@@ -795,6 +796,11 @@ int taos_fetch_block(TAOS_RES *res, TAOS_ROW *rows) {
}
return nRows;
#endif
(*rows) = taos_fetch_row(res);
return ((*rows) != NULL)? 1:0;
}
int taos_select_db(TAOS *taos, const char *db) {
......
src/client/src/tscStream.c
浏览文件 @ 438174a6
......@@ -28,7 +28,7 @@
static void tscProcessStreamQueryCallback(void *param, TAOS_RES *tres, int numOfRows);
static void tscProcessStreamRetrieveResult(void *param, TAOS_RES *res, int numOfRows);
static void tscSetNextLaunchTimer(SSqlStream *pStream, SSqlObj *pSql);
void tscSetNextLaunchTimer(SSqlStream *pStream, SSqlObj *pSql);
static void tscSetRetryTimer(SSqlStream *pStream, SSqlObj *pSql, int64_t timer);
static int64_t getDelayValueAfterTimewindowClosed(SSqlStream* pStream, int64_t launchDelay) {
......@@ -97,6 +97,18 @@ static void tscProcessStreamLaunchQuery(SSchedMsg *pMsg) {
return;
}
if ((UTIL_METER_IS_SUPERTABLE(pMeterMetaInfo)
&& ( pMeterMetaInfo->pMeterMeta == NULL
|| pMeterMetaInfo->pMetricMeta == NULL
|| pMeterMetaInfo->pMetricMeta->numOfMeters == 0
|| pMeterMetaInfo->pMetricMeta->numOfVnodes == 0))
|| (!(UTIL_METER_IS_SUPERTABLE(pMeterMetaInfo)) && (pMeterMetaInfo->pMeterMeta == NULL))) {
tscTrace("%p no table in metricmeta, no launch query", pSql);
tscClearMeterMetaInfo(pMeterMetaInfo, false);
tscSetNextLaunchTimer(pStream, pSql);
return;
}
tscTrace("%p stream:%p start stream query on:%s", pSql, pStream, pMeterMetaInfo->name);
tscProcessSql(pStream->pSql);
......@@ -323,7 +335,7 @@ static int64_t getLaunchTimeDelay(const SSqlStream* pStream) {
}
static void tscSetNextLaunchTimer(SSqlStream *pStream, SSqlObj *pSql) {
void tscSetNextLaunchTimer(SSqlStream *pStream, SSqlObj *pSql) {
int64_t timer = 0;
SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(&pSql->cmd, 0);
......
src/client/src/tscSyntaxtreefunction.c
浏览文件 @ 438174a6
......@@ -312,7 +312,7 @@ void calc_fn_i32_i32_sub(void *left, void *right, int32_t numLeft, int32_t numRi
if (numLeft == numRight) {
for (; i >= 0 && i < numRight; i += step, pOutput += 1) {
if (isNull((char *)&(pLeft[i]), TSDB_DATA_TYPE_INT) || isNull((char *)&(pRight[i]), TSDB_DATA_TYPE_INT)) {
setNull((char *)&(pOutput[i]), TSDB_DATA_TYPE_DOUBLE, tDataTypeDesc[TSDB_DATA_TYPE_DOUBLE].nSize);
setNull((char *)&(pOutput), TSDB_DATA_TYPE_DOUBLE, tDataTypeDesc[TSDB_DATA_TYPE_DOUBLE].nSize);
continue;
}
*pOutput = (double)pLeft[i] - pRight[i];
......
src/client/src/tscUtil.c
浏览文件 @ 438174a6
......@@ -457,9 +457,11 @@ void tscFreeSqlObjPartial(SSqlObj* pSql) {
pCmd->command = 0;
// pSql->sqlstr will be used by tscBuildQueryStreamDesc
if (pObj->signature == pObj) {
pthread_mutex_lock(&pObj->mutex);
tfree(pSql->sqlstr);
pthread_mutex_unlock(&pObj->mutex);
}
tscFreeSqlResult(pSql);
tfree(pSql->pSubs);
......@@ -2024,7 +2026,7 @@ SSqlObj* createSubqueryObj(SSqlObj* pSql, int16_t tableIndex, void (*fp)(), void
}
// create the fields info from the sql functions
SColumnList columnList = {.num = 1};
SColumnList columnList = {.num = 0};
// for avg/last/first/histo.. query, the output type is binary not numeric data type
for(int32_t k = 0; k < numOfOutputCols; ++k) {
......@@ -2079,7 +2081,14 @@ SSqlObj* createSubqueryObj(SSqlObj* pSql, int16_t tableIndex, void (*fp)(), void
pMeterMetaInfo->tagColumnIndex);
}
assert(pFinalInfo->pMeterMeta != NULL && pNewQueryInfo->numOfTables == 1);
if (pFinalInfo->pMeterMeta == NULL) {
tscError("%p new subquery failed for get pMeterMeta is NULL from cache", pSql);
tscFreeSqlObj(pNew);
return NULL;
}
assert(pNewQueryInfo->numOfTables == 1);
if (UTIL_METER_IS_SUPERTABLE(pMeterMetaInfo)) {
assert(pFinalInfo->pMetricMeta != NULL);
}
......
src/connector/jdbc/pom.xml
浏览文件 @ 438174a6
......@@ -37,7 +37,7 @@
<properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<java.version>1.8</java.version>
<java.version>1.7</java.version>
<maven-compiler-plugin.version>3.6.0</maven-compiler-plugin.version>
......
src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBConnection.java
浏览文件 @ 438174a6
......@@ -84,6 +84,14 @@ public class TSDBConnection implements Connection {
}
}
public TSDBSubscribe createSubscribe() throws SQLException {
if (!this.connector.isClosed()) {
return new TSDBSubscribe(this.connector);
} else {
throw new SQLException(TSDBConstants.FixErrMsg(TSDBConstants.JNI_CONNECTION_NULL));
}
}
public PreparedStatement prepareStatement(String sql) throws SQLException {
if (!this.connector.isClosed()) {
return new TSDBPreparedStatement(this.connector, sql);
......
src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBJNIConnector.java
浏览文件 @ 438174a6
......@@ -22,8 +22,8 @@ public class TSDBJNIConnector {
static volatile Boolean isInitialized = false;
static {
System.loadLibrary("taos");
System.out.println("java.library.path:" + System.getProperty("java.library.path"));
System.loadLibrary("taos");
}
/**
......@@ -261,31 +261,31 @@ public class TSDBJNIConnector {
/**
* Subscribe to a table in TSDB
*/
public long subscribe(String host, String user, String password, String database, String table, long time, int period) {
return subscribeImp(host, user, password, database, table, time, period);
public long subscribe(String topic, String sql, boolean restart, int period) {
return subscribeImp(this.taos, restart, topic, sql, period);
}
private native long subscribeImp(String host, String user, String password, String database, String table, long time, int period);
public native long subscribeImp(long connection, boolean restart, String topic, String sql, int period);
/**
* Consume a subscribed table
*/
public TSDBResultSetRowData consume(long subscription) {
public long consume(long subscription) {
return this.consumeImp(subscription);
}
private native TSDBResultSetRowData consumeImp(long subscription);
private native long consumeImp(long subscription);
/**
* Unsubscribe a table
*
* @param subscription
*/
public void unsubscribe(long subscription) {
unsubscribeImp(subscription);
public void unsubscribe(long subscription, boolean isKeep) {
unsubscribeImp(subscription, isKeep);
}
private native void unsubscribeImp(long subscription);
private native void unsubscribeImp(long subscription, boolean isKeep);
/**
* Validate if a <I>create table</I> sql statement is correct without actually creating that table
......@@ -293,7 +293,7 @@ public class TSDBJNIConnector {
public boolean validateCreateTableSql(String sql) {
long connection = taos;
int res = validateCreateTableSqlImp(connection, sql.getBytes());
return res != 0 ? false : true;
return res == 0;
}
private native int validateCreateTableSqlImp(long connection, byte[] sqlBytes);
......
src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBSubscribe.java 0 → 100644
浏览文件 @ 438174a6
/***************************************************************************
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*****************************************************************************/
package com.taosdata.jdbc;
import javax.management.OperationsException;
import java.sql.SQLException;
import java.util.Map;
import java.util.TimerTask;
import java.util.concurrent.*;
public class TSDBSubscribe {
private TSDBJNIConnector connecter = null;
private static ScheduledExecutorService pool;
private static Map<Long, TSDBTimerTask> timerTaskMap = new ConcurrentHashMap<>();
private static Map<Long, ScheduledFuture> scheduledMap = new ConcurrentHashMap();
private static class TimerInstance {
private static final ScheduledExecutorService instance = Executors.newScheduledThreadPool(1);
}
public static ScheduledExecutorService getTimerInstance() {
return TimerInstance.instance;
}
public TSDBSubscribe(TSDBJNIConnector connecter) throws SQLException {
if (null != connecter) {
this.connecter = connecter;
} else {
throw new SQLException(TSDBConstants.FixErrMsg(TSDBConstants.JNI_CONNECTION_NULL));
}
}
/**
* sync subscribe
*
* @param topic
* @param sql
* @param restart
* @param period
* @throws SQLException
*/
public long subscribe(String topic, String sql, boolean restart, int period) throws SQLException {
if (this.connecter.isClosed()) {
throw new SQLException(TSDBConstants.FixErrMsg(TSDBConstants.JNI_CONNECTION_NULL));
}
if (period < 1000) {
throw new SQLException(TSDBConstants.WrapErrMsg(TSDBConstants.INVALID_VARIABLES));
}
return this.connecter.subscribe(topic, sql, restart, period);
}
/**
* async subscribe
*
* @param topic
* @param sql
* @param restart
* @param period
* @param callBack
* @throws SQLException
*/
public long subscribe(String topic, String sql, boolean restart, int period, TSDBSubscribeCallBack callBack) throws SQLException {
if (this.connecter.isClosed()) {
throw new SQLException(TSDBConstants.FixErrMsg(TSDBConstants.JNI_CONNECTION_NULL));
}
final long subscription = this.connecter.subscribe(topic, sql, restart, period);
if (null != callBack) {
pool = getTimerInstance();
TSDBTimerTask timerTask = new TSDBTimerTask(subscription, callBack);
timerTaskMap.put(subscription, timerTask);
ScheduledFuture scheduledFuture = pool.scheduleAtFixedRate(timerTask, 1, 1000, TimeUnit.MILLISECONDS);
scheduledMap.put(subscription, scheduledFuture);
}
return subscription;
}
public TSDBResultSet consume(long subscription) throws OperationsException, SQLException {
if (this.connecter.isClosed()) {
throw new SQLException(TSDBConstants.FixErrMsg(TSDBConstants.JNI_CONNECTION_NULL));
}
if (0 == subscription) {
throw new OperationsException("Invalid use of consume");
}
long resultSetPointer = this.connecter.consume(subscription);
if (resultSetPointer == TSDBConstants.JNI_CONNECTION_NULL) {
throw new SQLException(TSDBConstants.FixErrMsg(TSDBConstants.JNI_CONNECTION_NULL));
} else if (resultSetPointer == TSDBConstants.JNI_NULL_POINTER) {
return null;
} else {
return new TSDBResultSet(this.connecter, resultSetPointer);
}
}
/**
* cancel subscribe
*
* @param subscription
* @param isKeep
* @throws SQLException
*/
public void unsubscribe(long subscription, boolean isKeep) throws SQLException {
if (this.connecter.isClosed()) {
throw new SQLException(TSDBConstants.FixErrMsg(TSDBConstants.JNI_CONNECTION_NULL));
}
if (null != timerTaskMap.get(subscription)) {
synchronized (timerTaskMap.get(subscription)) {
while (1 == timerTaskMap.get(subscription).getState()) {
try {
Thread.sleep(10);
} catch (Exception e) {
throw new RuntimeException(e);
}
}
timerTaskMap.get(subscription).setState(2);
if (!timerTaskMap.isEmpty() && timerTaskMap.containsKey(subscription)) {
timerTaskMap.get(subscription).cancel();
timerTaskMap.remove(subscription);
scheduledMap.get(subscription).cancel(false);
scheduledMap.remove(subscription);
}
this.connecter.unsubscribe(subscription, isKeep);
}
} else {
this.connecter.unsubscribe(subscription, isKeep);
}
}
class TSDBTimerTask extends TimerTask {
private long subscription;
private TSDBSubscribeCallBack callBack;
// 0: not running 1: running 2: cancel
private int state = 0;
public TSDBTimerTask(long subscription, TSDBSubscribeCallBack callBack) {
this.subscription = subscription;
this.callBack = callBack;
}
public int getState() {
return this.state;
}
public void setState(int state) {
this.state = state;
}
@Override
public void run() {
synchronized (this) {
if (2 == state) {
return;
}
state = 1;
try {
TSDBResultSet resultSet = consume(subscription);
callBack.invoke(resultSet);
} catch (Exception e) {
this.cancel();
throw new RuntimeException(e);
}
state = 0;
}
}
}
}
src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBSubscribeCallBack.java 0 → 100644
浏览文件 @ 438174a6
/***************************************************************************
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*****************************************************************************/
package com.taosdata.jdbc;
public interface TSDBSubscribeCallBack {
void invoke(TSDBResultSet resultSet);
}
src/connector/jdbc/src/test/java/TestAsyncTSDBSubscribe.java 0 → 100644
浏览文件 @ 438174a6
import com.taosdata.jdbc.*;
import org.apache.commons.lang3.StringUtils;
import java.sql.Connection;
import java.sql.DriverManager;
import java.util.Properties;
public class TestAsyncTSDBSubscribe {
public static void main(String[] args) {
String usage = "java -cp taos-jdbcdriver-1.0.3_dev-dist.jar com.taosdata.jdbc.TSDBSubscribe -db dbName -topic topicName " +
"-tname tableName -h host";
if (args.length < 2) {
System.err.println(usage);
return;
}
String dbName = "";
String tName = "";
String host = "localhost";
String topic = "";
for (int i = 0; i < args.length; i++) {
if ("-db".equalsIgnoreCase(args[i]) && i < args.length - 1) {
dbName = args[++i];
}
if ("-tname".equalsIgnoreCase(args[i]) && i < args.length - 1) {
tName = args[++i];
}
if ("-h".equalsIgnoreCase(args[i]) && i < args.length - 1) {
host = args[++i];
}
if ("-topic".equalsIgnoreCase(args[i]) && i < args.length - 1) {
topic = args[++i];
}
}
if (StringUtils.isEmpty(dbName) || StringUtils.isEmpty(tName) || StringUtils.isEmpty(topic)) {
System.err.println(usage);
return;
}
Connection connection = null;
TSDBSubscribe subscribe = null;
long subscribId = 0;
try {
Class.forName("com.taosdata.jdbc.TSDBDriver");
Properties properties = new Properties();
properties.setProperty(TSDBDriver.PROPERTY_KEY_HOST, host);
connection = DriverManager.getConnection("jdbc:TAOS://" + host + ":0/" + dbName + "?user=root&password=taosdata", properties);
String rawSql = "select * from " + tName + ";";
subscribe = ((TSDBConnection) connection).createSubscribe();
subscribId = subscribe.subscribe(topic, rawSql, false, 1000, new CallBack("first"));
long subscribId2 = subscribe.subscribe("test", rawSql, false, 1000, new CallBack("second"));
int a = 0;
Thread.sleep(2000);
subscribe.unsubscribe(subscribId, true);
System.err.println("cancel subscribe");
} catch (Exception e) {
e.printStackTrace();
}
}
private static class CallBack implements TSDBSubscribeCallBack {
private String name = "";
public CallBack(String name) {
this.name = name;
}
@Override
public void invoke(TSDBResultSet resultSet) {
try {
while (null !=resultSet && resultSet.next()) {
System.out.print("callback_" + name + ": ");
for (int i = 1; i <= resultSet.getMetaData().getColumnCount(); i++) {
System.out.printf(i + ": " + resultSet.getString(i) + "\t");
}
System.out.println();
}
} catch (Exception e) {
e.printStackTrace();
}
}
}
}
src/connector/jdbc/src/test/java/TestPreparedStatement.java
浏览文件 @ 438174a6
......@@ -11,8 +11,14 @@ public class TestPreparedStatement {
Class.forName("com.taosdata.jdbc.TSDBDriver");
Properties properties = new Properties();
properties.setProperty(TSDBDriver.PROPERTY_KEY_HOST, "192.168.1.117");
Connection connection = DriverManager.getConnection("jdbc:TAOS://192.168.1.117:0/?user=root&password=taosdata", properties);
Connection connection = DriverManager.getConnection("jdbc:TAOS://10.211.55.3:0/log?user=root&password=taosdata", properties);
String createSql = "create table t (ts timestamp, speed int);";
Statement statement = connection.createStatement();
statement.executeQuery(createSql);
String rawSql = "SELECT ts, c1 FROM (select c1, ts from db.tb1) SUB_QRY";
if (1 < 2) {
return;
}
// String[] params = new String[]{"ts", "c1"};
PreparedStatement pstmt = (TSDBPreparedStatement) connection.prepareStatement(rawSql);
ResultSet resSet = pstmt.executeQuery();
......
src/connector/jdbc/src/test/java/TestTSDBResultSetRowData.java 0 → 100644
浏览文件 @ 438174a6
import com.taosdata.jdbc.ColumnMetaData;
import com.taosdata.jdbc.DatabaseMetaDataResultSet;
import com.taosdata.jdbc.TSDBResultSetRowData;
import java.sql.SQLException;
import java.util.ArrayList;
import java.util.List;
public class TestTSDBResultSetRowData {
public static void main(String[] args) throws SQLException {
DatabaseMetaDataResultSet resultSet = new DatabaseMetaDataResultSet();
List<ColumnMetaData> columnMetaDataList = new ArrayList(1);
ColumnMetaData colMetaData = new ColumnMetaData();
colMetaData.setColIndex(0);
colMetaData.setColName("TABLE_TYPE");
colMetaData.setColSize(10);
colMetaData.setColType(8);
columnMetaDataList.add(colMetaData);
List<TSDBResultSetRowData> rowDataList = new ArrayList(2);
TSDBResultSetRowData rowData = new TSDBResultSetRowData(2);
rowData.setString(0, "TABLE");
rowDataList.add(rowData);
rowData = new TSDBResultSetRowData(2);
rowData.setString(0, "STABLE");
rowDataList.add(rowData);
resultSet.setColumnMetaDataList(columnMetaDataList);
resultSet.setRowDataList(rowDataList);
while (resultSet.next()) {
System.out.println(resultSet.getString(1));
}
}
}
src/connector/jdbc/src/test/java/TestTSDBSubscribe.java 0 → 100644
浏览文件 @ 438174a6
import com.taosdata.jdbc.TSDBConnection;
import com.taosdata.jdbc.TSDBDriver;
import com.taosdata.jdbc.TSDBResultSet;
import com.taosdata.jdbc.TSDBSubscribe;
import org.apache.commons.lang3.StringUtils;
import java.sql.Connection;
import java.sql.DriverManager;
import java.util.Properties;
public class TestTSDBSubscribe {
public static void main(String[] args) throws Exception {
String usage = "java -cp taos-jdbcdriver-1.0.3_dev-dist.jar com.taosdata.jdbc.TSDBSubscribe -db dbName " +
"-topic topicName -tname tableName -h host";
if (args.length < 2) {
System.err.println(usage);
return;
}
String dbName = "";
String tName = "";
String host = "localhost";
String topic = "";
for (int i = 0; i < args.length; i++) {
if ("-db".equalsIgnoreCase(args[i]) && i < args.length - 1) {
dbName = args[++i];
}
if ("-tname".equalsIgnoreCase(args[i]) && i < args.length - 1) {
tName = args[++i];
}
if ("-h".equalsIgnoreCase(args[i]) && i < args.length - 1) {
host = args[++i];
}
if ("-topic".equalsIgnoreCase(args[i]) && i < args.length - 1) {
topic = args[++i];
}
}
if (StringUtils.isEmpty(dbName) || StringUtils.isEmpty(tName) || StringUtils.isEmpty(topic)) {
System.err.println(usage);
return;
}
Connection connection = null;
TSDBSubscribe subscribe = null;
long subscribId = 0;
try {
Class.forName("com.taosdata.jdbc.TSDBDriver");
Properties properties = new Properties();
properties.setProperty(TSDBDriver.PROPERTY_KEY_HOST, host);
connection = DriverManager.getConnection("jdbc:TAOS://" + host + ":0/" + dbName + "?user=root&password=taosdata"
, properties);
String rawSql = "select * from " + tName + ";";
subscribe = ((TSDBConnection) connection).createSubscribe();
subscribId = subscribe.subscribe(topic, rawSql, false, 1000);
int a = 0;
while (true) {
Thread.sleep(900);
TSDBResultSet resSet = subscribe.consume(subscribId);
while (resSet.next()) {
for (int i = 1; i <= resSet.getMetaData().getColumnCount(); i++) {
System.out.printf(i + ": " + resSet.getString(i) + "\t");
}
System.out.println("\n======" + a + "==========");
}
a++;
if (a >= 10) {
break;
}
}
} catch (Exception e) {
e.printStackTrace();
} finally {
if (null != subscribe && 0 != subscribId) {
subscribe.unsubscribe(subscribId, true);
}
if (null != connection) {
connection.close();
}
}
}
}
src/connector/python/linux/python2/taos/cinterface.py
浏览文件 @ 438174a6
......@@ -316,6 +316,9 @@ class CTaosInterface(object):
blocks = [None] * len(fields)
for i in range(len(fields)):
data = ctypes.cast(pblock, ctypes.POINTER(ctypes.c_void_p))[i]
if data == None:
blocks[i] = [None] * num_of_rows
continue
if fields[i]['type'] not in _CONVERT_FUNC:
raise DatabaseError("Invalid data type returned from database")
......
src/connector/python/linux/python3/taos/cinterface.py
浏览文件 @ 438174a6
......@@ -316,6 +316,9 @@ class CTaosInterface(object):
blocks = [None] * len(fields)
for i in range(len(fields)):
data = ctypes.cast(pblock, ctypes.POINTER(ctypes.c_void_p))[i]
if data == None:
blocks[i] = [None] * num_of_rows
continue
if fields[i]['type'] not in _CONVERT_FUNC:
raise DatabaseError("Invalid data type returned from database")
......
src/inc/taoserror.h
浏览文件 @ 438174a6
......@@ -138,8 +138,9 @@ extern "C" {
#define TSDB_CODE_FAILED_TO_LOCK_RESOURCES 117
#define TSDB_CODE_TABLE_ID_MISMATCH 118
#define TSDB_CODE_QUERY_CACHE_ERASED 119
#define TSDB_CODE_AUTH_BANNED_PERIOD 120
#define TSDB_CODE_MAX_ERROR_CODE 120
#define TSDB_CODE_MAX_ERROR_CODE 121
#ifdef __cplusplus
}
......
src/inc/taosmsg.h
浏览文件 @ 438174a6
......@@ -439,9 +439,15 @@ typedef struct SSqlFuncExprMsg {
} SSqlFuncExprMsg;
typedef struct SSqlBinaryExprInfo {
union {
struct tSQLBinaryExpr *pBinExpr; /* for binary expression */
int64_t resvSpace0;
};
int32_t numOfCols; /* binary expression involves the readed number of columns*/
union {
SColIndexEx * pReqColumns; /* source column list */
int64_t resvSpace1;
};
} SSqlBinaryExprInfo;
typedef struct SSqlFunctionExpr {
......@@ -482,7 +488,11 @@ typedef struct SColumnInfo {
int16_t type;
int16_t bytes;
int16_t numOfFilters;
union {
SColumnFilterInfo *filters;
int64_t resvSpace;
};
} SColumnInfo;
/*
......@@ -811,6 +821,8 @@ typedef struct {
typedef struct {
uint32_t queryId;
uint32_t streamId;
uint32_t totalDnodes;
uint32_t onlineDnodes;
char killConnection;
SIpList ipList;
} SHeartBeatRsp;
......
src/inc/tglobalcfg.h
浏览文件 @ 438174a6
......@@ -150,6 +150,7 @@ extern int tsHttpMaxThreads;
extern int tsHttpEnableCompress;
extern int tsHttpEnableRecordSql;
extern int tsTelegrafUseFieldNum;
extern int tsMaxAuthRetry;
extern int tsTscEnableRecordSql;
extern int tsAnyIp;
......@@ -255,7 +256,7 @@ extern int tsGlobalConfigNum;
extern char * tsCfgStatusStr[];
SGlobalConfig *tsGetConfigOption(const char *option);
#define TSDB_CFG_MAX_NUM 110
#define TSDB_CFG_MAX_NUM 111
#define TSDB_CFG_PRINT_LEN 23
#define TSDB_CFG_OPTION_LEN 24
#define TSDB_CFG_VALUE_LEN 41
......
src/inc/trpc.h
浏览文件 @ 438174a6
......@@ -62,6 +62,7 @@ typedef struct {
void (*efp)(int cid); // call back function to process not activated chann
int (*afp)(char *meterId, char *spi, char *encrypt, uint8_t *secret,
uint8_t *ckey); // call back to retrieve auth info
int (*ufp)(char *user, int32_t *failCount, int32_t *allowTime, bool opSet); // callback to update auth retry info
} SRpcInit;
typedef struct {
......
src/inc/ttime.h
浏览文件 @ 438174a6
......@@ -44,6 +44,7 @@ int32_t getTimestampInUsFromStr(char* token, int32_t tokenlen, int64_t* ts);
int32_t taosParseTime(char* timestr, int64_t* time, int32_t len, int32_t timePrec);
void deltaToUtcInitOnce();
int32_t taosTimeSecToString(time_t ts, char* outstr);
#ifdef __cplusplus
}
#endif
......
src/inc/tutil.h
浏览文件 @ 438174a6
......@@ -130,6 +130,7 @@ extern "C" {
#define POW2(x) ((x) * (x))
size_t twcslen(const wchar_t *wcs);
int32_t strdequote(char *src);
void strtrim(char *src);
......
src/kit/shell/CMakeLists.txt
浏览文件 @ 438174a6
......@@ -19,7 +19,7 @@ IF ((TD_LINUX_64) OR (TD_LINUX_32 AND TD_ARM))
ENDIF ()
SET_TARGET_PROPERTIES(shell PROPERTIES OUTPUT_NAME taos)
ELSEIF (TD_WINDOWS_64)
ELSEIF (TD_WINDOWS_64 OR TD_WINDOWS_32)
INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/pthread)
INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/regex)
LIST(APPEND SRC ./src/shellEngine.c)
......
src/kit/shell/src/shellEngine.c
浏览文件 @ 438174a6
......@@ -808,7 +808,8 @@ void shellGetGrantInfo(void *con) {
if (code == TSDB_CODE_OPS_NOT_SUPPORT) {
fprintf(stdout, "Server is Community Edition, version is %s\n\n", taos_get_server_info(con));
} else {
fprintf(stderr, "Failed to check Server Edition, Reason:%d:%s\n\n", taos_errno(con), taos_errstr(con));
//fprintf(stderr, "Failed to check Server Edition, Reason:%d:%s\n\n", taos_errno(con), taos_errstr(con));
fprintf(stdout, "Server is Enterprise Edition, version is %s\n\n", taos_get_server_info(con));
}
return;
}
......
src/kit/taosdump/taosdump.c
浏览文件 @ 438174a6
......@@ -27,6 +27,7 @@
#include <unistd.h>
#include <wordexp.h>
#include <iconv.h>
#include <time.h>
#include "taos.h"
#include "taosmsg.h"
......@@ -162,6 +163,7 @@ static struct argp_option options[] = {
{"password", 'p', "PASSWORD", 0, "User password to connect to server. Default is taosdata.", 0},
{"port", 'P', "PORT", 0, "Port to connect", 0},
{"cversion", 'v', "CVERION", 0, "client version", 0},
{"mysqlFlag", 'q', "MYSQLFLAG", 0, "mysqlFlag, Default is 0", 0},
// input/output file
{"outpath", 'o', "OUTPATH", 0, "Output file path.", 1},
{"inpath", 'i', "INPATH", 0, "Input file path.", 1},
......@@ -189,6 +191,7 @@ struct arguments {
char *password;
uint16_t port;
char cversion[TSDB_FILENAME_LEN+1];
uint16_t mysqlFlag;
// output file
char outpath[TSDB_FILENAME_LEN+1];
char inpath[TSDB_FILENAME_LEN+1];
......@@ -236,6 +239,9 @@ static error_t parse_opt(int key, char *arg, struct argp_state *state) {
case 'P':
arguments->port = atoi(arg);
break;
case 'q':
arguments->mysqlFlag = atoi(arg);
break;
case 'v':
if (wordexp(arg, &full_path, 0) != 0) {
fprintf(stderr, "Invalid client vesion %s\n", arg);
......@@ -341,6 +347,7 @@ struct arguments tsArguments = {
"taosdata",
0,
"",
0,
// outpath and inpath
"",
"",
......@@ -385,6 +392,7 @@ int main(int argc, char *argv[]) {
printf("password: %s\n", tsArguments.password);
printf("port: %u\n", tsArguments.port);
printf("cversion: %s\n", tsArguments.cversion);
printf("mysqlFlag: %d", tsArguments.mysqlFlag);
printf("outpath: %s\n", tsArguments.outpath);
printf("inpath: %s\n", tsArguments.inpath);
printf("encode: %s\n", tsArguments.encode);
......@@ -918,6 +926,8 @@ void taosDumpCreateDbClause(SDbInfo *dbInfo, bool isDumpProperty, FILE *fp) {
dbInfo->ablocks, dbInfo->tblocks, dbInfo->ctime, dbInfo->clog, dbInfo->comp);
}
pstr += sprintf(pstr, ";");
fprintf(fp, "%s\n\n", tmpCommand);
free(tmpCommand);
}
......@@ -1236,7 +1246,7 @@ void taosDumpCreateTableClause(STableDef *tableDes, int numOfCols, FILE *fp) {
}
}
pstr += sprintf(pstr, ")");
pstr += sprintf(pstr, ");");
fprintf(fp, "%s\n", tmpBuf);
......@@ -1289,7 +1299,7 @@ void taosDumpCreateMTableClause(STableDef *tableDes, char *metric, int numOfCols
/* } */
}
pstr += sprintf(pstr, ")");
pstr += sprintf(pstr, ");");
fprintf(fp, "%s\n", tmpBuf);
free(tmpBuf);
......@@ -1359,14 +1369,31 @@ int taosDumpTableData(FILE *fp, char *tbname, struct arguments *arguments, TAOS*
return -1;
}
char sqlStr[8] = "\0";
if (arguments->mysqlFlag) {
sprintf(sqlStr, "INSERT");
} else {
sprintf(sqlStr, "IMPORT");
}
int rowFlag = 0;
count = 0;
while ((row = taos_fetch_row(tmpResult)) != NULL) {
pstr = tmpBuffer;
if (count == 0) {
pstr += sprintf(pstr, "IMPORT INTO %s VALUES (", tbname);
pstr += sprintf(pstr, "%s INTO %s VALUES (", sqlStr, tbname);
} else {
if (arguments->mysqlFlag) {
if (0 == rowFlag) {
pstr += sprintf(pstr, "(");
rowFlag++;
} else {
pstr += sprintf(pstr, ", (");
}
} else {
pstr += sprintf(pstr, "(");
}
}
for (int col = 0; col < numFields; col++) {
......@@ -1410,12 +1437,22 @@ int taosDumpTableData(FILE *fp, char *tbname, struct arguments *arguments, TAOS*
pstr += sprintf(pstr, "\'%s\'", tbuf);
break;
case TSDB_DATA_TYPE_TIMESTAMP:
if (!arguments->mysqlFlag) {
pstr += sprintf(pstr, "%" PRId64 "", *(int64_t *)row[col]);
} else {
char buf[64] = "\0";
int64_t ts = *((int64_t *)row[col]);
time_t tt = (time_t)(ts / 1000);
struct tm *ptm = localtime(&tt);
strftime(buf, 64, "%y-%m-%d %H:%M:%S", ptm);
pstr += sprintf(pstr, "\'%s.%03d\'", buf, (int)(ts % 1000));
}
break;
default:
break;
}
}
pstr += sprintf(pstr, ") ");
totalRows++;
......@@ -1423,7 +1460,7 @@ int taosDumpTableData(FILE *fp, char *tbname, struct arguments *arguments, TAOS*
fprintf(fp, "%s", tmpBuffer);
if (count >= arguments->data_batch) {
fprintf(fp, "\n");
fprintf(fp, ";\n");
count = 0;
} //else {
//fprintf(fp, "\\\n");
......
src/os/windows/CMakeLists.txt
浏览文件 @ 438174a6
CMAKE_MINIMUM_REQUIRED(VERSION 2.8)
PROJECT(TDengine)
IF (TD_WINDOWS_64)
IF (TD_WINDOWS_64 OR TD_WINDOWS_32)
INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/pthread)
INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/inc)
INCLUDE_DIRECTORIES(inc)
......
src/os/windows/inc/os.h
浏览文件 @ 438174a6
......@@ -142,8 +142,11 @@ extern "C" {
#define atomic_exchange_16(ptr, val) _InterlockedExchange16((short volatile*)(ptr), (short)(val))
#define atomic_exchange_32(ptr, val) _InterlockedExchange((long volatile*)(ptr), (long)(val))
#define atomic_exchange_64(ptr, val) _InterlockedExchange64((__int64 volatile*)(ptr), (__int64)(val))
#define atomic_exchange_ptr(ptr, val) _InterlockedExchangePointer((void* volatile*)(ptr), (void*)(val))
#if (_MSC_VER == 1800)
#define atomic_exchange_ptr(ptr, val) InterlockedExchangePointer((void* volatile*)(ptr), (void*)(val))
#else
#define atomic_exchange_ptr(ptr, val) _InterlockedExchangePointer((void* volatile*)(ptr), (void*)(val))
#endif
#ifdef _TD_GO_DLL_
#define atomic_val_compare_exchange_8 __sync_val_compare_and_swap
#else
......
src/os/windows/src/twindows.c
浏览文件 @ 438174a6
......@@ -93,7 +93,11 @@ long interlocked_add_fetch_32(long volatile* ptr, long val) {
}
__int64 interlocked_add_fetch_64(__int64 volatile* ptr, __int64 val) {
#ifdef _WIN64
return _InterlockedExchangeAdd64(ptr, val) + val;
#else
return _InterlockedExchangeAdd(ptr, val) + val;
#endif
}
// and
......@@ -377,9 +381,29 @@ int fsendfile(FILE* out_file, FILE* in_file, int64_t* offset, int32_t count) {
return writeLen;
}
unsigned char _MyBitScanForward64(unsigned long *ret, uint64_t x) {
unsigned long x0 = (unsigned long)x, top, bottom;
_BitScanForward(&top, (unsigned long)(x >> 32));
_BitScanForward(&bottom, x0);
*ret = x0 ? bottom : 32 + top;
return x != 0;
}
unsigned char _MyBitScanReverse64(unsigned long *ret, uint64_t x) {
unsigned long x1 = (unsigned long)(x >> 32), top, bottom;
_BitScanReverse(&top, x1);
_BitScanReverse(&bottom, (unsigned long)x);
*ret = x1 ? top + 32 : bottom;
return x != 0;
}
int32_t BUILDIN_CLZL(uint64_t val) {
unsigned long r = 0;
#ifdef _WIN64
_BitScanReverse64(&r, val);
#else
_MyBitScanReverse64(&r, val);
#endif
return (int)(r >> 3);
}
......@@ -391,7 +415,11 @@ int32_t BUILDIN_CLZ(uint32_t val) {
int32_t BUILDIN_CTZL(uint64_t val) {
unsigned long r = 0;
#ifdef _WIN64
_BitScanForward64(&r, val);
#else
_MyBitScanForward64(&r, val);
#endif
return (int)(r >> 3);
}
......
src/rpc/CMakeLists.txt
浏览文件 @ 438174a6
......@@ -12,7 +12,7 @@ ELSEIF (TD_DARWIN_64)
LIST(APPEND SRC ./src/trpc.c)
LIST(APPEND SRC ./src/tstring.c)
LIST(APPEND SRC ./src/tudp.c)
ELSEIF (TD_WINDOWS_64)
ELSEIF (TD_WINDOWS_64 OR TD_WINDOWS_32)
INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/pthread)
LIST(APPEND SRC ./src/thaship.c)
LIST(APPEND SRC ./src/trpc.c)
......
src/rpc/src/trpc.c
浏览文件 @ 438174a6
......@@ -30,6 +30,7 @@
#include "tudp.h"
#include "tutil.h"
#include "lz4.h"
#include "tglobalcfg.h"
typedef struct _msg_node {
struct _msg_node *next;
......@@ -101,6 +102,7 @@ typedef struct rpc_server {
void *(*fp)(char *, void *ahandle, void *thandle);
void (*efp)(int); // FP to report error
int (*afp)(char *meterId, char *spi, char *encrypt, uint8_t *secret, uint8_t *ckey); // FP to retrieve auth info
int (*ufp)(char *user, int32_t *failCount, int32_t *allowTime, bool opSet); // FP to update auth fail retry info
SRpcChann *channList;
} STaosRpc;
......@@ -408,6 +410,7 @@ void *taosOpenRpc(SRpcInit *pRpc) {
pServer->qhandle = pRpc->qhandle;
pServer->efp = pRpc->efp;
pServer->afp = pRpc->afp;
pServer->ufp = pRpc->ufp;
int size = (int)sizeof(SRpcChann) * pRpc->numOfChanns;
pServer->channList = (SRpcChann *)malloc((size_t)size);
......@@ -894,9 +897,11 @@ int taosProcessMsgHeader(STaosHeader *pHeader, SRpcConn **ppConn, STaosRpc *pSer
// authentication
STaosDigest *pDigest = (STaosDigest *)((char *)pHeader + dataLen - sizeof(STaosDigest));
int32_t delta;
int32_t delta, authTime,failedCount,authAllowTime;
delta = (int32_t)htonl(pDigest->timeStamp);
delta -= (int32_t)taosGetTimestampSec();
authTime = (int32_t)taosGetTimestampSec();
if (abs(delta) > 900) {
tWarn("%s cid:%d sid:%d id:%s, time diff:%d is too big, msg discarded pConn:%p, timestamp:%d", pServer->label,
chann, sid, pConn->meterId, delta, pConn, htonl(pDigest->timeStamp));
......@@ -905,9 +910,36 @@ int taosProcessMsgHeader(STaosHeader *pHeader, SRpcConn **ppConn, STaosRpc *pSer
goto _exit;
}
if (pServer->ufp) {
int ret = (*pServer->ufp)(pHeader->meterId,&failedCount,&authAllowTime,false);
if (0 == ret) {
if (authTime < authAllowTime) {
char ipstr[24];
tinet_ntoa(ipstr, ip);
char timestr[50];
taosTimeSecToString((time_t)authAllowTime,timestr);
mLError("user:%s login from %s, authentication not allowed until %s", pHeader->meterId, ipstr,timestr);
tTrace("%s cid:%d sid:%d id:%s, auth not allowed because failed authentication exceeds max limit, msg discarded pConn:%p, until %s", pServer->label, chann, sid,
pConn->meterId, pConn, timestr);
code = TSDB_CODE_AUTH_BANNED_PERIOD;
goto _exit;
}
}else {
code = ret;
goto _exit;
}
}
if (taosAuthenticateMsg((uint8_t *)pHeader, dataLen - TSDB_AUTH_LEN, pDigest->auth, pConn->secret) < 0) {
char ipstr[24];
tinet_ntoa(ipstr, ip);
failedCount++;
if (failedCount >= tsMaxAuthRetry) {
authAllowTime = authTime + 600;//ban the user for 600 seconds
failedCount = 0;
}
(*pServer->ufp)(pHeader->meterId,&failedCount,&authAllowTime,true);
mLError("user:%s login from %s, authentication failed", pHeader->meterId, ipstr);
tError("%s cid:%d sid:%d id:%s, authentication failed, msg discarded pConn:%p", pServer->label, chann, sid,
pConn->meterId, pConn);
......@@ -1152,9 +1184,9 @@ void *taosProcessDataFromPeer(char *data, int dataLen, uint32_t ip, uint16_t por
taosTmrReset(taosProcessIdleTimer, pServer->idleTime, pConn, pChann->tmrCtrl, &pConn->pIdleTimer);
}
if (code == TSDB_CODE_ALREADY_PROCESSED) {
tTrace("%s cid:%d sid:%d id:%s, %s wont be processed, source:0x%08x dest:0x%08x tranId:%d pConn:%p", pServer->label,
chann, sid, pHeader->meterId, taosMsg[pHeader->msgType], pHeader->sourceId, htonl(pHeader->destId),
if (code == TSDB_CODE_ALREADY_PROCESSED || code == TSDB_CODE_LAST_SESSION_NOT_FINISHED) {
tTrace("%s code:%d, cid:%d sid:%d id:%s, %s wont be processed, source:0x%08x dest:0x%08x tranId:%d pConn:%p", pServer->label,
code, chann, sid, pHeader->meterId, taosMsg[pHeader->msgType], pHeader->sourceId, htonl(pHeader->destId),
pHeader->tranId, pConn);
free(data);
return pConn;
......
src/rpc/src/tstring.c
浏览文件 @ 438174a6
......@@ -244,4 +244,5 @@ char *tsError[] = {"success",
"failed to lock resources",
"table id/uid mismatch",
"client query cache erased", // 119
"too many authentication failed, try 10 minutes later", //120
};
src/sdb/src/sdbEngine.c
浏览文件 @ 438174a6
......@@ -812,11 +812,10 @@ void sdbResetTable(SSdbTable *pTable) {
SRowHead *rowHead = NULL;
void * pMetaRow = NULL;
int64_t oldId = pTable->id;
//TODO: check
//int oldNumOfRows = pTable->numOfRows;
int oldNumOfRows = pTable->numOfRows;
if (sdbOpenSdbFile(pTable) < 0) return;
//pTable->numOfRows = oldNumOfRows;
pTable->numOfRows = oldNumOfRows;
total_size = sizeof(SRowHead) + pTable->maxRowSize + sizeof(TSCKSUM);
rowHead = (SRowHead *)malloc(total_size);
......
src/system/detail/inc/mgmt.h
浏览文件 @ 438174a6
......@@ -177,6 +177,8 @@ typedef struct _user_obj {
char reserved[16];
char updateEnd[1];
struct _user_obj *prev, *next;
int32_t authAllowTime;
int32_t authFailCount;
} SUserObj;
typedef struct {
......@@ -272,6 +274,7 @@ int mgmtSendOneFreeVnodeMsg(SVnodeGid *pVnodeGid);
int mgmtInitShell();
void mgmtCleanUpShell();
int mgmtRetriveUserAuthInfo(char *user, char *spi, char *encrypt, uint8_t *secret, uint8_t *ckey);
int mgmtGetSetUserAuthFailInfo(char *user, int32_t *failedCount, int32_t *allowTime, bool opSet);
// acct API
int mgmtInitAccts();
......@@ -429,6 +432,8 @@ bool mgmtAddVnode(SVgObj *pVgroup, SDnodeObj *pSrcDnode, SDnodeObj *pDestDnode);
void mgmtSetModuleInDnode(SDnodeObj *pDnode, int moduleType);
int mgmtUnSetModuleInDnode(SDnodeObj *pDnode, int moduleType);
void mgmtGetDnodeOnlineNum(int32_t *totalDnodes, int32_t *onlineDnodes);
extern int (*mgmtGetMetaFp[])(SMeterMeta *pMeta, SShowObj *pShow, SConnObj *pConn);
extern int (*mgmtRetrieveFp[])(SShowObj *pShow, char *data, int rows, SConnObj *pConn);
......
src/system/detail/src/mgmtShell.c
浏览文件 @ 438174a6
......@@ -87,6 +87,7 @@ int mgmtInitShell() {
rpcInit.idleTime = tsShellActivityTimer * 2000;
rpcInit.qhandle = mgmtQhandle;
rpcInit.afp = mgmtRetriveUserAuthInfo;
rpcInit.ufp = mgmtGetSetUserAuthFailInfo;
pShellConn = taosOpenRpc(&rpcInit);
if (pShellConn == NULL) {
......@@ -1202,6 +1203,10 @@ int mgmtProcessHeartBeatMsg(char *cont, int contLen, SConnObj *pConn) {
pConn->streamId = 0;
pHBRsp->killConnection = pConn->killConnection;
mgmtGetDnodeOnlineNum(&pHBRsp->totalDnodes, &pHBRsp->onlineDnodes);
pHBRsp->totalDnodes = htonl(pHBRsp->totalDnodes);
pHBRsp->onlineDnodes = htonl(pHBRsp->onlineDnodes);
if (pConn->usePublicIp) {
if (pSdbPublicIpList != NULL) {
int size = pSdbPublicIpList->numOfIps * 4;
......@@ -1271,6 +1276,21 @@ int mgmtRetriveUserAuthInfo(char *user, char *spi, char *encrypt, uint8_t *secre
return 0;
}
int mgmtGetSetUserAuthFailInfo(char *user, int32_t *failCount, int32_t *allowTime, bool opSet) {
SUserObj *pUser = NULL;
pUser = mgmtGetUser(user);
if (pUser == NULL) return TSDB_CODE_INVALID_USER;
if (opSet) {
pUser->authAllowTime = *allowTime;
pUser->authFailCount = *failCount;
}else {
*allowTime = pUser->authAllowTime;
*failCount = pUser->authFailCount;
}
return 0;
}
int mgmtProcessConnectMsg(char *pMsg, int msgLen, SConnObj *pConn) {
STaosRsp * pRsp;
SConnectRsp *pConnectRsp;
......
src/system/detail/src/mgmtSupertableQuery.c
浏览文件 @ 438174a6
......@@ -577,7 +577,7 @@ static int32_t compareWStrPatternComp(const void* pLeft, const void* pRight) {
const wchar_t* pattern = pRight;
const wchar_t* str = pLeft;
int32_t ret = WCSPatternMatch(pattern, str, wcslen(str), &pInfo);
int32_t ret = WCSPatternMatch(pattern, str, twcslen(str), &pInfo);
return (ret == TSDB_PATTERN_MATCH) ? 0 : 1;
}
......
src/system/detail/src/vnodeFile.c
浏览文件 @ 438174a6
......@@ -1452,7 +1452,8 @@ int vnodeForwardStartPosition(SQuery *pQuery, SCompBlock *pBlock, int32_t slotId
int step = QUERY_IS_ASC_QUERY(pQuery) ? 1 : -1;
if (pQuery->limit.offset > 0 && pQuery->numOfFilterCols == 0) {
int maxReads = QUERY_IS_ASC_QUERY(pQuery) ? pBlock->numOfPoints - pQuery->pos : pQuery->pos + 1;
assert(pBlock[slotIdx].numOfPoints > pQuery->pos);
int maxReads = QUERY_IS_ASC_QUERY(pQuery) ? pBlock[slotIdx].numOfPoints - pQuery->pos : pQuery->pos + 1;
if (pQuery->limit.offset < maxReads) { // start position in current block
if (QUERY_IS_ASC_QUERY(pQuery)) {
......
src/system/detail/src/vnodeQueryImpl.c
浏览文件 @ 438174a6
......@@ -35,6 +35,8 @@
#include "vnodeQueryImpl.h"
#include "vnodeStatus.h"
#include <dirent.h>
enum {
TS_JOIN_TS_EQUAL = 0,
TS_JOIN_TS_NOT_EQUALS = 1,
......@@ -1529,7 +1531,7 @@ static STimeWindow getActiveTimeWindow(SWindowResInfo *pWindowResInfo, int64_t t
w.ekey = w.skey + pQuery->intervalTime - 1;
}
assert(ts >= w.skey && ts <= w.ekey && w.skey != 0);
assert(ts >= w.skey && ts <= w.ekey/* && w.skey != 0*/);
return w;
}
......@@ -1646,7 +1648,7 @@ static void doCheckQueryCompleted(SQueryRuntimeEnv *pRuntimeEnv, TSKEY lastKey,
setQueryStatus(pQuery, QUERY_COMPLETED | QUERY_RESBUF_FULL);
} else { // set the current index to be the last unclosed window
int32_t i = 0;
int64_t skey = 0;
int64_t skey = INT64_MIN;
for (i = 0; i < pWindowResInfo->size; ++i) {
SWindowResult *pResult = &pWindowResInfo->pResult[i];
......@@ -1668,7 +1670,7 @@ static void doCheckQueryCompleted(SQueryRuntimeEnv *pRuntimeEnv, TSKEY lastKey,
}
// all windows are closed, set the last one to be the skey
if (skey == 0) {
if (skey == INT64_MIN) {
assert(i == pWindowResInfo->size);
pWindowResInfo->curIndex = pWindowResInfo->size - 1;
} else {
......@@ -1686,7 +1688,7 @@ static void doCheckQueryCompleted(SQueryRuntimeEnv *pRuntimeEnv, TSKEY lastKey,
dTrace("QInfo:%p total window:%d, closed:%d", GET_QINFO_ADDR(pQuery), pWindowResInfo->size, n);
}
assert(pWindowResInfo->prevSKey != 0);
assert(pWindowResInfo->prevSKey != INT64_MIN);
}
static int32_t getNumOfRowsInTimeWindow(SQuery *pQuery, SBlockInfo *pBlockInfo, TSKEY *pPrimaryColumn, int32_t startPos,
......@@ -2799,6 +2801,7 @@ static int32_t rowwiseApplyAllFunctions(SQueryRuntimeEnv *pRuntimeEnv, int32_t *
}
}
if (lastIndex >= 0) {
// save the last accessed row of current data block for interpolation
int32_t index = GET_COL_DATA_POS(pQuery, lastIndex, step);
for(int32_t i = 0; i < pQuery->numOfCols; ++i) {
......@@ -2807,6 +2810,7 @@ static int32_t rowwiseApplyAllFunctions(SQueryRuntimeEnv *pRuntimeEnv, int32_t *
memcpy(pRuntimeEnv->lastRowInBlock[i], pRuntimeEnv->colDataBuffer[i]->data + s, pColInfo->bytes);
}
}
free(sasArray);
......@@ -3305,7 +3309,6 @@ static int64_t getOldestKey(int32_t numOfFiles, int64_t fileId, SVnodeCfg *pCfg)
}
bool isQueryKilled(SQuery *pQuery) {
return false;
SQInfo *pQInfo = (SQInfo *)GET_QINFO_ADDR(pQuery);
/*
......@@ -4019,17 +4022,8 @@ bool normalizedFirstQueryRange(bool dataInDisk, bool dataInCache, STableQuerySup
*key = nextKey;
}
// needs the data before the begin timestamp of query time window
if (nextKey != pQuery->skey) {
if (!pRuntimeEnv->hasTimeWindow) {
pQuery->skey = nextKey; // change the query skey
pQuery->lastKey = pQuery->skey;
}
return true;
} else {
return doGetQueryPos(nextKey, pSupporter, pPointInterpSupporter);
}
}
// set no data in file
pQuery->fileId = -1;
......@@ -4655,7 +4649,7 @@ static void doSetInterpVal(SQLFunctionCtx *pCtx, TSKEY ts, int16_t type, int32_t
len = t + 1 + TSDB_KEYSIZE;
pCtx->param[index].pz = calloc(1, len);
} else if (type == TSDB_DATA_TYPE_NCHAR) {
t = wcslen((const wchar_t *)data);
t = twcslen((const wchar_t *)data);
len = (t + 1) * TSDB_NCHAR_SIZE + TSDB_KEYSIZE;
pCtx->param[index].pz = calloc(1, len);
......
src/system/detail/src/vnodeRead.c
浏览文件 @ 438174a6
......@@ -496,6 +496,9 @@ void vnodeDecRefCount(void *param) {
assert(vnodeIsQInfoValid(pQInfo));
int32_t ref = atomic_sub_fetch_32(&pQInfo->refCount, 1);
if (ref < 0) {
return; // avoid two threads dec ref count
}
assert(ref >= 0);
dTrace("QInfo:%p decrease obj refcount, %d", pQInfo, ref);
......
src/system/detail/src/vnodeStore.c
浏览文件 @ 438174a6
......@@ -24,6 +24,8 @@
#include "vnodeUtil.h"
#include "vnodeStatus.h"
#include <dirent.h>
int tsMaxVnode = -1;
int tsOpenVnodes = 0;
SVnodeObj *vnodeList = NULL;
......
src/system/lite/src/mgmtShell.spec.c
浏览文件 @ 438174a6
......@@ -46,3 +46,8 @@ int mgmtProcessDropAcctMsg(char *pMsg, int msgLen, SConnObj *pConn) {
int mgmtProcessCreateAcctMsg(char *pMsg, int msgLen, SConnObj *pConn) {
return taosSendSimpleRsp(pConn->thandle, TSDB_MSG_TYPE_CREATE_ACCT_RSP, TSDB_CODE_OPS_NOT_SUPPORT);
}
void mgmtGetDnodeOnlineNum(int32_t *totalDnodes, int32_t *onlineDnodes) {
*totalDnodes = 1;
*onlineDnodes = 1;
}
\ No newline at end of file
src/util/CMakeLists.txt
浏览文件 @ 438174a6
......@@ -28,7 +28,7 @@ IF ((TD_LINUX_64) OR (TD_LINUX_32 AND TD_ARM))
MESSAGE(STATUS "Failed to find iconv, use default encoding method")
ENDIF ()
ENDIF ()
ELSEIF (TD_WINDOWS_64)
ELSEIF (TD_WINDOWS_64 OR TD_WINDOWS_32)
ADD_DEFINITIONS(-DUSE_LIBICONV)
INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/pthread)
INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/iconv)
......
src/util/src/tcache.c
浏览文件 @ 438174a6
......@@ -587,6 +587,8 @@ void *taosAddDataIntoCache(void *handle, char *key, char *pData, int dataSize, i
"size:%" PRId64 " bytes, collision:%d",
pNode->key, pNode, HASH_INDEX(pNode->hashVal, pObj->capacity), pNode->addTime, pNode->time, pObj->size,
pObj->totalSize, pObj->statistics.numOfCollision);
} else {
pError("key:%s failed to added into cache, out of memory", key);
}
} else { // old data exists, update the node
pNode = taosUpdateCacheImpl(pObj, pOldNode, key, keyLen, pData, dataSize, keepTime * 1000L);
......
src/util/src/textbuffer.c
浏览文件 @ 438174a6
......@@ -41,11 +41,13 @@ void getTmpfilePath(const char *fileNamePrefix, char *dstPath) {
char *tmpDir = "/tmp/";
#endif
int64_t ts = taosGetTimestampUs();
strcpy(tmpPath, tmpDir);
strcat(tmpPath, tdengineTmpFileNamePrefix);
strcat(tmpPath, fileNamePrefix);
strcat(tmpPath, "-%llu-%u");
snprintf(dstPath, MAX_TMPFILE_PATH_LENGTH, tmpPath, taosGetPthreadId(), atomic_add_fetch_32(&tmpFileSerialNum, 1));
strcat(tmpPath, "-%d-%llu-%u-%llu");
snprintf(dstPath, MAX_TMPFILE_PATH_LENGTH, tmpPath, getpid(), taosGetPthreadId(), atomic_add_fetch_32(&tmpFileSerialNum, 1), ts);
}
/*
......
src/util/src/tglobalcfg.c
浏览文件 @ 438174a6
......@@ -134,6 +134,7 @@ int tsEnableHttpModule = 1;
int tsEnableMonitorModule = 1;
int tsRestRowLimit = 10240;
int tsMaxSQLStringLen = TSDB_MAX_SQL_LEN;
int tsMaxAuthRetry = 5;
// the maximum number of results for projection query on super table that are returned from
// one virtual node, to order according to timestamp
......@@ -690,6 +691,10 @@ static void doInitGlobalConfig() {
TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_CLIENT | TSDB_CFG_CTYPE_B_SHOW,
TSDB_MAX_SQL_LEN, TSDB_MAX_ALLOWED_SQL_LEN, 0, TSDB_CFG_UTYPE_BYTE);
tsInitConfigOption(cfg++, "maxAuthRetryTime", &tsMaxAuthRetry, TSDB_CFG_VTYPE_INT,
TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_CLIENT | TSDB_CFG_CTYPE_B_SHOW,
1, 10, 0, TSDB_CFG_UTYPE_BYTE);
tsInitConfigOption(cfg++, "maxNumOfOrderedRes", &tsMaxNumOfOrderedResults, TSDB_CFG_VTYPE_INT,
TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_CLIENT | TSDB_CFG_CTYPE_B_SHOW,
TSDB_MAX_SQL_LEN, TSDB_MAX_ALLOWED_SQL_LEN, 0, TSDB_CFG_UTYPE_NONE);
......
src/util/src/tnote.c
浏览文件 @ 438174a6
......@@ -77,7 +77,7 @@ void taosUnLockNote(int fd, taosNoteInfo * pNote)
void *taosThreadToOpenNewNote(void *param)
{
char name[NOTE_FILE_NAME_LEN];
char name[NOTE_FILE_NAME_LEN + 16];
taosNoteInfo * pNote = (taosNoteInfo *)param;
pNote->taosNoteFlag ^= 1;
......@@ -170,7 +170,7 @@ void taosGetNoteName(char *fn, taosNoteInfo * pNote)
int taosOpenNoteWithMaxLines(char *fn, int maxLines, int maxNoteNum, taosNoteInfo * pNote)
{
char name[NOTE_FILE_NAME_LEN] = "\0";
char name[NOTE_FILE_NAME_LEN + 16] = "\0";
struct stat notestat0, notestat1;
int size;
......
src/util/src/ttime.c
浏览文件 @ 438174a6
......@@ -19,6 +19,7 @@
#include <errno.h>
#include <stdio.h>
#include <stdlib.h>
#include <time.h>
#include "tsdb.h"
#include "ttime.h"
......@@ -158,6 +159,19 @@ int32_t taosParseTime(char* timestr, int64_t* time, int32_t len, int32_t timePre
}
}
int32_t taosTimeSecToString(time_t ts,char* outstr) {
if (NULL == outstr) {
return 1;
}
struct tm *t;
t = localtime(&ts);
if (NULL == t) return 1;
sprintf(outstr,"%4d-%02d-%02d %02d:%02d:%02d\n", t->tm_year + 1900, t->tm_mon + 1, t->tm_mday, t->tm_hour, t->tm_min, t->tm_sec);
return 0;
}
char* forwardToTimeStringEnd(char* str) {
int32_t i = 0;
int32_t numOfSep = 0;
......
src/util/src/ttypes.c
浏览文件 @ 438174a6
......@@ -233,7 +233,7 @@ int32_t tVariantToString(tVariant *pVar, char *dst) {
case TSDB_DATA_TYPE_NCHAR: {
dst[0] = '\'';
taosUcs4ToMbs(pVar->wpz, (wcslen(pVar->wpz) + 1) * TSDB_NCHAR_SIZE, dst + 1);
taosUcs4ToMbs(pVar->wpz, (twcslen(pVar->wpz) + 1) * TSDB_NCHAR_SIZE, dst + 1);
int32_t len = strlen(dst);
dst[len] = '\'';
dst[len + 1] = 0;
......@@ -466,7 +466,7 @@ static int32_t toNchar(tVariant *pVariant, char **pDest, int32_t *pDestSize) {
}
pVariant->wpz = pWStr;
*pDestSize = wcslen(pVariant->wpz);
*pDestSize = twcslen(pVariant->wpz);
// shrink the allocate memory, no need to check here.
char* tmp = realloc(pVariant->wpz, (*pDestSize + 1)*TSDB_NCHAR_SIZE);
......
src/util/src/tutil.c
浏览文件 @ 438174a6
......@@ -27,6 +27,23 @@
#include "tlog.h"
#include "taoserror.h"
size_t twcslen(const wchar_t *wcs) {
int *wstr = (int *)wcs;
if (NULL == wstr) {
return 0;
}
size_t n = 0;
while (1) {
if (0 == *wstr++) {
break;
}
n++;
}
return n;
}
int32_t strdequote(char *z) {
if (z == NULL) {
return 0;
......
src/util/src/version.c
浏览文件 @ 438174a6
char version[64] = "1.6.6.1";
char version[64] = "1.6.5.9";
char compatible_version[64] = "1.6.0.0";
char gitinfo[128] = "0b5b412ef0ae2449ece538601a29b899b2b727b9";
char gitinfoOfInternal[128] = "8ae0d83a3610b9b4726373dd3073e4a8f444fb26";
......
tests/comparisonTest/dataGenerator/com/taosdata/generator/DataGenerator.java
浏览文件 @ 438174a6
......@@ -108,7 +108,6 @@ public class DataGenerator {
}
private static void getDataInOneFile(String path, int rowsPerDevice, int num, int humidityDistRadius, int tempDistRadius) throws IOException {
DecimalFormat df = new DecimalFormat("0.0000");
long startTime = dataStartTime;
FileWriter fw = new FileWriter(new File(path));
......@@ -135,13 +134,13 @@ public class DataGenerator {
for (int j = 0; j < rowsPerDevice; ++j) {
int humidity = (int) humidityDataGen.next();
double temp = tempDataGen.next();
int temp = (int) tempDataGen.next();
int deviceGroup = deviceId % 100;
StringBuffer sb = new StringBuffer();
sb.append(deviceId).append(" ").append(tagPrefix).append(deviceId).append(" ").append(deviceGroup)
.append(" ").append(dataStartTime).append(" ").append(humidity).append(" ")
.append(df.format(temp));
.append(temp);
bw.write(sb.toString());
bw.write("\n");
......
tests/examples/JDBC/JDBCDemo/src/main/java/TSDBSyncSample.java
浏览文件 @ 438174a6
......@@ -4,7 +4,7 @@ public class TSDBSyncSample {
private static final String JDBC_PROTOCAL = "jdbc:TAOS://";
private static final String TSDB_DRIVER = "com.taosdata.jdbc.TSDBDriver";
private String host = "127.0.0.1";
private String host = "10.211.55.3";
private String user = "root";
private String password = "taosdata";
private int port = 0;
......
tests/examples/JDBC/JDBCDemo/src/main/java/TestAsyncTSDBSubscribeSample.java 0 → 100644
浏览文件 @ 438174a6
import com.taosdata.jdbc.*;
import org.apache.commons.lang3.StringUtils;
import java.sql.Connection;
import java.sql.DriverManager;
import java.util.Properties;
public class TestAsyncTSDBSubscribeSample {
public static void main(String[] args) {
String dbName = "log";
String tName = "dn_10_211_55_3";
String host = "10.211.55.3";
String topic = "test";
Connection connection = null;
TSDBSubscribe subscribe = null;
long subscribId = 0;
try {
Class.forName("com.taosdata.jdbc.TSDBDriver");
Properties properties = new Properties();
properties.setProperty(TSDBDriver.PROPERTY_KEY_HOST, host);
connection = DriverManager.getConnection("jdbc:TAOS://" + host + ":0/" + dbName + "?user=root&password=taosdata", properties);
String rawSql = "select * from " + tName + ";";
subscribe = ((TSDBConnection) connection).createSubscribe();
subscribId = subscribe.subscribe(topic, rawSql, false, 1000, new CallBack("first"));
long subscribId2 = subscribe.subscribe("test", rawSql, false, 1000, new CallBack("second"));
int a = 0;
Thread.sleep(2000);
subscribe.unsubscribe(subscribId, true);
System.err.println("cancel subscribe");
} catch (Exception e) {
e.printStackTrace();
}
}
private static class CallBack implements TSDBSubscribeCallBack {
private String name = "";
public CallBack(String name) {
this.name = name;
}
@Override
public void invoke(TSDBResultSet resultSet) {
try {
while (null !=resultSet && resultSet.next()) {
System.out.print("callback_" + name + ": ");
for (int i = 1; i <= resultSet.getMetaData().getColumnCount(); i++) {
System.out.printf(i + ": " + resultSet.getString(i) + "\t");
}
System.out.println();
}
} catch (Exception e) {
e.printStackTrace();
}
}
}
}
tests/examples/JDBC/JDBCDemo/src/main/java/TestTSDBSubscribeSample.java 0 → 100644
浏览文件 @ 438174a6
import com.taosdata.jdbc.TSDBConnection;
import com.taosdata.jdbc.TSDBDriver;
import com.taosdata.jdbc.TSDBResultSet;
import com.taosdata.jdbc.TSDBSubscribe;
import org.apache.commons.lang3.StringUtils;
import java.sql.Connection;
import java.sql.DriverManager;
import java.util.Properties;
public class TestTSDBSubscribeSample {
public static void main(String[] args) throws Exception {
// use log db
String dbName = "log";
String tName = "dn_10_211_55_3";
String host = "10.211.55.3";
String topic = "test";
Connection connection = null;
TSDBSubscribe subscribe = null;
long subscribId = 0;
try {
Class.forName("com.taosdata.jdbc.TSDBDriver");
Properties properties = new Properties();
properties.setProperty(TSDBDriver.PROPERTY_KEY_HOST, host);
connection = DriverManager.getConnection("jdbc:TAOS://" + host + ":0/" + dbName + "?user=root&password=taosdata"
, properties);
String rawSql = "select * from " + tName + ";";
subscribe = ((TSDBConnection) connection).createSubscribe();
subscribId = subscribe.subscribe(topic, rawSql, false, 1000);
int a = 0;
while (true) {
Thread.sleep(1000);
TSDBResultSet resSet = subscribe.consume(subscribId);
while (resSet.next()) {
for (int i = 1; i <= resSet.getMetaData().getColumnCount(); i++) {
System.out.printf(i + ": " + resSet.getString(i) + "\t");
}
System.out.println("\n================");
}
a++;
if (a >= 10) {
break;
}
}
} catch (Exception e) {
e.printStackTrace();
} finally {
if (null != subscribe && 0 != subscribId) {
subscribe.unsubscribe(subscribId, true);
}
if (null != connection) {
connection.close();
}
}
}
}
tests/examples/c/CMakeLists.txt
浏览文件 @ 438174a6
PROJECT(TDengine)
IF (TD_WINDOWS_64)
IF (TD_WINDOWS_64 OR TD_WINDOWS_32)
INCLUDE_DIRECTORIES(${TD_ROOT_DIR}/deps/pthread)
ENDIF ()
......
Markdown is supported
0% 或 .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑！
想要评论请 注册 或