diff --git a/CMakeLists.txt b/CMakeLists.txt index 163789a1e4458b0c8541a674d70f87837227ca6a..634703a791b373f8847564cd6da33473fcfe6886 100755 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -53,6 +53,7 @@ IF (NOT DEFINED TD_CLUSTER) SET(TD_MIPS_32 FALSE) SET(TD_DARWIN_64 FALSE) SET(TD_WINDOWS_64 FALSE) + SET(TD_WINDOWS_32 FALSE) SET(TD_PAGMODE_LITE FALSE) IF (${PAGMODE} MATCHES "lite") @@ -127,6 +128,11 @@ IF (NOT DEFINED TD_CLUSTER) SET(TD_OS_DIR ${TD_COMMUNITY_DIR}/src/os/windows) ADD_DEFINITIONS(-D_M_X64) MESSAGE(STATUS "The current platform is Windows 64-bit") + ELSEIF(${CMAKE_SIZEOF_VOID_P} MATCHES 4) + SET(TD_WINDOWS_32 TRUE) + SET(TD_OS_DIR ${TD_COMMUNITY_DIR}/src/os/windows) + #ADD_DEFINITIONS(-D_M_X64) + MESSAGE(STATUS "The current platform is Windows 32-bit") ELSE () MESSAGE(FATAL_ERROR "The current platform is Windows 32-bit, not supported yet") EXIT () @@ -194,7 +200,7 @@ IF (NOT DEFINED TD_CLUSTER) link_library(/usr/lib/libargp.a) ADD_DEFINITIONS(-D_ALPINE) ENDIF () - ELSEIF (TD_WINDOWS_64) + ELSEIF (TD_WINDOWS_64 OR TD_WINDOWS_32) SET(CMAKE_GENERATOR "NMake Makefiles" CACHE INTERNAL "" FORCE) IF (NOT TD_GODLL) SET(COMMON_FLAGS "/nologo /WX- /Oi /Oy- /Gm- /EHsc /MT /GS /Gy /fp:precise /Zc:wchar_t /Zc:forScope /Gd /errorReport:prompt /analyze-") @@ -235,7 +241,7 @@ IF (NOT DEFINED TD_CLUSTER) ELSEIF (${CMAKE_BUILD_TYPE} MATCHES "Release") MESSAGE(STATUS "Build Release Version") ELSE () - IF (TD_WINDOWS_64) + IF (TD_WINDOWS_64 OR TD_WINDOWS_32) SET(CMAKE_BUILD_TYPE "Release") MESSAGE(STATUS "Build Release Version in Windows as default") ELSE () @@ -268,7 +274,7 @@ IF (NOT DEFINED TD_CLUSTER) INSTALL(CODE "MESSAGE(\"make install script: ${TD_MAKE_INSTALL_SH}\")") INSTALL(CODE "execute_process(COMMAND chmod 777 ${TD_MAKE_INSTALL_SH})") INSTALL(CODE "execute_process(COMMAND ${TD_MAKE_INSTALL_SH} ${TD_COMMUNITY_DIR} ${PROJECT_BINARY_DIR})") - ELSEIF (TD_WINDOWS_64) + ELSEIF (TD_WINDOWS_64 OR TD_WINDOWS_32) SET(CMAKE_INSTALL_PREFIX C:/TDengine) IF (NOT TD_GODLL) INSTALL(DIRECTORY ${TD_COMMUNITY_DIR}/src/connector/go DESTINATION connector) @@ -277,6 +283,7 @@ IF (NOT DEFINED TD_CLUSTER) INSTALL(DIRECTORY ${TD_COMMUNITY_DIR}/tests/examples DESTINATION .) INSTALL(DIRECTORY ${TD_COMMUNITY_DIR}/packaging/cfg DESTINATION .) INSTALL(FILES ${TD_COMMUNITY_DIR}/src/inc/taos.h DESTINATION include) + INSTALL(FILES ${TD_COMMUNITY_DIR}/src/inc/taoserror.h DESTINATION include) INSTALL(FILES ${LIBRARY_OUTPUT_PATH}/taos.lib DESTINATION driver) INSTALL(FILES ${LIBRARY_OUTPUT_PATH}/taos.exp DESTINATION driver) INSTALL(FILES ${LIBRARY_OUTPUT_PATH}/taos.dll DESTINATION driver) diff --git a/bailongma/Dockerfile b/bailongma/Dockerfile new file mode 100644 index 0000000000000000000000000000000000000000..68e2a71ad39ebc652da551838f563759ed82271f --- /dev/null +++ b/bailongma/Dockerfile @@ -0,0 +1,59 @@ +## Builder image +#FROM tdengine/tdengine:dev as builder1 +FROM golang:latest as builder1 + + + +WORKDIR /root + +#COPY --from=builder1 /usr/include/taos.h /usr/include/ +#COPY --from=builder1 /usr/lib/libtaos.so /usr/lib/libtaos.so +#COPY --from=builder1 /usr/include/taos.h /usr/include/ +#COPY --from=builder1 /usr/lib/libtaos.so.1 /usr/lib/ +COPY blm_telegraf /root +COPY blm_prometheus /root +COPY TDengine-server-1.6.5.9-Linux-x64.tar.gz /root +COPY go.mod /root +COPY go.sum /root +RUN tar -zxf TDengine-server-1.6.5.9-Linux-x64.tar.gz . +WORKDIR /root/TDengine-server +RUN sh install.sh + +#RUN mkdir /usr/lib/ld + + +WORKDIR /root/blm_telegraf +RUN go build + +WORKDIR /root/blm_prometheus +RUN go build +#RUN ln -s /usr/lib/libtaos.so.1 /usr/lib/libtaos.so +FROM centos:7 + + +WORKDIR /root + +COPY --from=builder /usr/include/taos.h /usr/include/ +COPY --from=builder /usr/lib/libtaos.so /usr/lib/libtaos.so +COPY --from=builder /usr/include/taos.h /usr/include/ +COPY --from=builder /usr/lib/libtaos.so.1 /usr/lib/ +COPY --from=builder /root/blm_telegraf/blm_telegraf /root/ +COPY --from=builder /root/blm_prometheus/blm_prometheus /root/ + + + +#RUN git config --global http.sslVerify false +#RUN git config --global http.postbuffer 524288000 + + +#RUN go get -v -u -insecure github.com/taosdata/TDengine/src/connector/go/src/taosSql +#RUN go get -v -u -insecure github.com/gogo/protobuf/proto +#RUN go get -v -u -insecure github.com/golang/snappy +#RUN go get -v -u -insecure github.com/prometheus/common/model +#RUN go get -v -u -insecure github.com/prometheus/prometheus/prompb +#RUN go get github.com/taosdata/driver-go/taosSql + + + + + diff --git a/bailongma/Dockerfile.prm b/bailongma/Dockerfile.prm new file mode 100644 index 0000000000000000000000000000000000000000..2df063e59a62e516d41b4624f646fd352c892064 --- /dev/null +++ b/bailongma/Dockerfile.prm @@ -0,0 +1,43 @@ +## Builder image +FROM tdengine/tdengine:1.6.5.9 as builder1 +FROM golang:latest as builder + + + +WORKDIR /root + +COPY --from=builder1 /usr/include/taos.h /usr/include/ +COPY --from=builder1 /usr/lib/libtaos.so /usr/lib/libtaos.so +COPY --from=builder1 /usr/include/taos.h /usr/include/ +COPY --from=builder1 /usr/lib/libtaos.so.1 /usr/lib/ +COPY blm_prometheus /root/blm_prometheus +COPY go.mod /root +COPY go.sum /root + + + +WORKDIR /root/blm_prometheus +RUN go build + +FROM centos:7 + + +WORKDIR /root +COPY --from=builder1 /usr/include/taos.h /usr/include/ +COPY --from=builder1 /usr/lib/libtaos.so.1 /usr/lib/ +RUN ln -s /usr/lib/libtaos.so.1 /usr/lib/libtaos.so + +COPY --from=builder /root/blm_prometheus /root + +ENV LD_LIBRARY_PATH="$LD_LIBRARY_PATH:/usr/lib" +ENV LANG=en_US.UTF-8 +ENV LANGUAGE=en_US:en +ENV LC_ALL=en_US.UTF-8 + + +#RUN git config --global http.sslVerify false +#RUN git config --global http.postbuffer 524288000 +VOLUME [ "/var/lib/taos", "/var/log/taos","/etc/taos/" ] + +ENTRYPOINT [ "/root/blm_prometheus" ] + diff --git a/bailongma/Dockerfile.tlg b/bailongma/Dockerfile.tlg new file mode 100644 index 0000000000000000000000000000000000000000..e8bcec135e2ca2c9dde803e47abfea7a75396a7b --- /dev/null +++ b/bailongma/Dockerfile.tlg @@ -0,0 +1,43 @@ +## Builder image +FROM tdengine/tdengine:1.6.5.9 as builder1 +FROM golang:latest as builder + + + +WORKDIR /root + +COPY --from=builder1 /usr/include/taos.h /usr/include/ +COPY --from=builder1 /usr/lib/libtaos.so /usr/lib/libtaos.so +COPY --from=builder1 /usr/include/taos.h /usr/include/ +COPY --from=builder1 /usr/lib/libtaos.so.1 /usr/lib/ +COPY blm_telegraf /root/blm_telegraf +COPY go.mod /root +COPY go.sum /root + + + +WORKDIR /root/blm_telegraf +RUN ls +RUN go build + +FROM centos:7 + + +WORKDIR /root +COPY --from=builder1 /usr/include/taos.h /usr/include/ +COPY --from=builder1 /usr/lib/libtaos.so.1 /usr/lib/ +RUN ln -s /usr/lib/libtaos.so.1 /usr/lib/libtaos.so +COPY --from=builder /root/blm_telegraf /root +ENV LD_LIBRARY_PATH="$LD_LIBRARY_PATH:/usr/lib" +ENV LANG=en_US.UTF-8 +ENV LANGUAGE=en_US:en +ENV LC_ALL=en_US.UTF-8 + + +VOLUME [ "/var/lib/taos", "/var/log/taos","/etc/taos/" ] + +ENTRYPOINT [ "/root/blm_telegraf"] + + + + diff --git a/bailongma/blm_prometheus/Dockerfile b/bailongma/blm_prometheus/Dockerfile new file mode 100644 index 0000000000000000000000000000000000000000..6699ad8b7834d251cd6a79406c185d12aeb4db6b --- /dev/null +++ b/bailongma/blm_prometheus/Dockerfile @@ -0,0 +1,41 @@ +## Builder image +FROM tdengine/tdengine:dev as builder1 +FROM tdengine/bailongma:dev as builder + + +COPY server.go /root/blm_prometheus/ + +WORKDIR /root/blm_prometheus + +# build enterprise version + +RUN go build +# # build community version +# RUN cmake .. -DVERSION=lite && cmake --build . + +## Target image + + +FROM centos:7 + +WORKDIR /root/ + +# COPY --from=builder /root/build/build/lib/libtaos.so /usr/lib/libtaos.so.1 +# RUN ln -s /usr/lib/libtaos.so.1 /usr/lib/libtaos.so +COPY --from=builder1 /usr/include/taos.h /usr/include/ +COPY --from=builder1 /usr/lib/libtaos.so.1 /usr/lib/libtaos.so.1 +RUN ln -s /usr/lib/libtaos.so.1 /usr/lib/libtaos.so + + +COPY --from=builder /root/blm_prometheus/blm_prometheus . + +#COPY community/packaging/cfg/taos.cfg /etc/taos/ + +ENV LD_LIBRARY_PATH="$LD_LIBRARY_PATH:/usr/lib" +ENV LANG=en_US.UTF-8 +ENV LANGUAGE=en_US:en +ENV LC_ALL=en_US.UTF-8 +EXPOSE 10203 +VOLUME [ "/var/lib/taos", "/var/log/taos","/etc/taos" ] + +ENTRYPOINT [ "/root/blm_prometheus" ] \ No newline at end of file diff --git a/bailongma/blm_prometheus/README.md b/bailongma/blm_prometheus/README.md new file mode 100644 index 0000000000000000000000000000000000000000..4a4c2696cfb9441fe4bb9c9b4e0c90f2b5ee1a52 --- /dev/null +++ b/bailongma/blm_prometheus/README.md @@ -0,0 +1,137 @@ +# Prometheus Remote Write Adapter for TDengine + +This is an adapter to support Prometheus remote write into TDengine. + +## Prerequisite + +before running the software, you need to install the `golang-1.10` or later version in your environment and install [TDengine][] so the program can use the lib of TDengine. + +To use it: + +``` +go build +``` +During the go build process, there maybe some errors arised because of lacking some needed packages. You can use `go get` the package to solve it +``` +go get github.com/gogo/protobuf/proto +go get github.com/golang/snappy +go get github.com/prometheus/common/model +go get github.com/taosdata/TDengine/src/connector/go/src/taosSql +go get github.com/prometheus/prometheus/prompb + +``` +After successful build, there will be a blm_prometheus in the same directory. + +## Running in background + +Using following command to run the program in background + +``` +nohup ./blm_prometheus --tdengine-ip 112.102.3.69 --batch-size 80 --http-workers 2 --sql-workers 2 --dbname prometheus --port 1234 > /dev/null 2>&1 & +``` +There are several options can be set: + +```sh +--tdengine-ip +set the IP of TDengine for example "192.168.0.1" + +--tdengine-name +set the domain name of TDengine, then blm-prometheus can lookup the ip address of TDengine. + +--tdengine-api-port +set the restful API port of TDengine. blm-prometheus will query the table schema info from TDengine to keep meta info synchronized. Default is 6020 + +--batch-size +set the size of how many records in one SQL cmd line writing into TDengine. There is a limitation that TDengine could only accept SQL line small than 64000 bytes, so usually the batch size should not exceed 200. Default is 100. + +--http-workers +set the number of workers who process the HTTP request. default is 10 + +--sql-workers +set the number of workers who process the database request. default is 10 + +--dbname +set the database name in TDengine, if not exists, a database will be created after this dbname. default is "prometheus". + +--dbuser +set the user name that have the right to access the TDengine. default is "root" + +--dbpassword +set the password of dbuser. default is "taosdata" + +--port +set the port that prometheus configuration remote_write. as showed above, in the prometheus.yaml. default is 10203 + + + +``` + +## Start prometheus + +Add the following to your prometheus's configuration `prometheus.yml` : + +```yaml +remote_write: + - url: "http://localhost:1234/receive" +``` + +Then start Prometheus: + +``` + prometheus +``` +Then you can check the TDengine if there is super table and tables. + +## Check the TDengine tables and datas + +Use the taos client shell to query the result. +``` +Welcome to the TDengine shell from linux, client version:1.6.4.0 server version:1.6.4.0 +Copyright (c) 2017 by TAOS Data, Inc. All rights reserved. + +This is the trial version and will expire at 2019-12-11 14:25:31. + +taos> use prometheus; +Database changed. + +taos> show stables; + name | created_time |columns| tags | tables | +==================================================================================================================== +prometheus_sd_kubernetes_cache_watch_events_sum | 19-11-15 17:45:07.594| 2| 3| 1| +prometheus_sd_kubernetes_cache_watch_events_count | 19-11-15 17:45:07.596| 2| 3| 1| +prometheus_sd_kubernetes_cache_watches_total | 19-11-15 17:45:07.598| 2| 3| 1| +prometheus_sd_kubernetes_events_total | 19-11-15 17:45:07.600| 2| 5| 15| +prometheus_target_scrape_pool_reloads_total | 19-11-15 17:45:07.672| 2| 3| 1| +prometheus_sd_received_updates_total | 19-11-15 17:45:07.674| 2| 4| 1| +prometheus_target_scrape_pool_reloads_failed_total | 19-11-15 17:45:07.730| 2| 3| 1| +prometheus_sd_updates_total | 19-11-15 17:45:07.732| 2| 4| 1| +prometheus_target_scrape_pool_sync_total | 19-11-15 17:45:07.734| 2| 4| 1| +...... + +go_memstats_gc_cpu_fraction | 19-11-15 17:45:06.599| 2| 3| 1| +Query OK, 211 row(s) in set (0.004891s) + + +taos> select * from prometheus_sd_updates_total; + +...... + + 19-11-16 14:24:00.271| 1.000000000|localhost:9090 |prometheus |codelab-monitor |scrape | + 19-11-16 14:24:05.271| 1.000000000|localhost:9090 |prometheus |codelab-monitor |scrape | +Query OK, 3029 row(s) in set (0.060828s) + + +``` + +## Support Kubernates liveness probe +The blm_prometheus support the liveness probe. + +When the service is running, GET the url`http://ip:port/health` will return 200 OK response which means the service is running healthy. If no response, means the service is dead and need to restart it. + + +## Limitations + +The TDengine limits the length of super table name, so if the name of prometheus metric exceeds 60 byte, it will be truncated to first 60 bytes. And the length of label name is limited within 50 byte. + + +[TDengine]:https://www.github.com/Taosdata/TDengine \ No newline at end of file diff --git a/bailongma/blm_prometheus/server.go b/bailongma/blm_prometheus/server.go new file mode 100644 index 0000000000000000000000000000000000000000..a2aafd578e01653613dc445f3ca0b47528917379 --- /dev/null +++ b/bailongma/blm_prometheus/server.go @@ -0,0 +1,797 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package main + +import ( + "bufio" + "container/list" + "crypto/md5" + "database/sql" + "encoding/json" + "flag" + "fmt" + "io" + "io/ioutil" + "log" + "net" + "net/http" + _ "net/http/pprof" + "os" + "path/filepath" + "sort" + "strconv" + "strings" + "sync" + "time" + + "github.com/gogo/protobuf/proto" + "github.com/golang/snappy" + "github.com/prometheus/common/model" + _ "github.com/taosdata/driver-go/taosSql" + + "github.com/prometheus/prometheus/prompb" +) + +type Bailongma struct { +} + +type nametag struct { + tagmap map[string]string + taglist *list.List + annotlen int +} + +var ( + daemonIP string + daemonName string + httpworkers int + sqlworkers int + batchSize int + buffersize int + dbname string + dbuser string + dbpassword string + rwport string + apiport string + debugprt int + taglen int + taglimit int = 1024 + tagnumlimit int + tablepervnode int +) + +// Global vars +var ( + bufPool sync.Pool + batchChans []chan string //multi table one chan + nodeChans []chan prompb.WriteRequest //multi node one chan + inputDone chan struct{} + workersGroup sync.WaitGroup + reportTags [][2]string + reportHostname string + taosDriverName string = "taosSql" + IsSTableCreated sync.Map + IsTableCreated sync.Map + tagstr string + blmLog *log.Logger + tdurl string + logNameDefault string = "/var/log/taos/blm_prometheus.log" +) +var scratchBufPool = &sync.Pool{ + New: func() interface{} { + return make([]byte, 0, 1024) + }, +} + +type FieldDescriptiion struct { + fname string + ftype string + flength int + fnote string +} +type tableStruct struct { + status string + head []string + data []FieldDescriptiion + rows int64 +} + +// Parse args: +func init() { + flag.StringVar(&daemonIP, "tdengine-ip", "127.0.0.1", "TDengine host IP.") + flag.StringVar(&daemonName, "tdengine-name", "", "TDengine host Name. in K8S, could be used to lookup TDengine's IP") + flag.StringVar(&apiport, "tdengine-api-port", "6020", "TDengine restful API port") + flag.IntVar(&batchSize, "batch-size", 100, "Batch size (input items).") + flag.IntVar(&httpworkers, "http-workers", 1, "Number of parallel http requests handler .") + flag.IntVar(&sqlworkers, "sql-workers", 1, "Number of parallel sql handler.") + flag.StringVar(&dbname, "dbname", "prometheus", "Database name where to store metrics") + flag.StringVar(&dbuser, "dbuser", "root", "User for host to send result metrics") + flag.StringVar(&dbpassword, "dbpassword", "taosdata", "User password for Host to send result metrics") + flag.StringVar(&rwport, "port", "10203", "remote write port") + flag.IntVar(&debugprt, "debugprt", 0, "if 0 not print, if 1 print the sql") + flag.IntVar(&taglen, "tag-length", 50, "the max length of tag string,default is 30") + flag.IntVar(&buffersize, "buffersize", 100, "the buffer size of metrics received") + flag.IntVar(&tagnumlimit, "tag-num", 16, "the number of tags in a super table, default is 8") + flag.IntVar(&tablepervnode, "table-num", 10000, "the number of tables per TDengine Vnode can create, default 10000") + + flag.Parse() + + if daemonName != "" { + s, _ := net.LookupIP(daemonName) + daemonIP = fmt.Sprintf("%s", s[0]) + + fmt.Println(daemonIP) + fmt.Println(s[0]) + daemonIP = daemonIP + ":0" + + tdurl = daemonName + } else { + tdurl = daemonIP + daemonIP = daemonIP + ":0" + } + + tagstr = fmt.Sprintf(" binary(%d)", taglen) + logFile, err := os.OpenFile(logNameDefault, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644) + if err != nil { + fmt.Println(err) + os.Exit(1) + } + blmLog = log.New(logFile, "", log.LstdFlags) + blmLog.SetPrefix("BLM_PRM") + blmLog.SetFlags(log.LstdFlags | log.Lshortfile) + blmLog.Printf("host: ip") + blmLog.Printf(daemonIP) + blmLog.Printf(" port: ") + blmLog.Printf(rwport) + blmLog.Printf(" database: ") + blmLog.Println(dbname) + +} + +func main() { + + for i := 0; i < httpworkers; i++ { + nodeChans = append(nodeChans, make(chan prompb.WriteRequest, buffersize)) + } + + createDatabase(dbname) + + for i := 0; i < httpworkers; i++ { + workersGroup.Add(1) + go NodeProcess(i) + } + + for i := 0; i < sqlworkers; i++ { + batchChans = append(batchChans, make(chan string, batchSize)) + } + + for i := 0; i < sqlworkers; i++ { + workersGroup.Add(1) + go processBatches(i) + } + + http.HandleFunc("/receive", func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusAccepted) + addr := strings.Split(r.RemoteAddr, ":") + idx := TAOShashID([]byte(addr[0])) + + compressed, err := ioutil.ReadAll(r.Body) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + r.Body.Close() + reqBuf, err := snappy.Decode(nil, compressed) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + + var req prompb.WriteRequest + if err := proto.Unmarshal(reqBuf, &req); err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + nodeChans[idx%httpworkers] <- req + + }) + http.HandleFunc("/check", func(w http.ResponseWriter, r *http.Request) { + + compressed, err := ioutil.ReadAll(r.Body) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + //blmLog.Println(string(compressed)) + var output string = "" + schema, ok := IsSTableCreated.Load(string(compressed)) + if !ok { + output = "the stable is not created!" + } else { + ntag := schema.(nametag) + tbtaglist := ntag.taglist + tbtagmap := ntag.tagmap + //annotlen := ntag.annotlen + output = "tags: " + for e := tbtaglist.Front(); e != nil; e = e.Next() { + output = output + e.Value.(string) + " | " + } + output = output + "\ntagmap: " + s := fmt.Sprintln(tbtagmap) + output = output + s + } + + res := queryTableStruct(string(compressed)) + output = output + "\nTable structure:\n" + res + s := fmt.Sprintf("query result:\n %s\n", output) + //blmLog.Println(s) + w.Write([]byte(s)) + w.WriteHeader(http.StatusOK) + }) + http.HandleFunc("/health", func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusNoContent) + }) + if debugprt == 5 { + TestSerialization() + } + blmLog.Fatal(http.ListenAndServe(":"+rwport, nil)) + +} + +func queryTableStruct(tbname string) string { + client := new(http.Client) + s := fmt.Sprintf("describe %s.%s", dbname, tbname) + body := strings.NewReader(s) + req, _ := http.NewRequest("GET", "http://"+tdurl+":"+apiport+"/rest/sql", body) + //fmt.Println("http://" + tdurl + ":" + apiport + "/rest/sql" + s) + req.SetBasicAuth(dbuser, dbpassword) + resp, err := client.Do(req) + + if err != nil { + blmLog.Println(err) + fmt.Println(err) + return "" + } else { + compressed, _ := ioutil.ReadAll(resp.Body) + defer resp.Body.Close() + return string(compressed) + } +} + +func TAOShashID(ba []byte) int { + var sum int = 0 + for i := 0; i < len(ba); i++ { + sum += int(ba[i] - '0') + } + return sum +} + +func NodeProcess(workerid int) error { + + for req := range nodeChans[workerid] { + + ProcessReq(req) + + } + + return nil +} + +func ProcessReq(req prompb.WriteRequest) error { + db, err := sql.Open(taosDriverName, dbuser+":"+dbpassword+"@/tcp("+daemonIP+")/"+dbname) + if err != nil { + blmLog.Fatalf("Open database error: %s\n", err) + } + defer db.Close() + + for _, ts := range req.Timeseries { + err = HandleStable(ts, db) + } + return err +} + +func HandleStable(ts *prompb.TimeSeries, db *sql.DB) error { + taglist := list.New() + tbtaglist := list.New() + tagmap := make(map[string]string) + tbtagmap := make(map[string]string) + m := make(model.Metric, len(ts.Labels)) + tagnum := tagnumlimit + var hasName bool = false + var metricsName string + var tbn string = "" + var nt nametag + //var annotlen int + //fmt.Println(ts) + j := 0 + for _, l := range ts.Labels { + m[model.LabelName(l.Name)] = model.LabelValue(l.Value) + + if string(l.Name) == "__name__" { + metricsName = string(l.Value) + tbn += metricsName + hasName = true + continue + } + j++ + ln := strings.ToLower(string(l.Name)) + OrderInsertS(ln, taglist) + //taglist.PushBack(ln) + s := string(l.Value) + tbn += s + //tagHash += s + if j <= tagnum { + OrderInsertS(ln, tbtaglist) + //tbtaglist.PushBack(ln) + if len(s) > taglen { + s = s[:taglen] + } + tbtagmap[ln] = "y" + } + tagmap[ln] = s + } + + if debugprt == 2 { + t := ts.Samples[0].Timestamp + var ns int64 = 0 + if t/1000000000 > 10 { + tm := t / 1000 + ns = t - tm*1000 + } + blmLog.Printf(" Ts: %s, value: %f, ", time.Unix(t/1000, ns), ts.Samples[0].Value) + blmLog.Println(ts) + } + + if !hasName { + info := fmt.Sprintf("no name metric") + panic(info) + } + stbname := tablenameEscape(metricsName) + var ok bool + schema, ok := IsSTableCreated.Load(stbname) + if !ok { // no local record of super table structure + nt.taglist = tbtaglist + nt.tagmap = tbtagmap + stbdescription := queryTableStruct(stbname) //query the super table from TDengine + var stbst map[string]interface{} + err := json.Unmarshal([]byte(stbdescription), &stbst) + if err == nil { //query tdengine table success! + status := stbst["status"].(string) + if status == "succ" { //yes, the super table was already created in TDengine + taostaglist := list.New() + taostagmap := make(map[string]string) + dt := stbst["data"] + for _, fd := range dt.([]interface{}) { + fdc := fd.([]interface{}) + if fdc[3].(string) == "tag" && fdc[0].(string) != "taghash" { + tmpstr := fdc[0].(string) + taostaglist.PushBack(tmpstr[2:]) + taostagmap[tmpstr[2:]] = "y" + } + } + nt.taglist = taostaglist + nt.tagmap = taostagmap + tbtaglist = nt.taglist + tbtagmap = nt.tagmap + // annotlen = ntag.annotlen + var sqlcmd string + i := 0 + for _, l := range ts.Labels { + k := strings.ToLower(string(l.Name)) + if k == "__name__" { + continue + } + i++ + if i < tagnumlimit { + _, ok := tbtagmap[k] + if !ok { + sqlcmd = "alter table " + stbname + " add tag t_" + k + tagstr + "\n" + _, err := execSql(dbname, sqlcmd, db) + if err != nil { + blmLog.Println(err) + errorcode := fmt.Sprintf("%s", err) + if strings.Contains(errorcode, "duplicated column names") { + tbtaglist.PushBack(k) + //OrderInsertS(k, tbtaglist) + tbtagmap[k] = "y" + } + } else { + tbtaglist.PushBack(k) + //OrderInsertS(k, tbtaglist) + tbtagmap[k] = "y" + } + } + } + } + IsSTableCreated.Store(stbname, nt) + } else { // no, the super table haven't been created in TDengine, create it. + var sqlcmd string + sqlcmd = "create table if not exists " + stbname + " (ts timestamp, value double) tags(taghash binary(34)" + for e := tbtaglist.Front(); e != nil; e = e.Next() { + sqlcmd = sqlcmd + ",t_" + e.Value.(string) + tagstr + } + //annotlen = taglimit - i*taglen + //nt.annotlen = annotlen + //annotationstr := fmt.Sprintf(" binary(%d)", annotlen) + //sqlcmd = sqlcmd + ", annotation " + annotationstr + ")\n" + sqlcmd = sqlcmd + ")\n" + _, err := execSql(dbname, sqlcmd, db) + if err == nil { + IsSTableCreated.Store(stbname, nt) + } else { + blmLog.Println(err) + } + } + } else { //query TDengine table error + blmLog.Println(err) + } + } else { + ntag := schema.(nametag) + tbtaglist = ntag.taglist + tbtagmap = ntag.tagmap + // annotlen = ntag.annotlen + var sqlcmd string + i := 0 + for _, l := range ts.Labels { + k := strings.ToLower(string(l.Name)) + if k == "__name__" { + continue + } + i++ + if i < tagnumlimit { + _, ok := tbtagmap[k] + if !ok { + sqlcmd = "alter table " + stbname + " add tag t_" + k + tagstr + "\n" + _, err := execSql(dbname, sqlcmd, db) + if err != nil { + blmLog.Println(err) + errorcode := fmt.Sprintf("%s", err) + if strings.Contains(errorcode, "duplicated column names") { + tbtaglist.PushBack(k) + //OrderInsertS(k, tbtaglist) + tbtagmap[k] = "y" + } + } else { + tbtaglist.PushBack(k) + //OrderInsertS(k, tbtaglist) + tbtagmap[k] = "y" + } + } + } + } + } + + tbnhash := "MD5_" + md5V2(tbn) + _, tbcreated := IsTableCreated.Load(tbnhash) + + if !tbcreated { + var sqlcmdhead, sqlcmd string + sqlcmdhead = "create table if not exists " + tbnhash + " using " + stbname + " tags(\"" + sqlcmd = "" + i := 0 + for e := tbtaglist.Front(); e != nil; e = e.Next() { + tagvalue, has := tagmap[e.Value.(string)] + if len(tagvalue) > taglen { + tagvalue = tagvalue[:taglen] + } + if i == 0 { + if has { + sqlcmd = sqlcmd + "\"" + tagvalue + "\"" + } else { + sqlcmd = sqlcmd + "null" + } + i++ + } else { + if has { + sqlcmd = sqlcmd + ",\"" + tagvalue + "\"" + } else { + sqlcmd = sqlcmd + ",null" + } + } + + } + /* + var annotation string = "" + for t := taglist.Front(); t != nil; t = t.Next() { + _, has := tbtagmap[t.Value.(string)] + if !has { + tagvalue, _ := tagmap[t.Value.(string)] + annotation += t.Value.(string) + "=" + tagvalue + "," + } + } + if len(annotation) > annotlen-2 { + annotation = annotation[:annotlen-2] + }*/ + //sqlcmd = sqlcmd + ",\"" + annotation + "\")\n" + var keys []string + var tagHash = "" + for t := range tagmap { + keys = append(keys, t) + } + sort.Strings(keys) + for _, k := range keys { + tagHash += tagmap[k] + } + + sqlcmd = sqlcmd + ")\n" + sqlcmd = sqlcmdhead + md5V2(tagHash) + "\"," + sqlcmd + _, err := execSql(dbname, sqlcmd, db) + if err == nil { + IsTableCreated.Store(tbnhash, true) + } + } + serilizeTDengine(ts, tbnhash, db) + return nil +} + +func tablenameEscape(mn string) string { + stbb := strings.ReplaceAll(mn, ":", "_") // replace : in the metrics name to adapt the TDengine + stbc := strings.ReplaceAll(stbb, ".", "_") + stbname := strings.ReplaceAll(stbc, "-", "_") + if len(stbname) > 60 { + stbname = stbname[:60] + } + return stbname +} + +func serilizeTDengine(m *prompb.TimeSeries, tbn string, db *sql.DB) error { + idx := TAOShashID([]byte(tbn)) + sqlcmd := " " + tbn + " values(" + vl := m.Samples[0].GetValue() + vls := strconv.FormatFloat(vl, 'E', -1, 64) + + if vls == "NaN" { + vls = "null" + } + tl := m.Samples[0].GetTimestamp() + tls := strconv.FormatInt(tl, 10) + sqlcmd = sqlcmd + tls + "," + vls + ")\n" + batchChans[idx%sqlworkers] <- sqlcmd + return nil +} + +func createDatabase(dbname string) { + db, err := sql.Open(taosDriverName, dbuser+":"+dbpassword+"@/tcp("+daemonIP+")/") + if err != nil { + log.Fatalf("Open database error: %s\n", err) + } + defer db.Close() + sqlcmd := fmt.Sprintf("create database if not exists %s tables %d", dbname, tablepervnode) + _, err = db.Exec(sqlcmd) + sqlcmd = fmt.Sprintf("use %s", dbname) + _, err = db.Exec(sqlcmd) + checkErr(err) + return +} + +func execSql(dbname string, sqlcmd string, db *sql.DB) (sql.Result, error) { + if len(sqlcmd) < 1 { + return nil, nil + } + if debugprt == 2 { + blmLog.Println(sqlcmd) + } + res, err := db.Exec(sqlcmd) + if err != nil { + blmLog.Printf("execSql Error: %s sqlcmd: %s\n", err, sqlcmd) + + } + return res, err +} + +func checkErr(err error) { + if err != nil { + blmLog.Println(err) + } +} + +func md5V2(str string) string { + data := []byte(str) + has := md5.Sum(data) + md5str := fmt.Sprintf("%x", has) + return md5str +} + +func processBatches(iworker int) { + var i int + db, err := sql.Open(taosDriverName, dbuser+":"+dbpassword+"@/tcp("+daemonIP+")/"+dbname) + if err != nil { + blmLog.Printf("processBatches Open database error: %s\n", err) + var count int = 5 + for { + if err != nil && count > 0 { + <-time.After(time.Second * 1) + _, err = sql.Open(taosDriverName, dbuser+":"+dbpassword+"@/tcp("+daemonIP+")/"+dbname) + count-- + } else { + if err != nil { + blmLog.Printf("processBatches Error: %s open database\n", err) + return + } + break + } + } + } + defer db.Close() + sqlcmd := make([]string, batchSize+1) + i = 0 + sqlcmd[i] = "Import into" + i++ + //blmLog.Printf("processBatches") + for onepoint := range batchChans[iworker] { + sqlcmd[i] = onepoint + i++ + if i > batchSize { + i = 1 + //blmLog.Printf(strings.Join(sqlcmd, "")) + _, err := db.Exec(strings.Join(sqlcmd, "")) + if err != nil { + blmLog.Printf("processBatches error %s", err) + var count int = 2 + for { + if err != nil && count > 0 { + <-time.After(time.Second * 1) + _, err = db.Exec(strings.Join(sqlcmd, "")) + count-- + } else { + if err != nil { + blmLog.Printf("Error: %s sqlcmd: %s\n", err, strings.Join(sqlcmd, "")) + } + break + } + + } + } + } + } + if i > 1 { + i = 1 + // blmLog.Printf(strings.Join(sqlcmd, "")) + _, err := db.Exec(strings.Join(sqlcmd, "")) + if err != nil { + var count int = 2 + for { + if err != nil && count > 0 { + <-time.After(time.Second * 1) + _, err = db.Exec(strings.Join(sqlcmd, "")) + count-- + } else { + if err != nil { + blmLog.Printf("Error: %s sqlcmd: %s\n", err, strings.Join(sqlcmd, "")) + } + break + } + } + } + } + + workersGroup.Done() +} + +func TestSerialization() { + var req prompb.WriteRequest + var ts []*prompb.TimeSeries + var tse prompb.TimeSeries + var sample *prompb.Sample + var label prompb.Label + var lbs []*prompb.Label + promPath, err := os.Getwd() + if err != nil { + fmt.Printf("can't get current dir :%s \n", err) + os.Exit(1) + } + promPath = filepath.Join(promPath, "testData/blm_prometheus.log") + testfile, err := os.OpenFile(promPath, os.O_RDWR, 0666) + if err != nil { + fmt.Println("Open file error!", err) + return + } + defer testfile.Close() + fmt.Println(promPath) + buf := bufio.NewReader(testfile) + i := 0 + lasttime := "20:40:20" + for { + line, err := buf.ReadString('\n') + if err != nil { + if err == io.EOF { + fmt.Println("File read ok! line:", i) + break + } else { + fmt.Println("Read file error!", err) + return + } + } + sa := strings.Split(line, " ") + + if strings.Contains(line, "server.go:201:") { + if sa[3] != lasttime { + nodeChans[0] <- req + lasttime = sa[3] + req.Timeseries = req.Timeseries[:0] + ts = ts[:0] + } + tse.Samples = make([]prompb.Sample, 0) + T, _ := strconv.ParseInt(sa[7][:(len(sa[7])-1)], 10, 64) + V, _ := strconv.ParseFloat(sa[9][:(len(sa[9])-1)], 64) + sample = &prompb.Sample{ + Value: V, + Timestamp: T, + } + tse.Samples = append(tse.Samples, *sample) + } else if strings.Contains(line, "server.go:202:") { + lbs = make([]*prompb.Label, 0) + lb := strings.Split(line[45:], "{") + label.Name = "__name__" + label.Value = lb[0] + lbs = append(lbs, &label) + lbc := strings.Split(lb[1][:len(lb[1])-1], ", ") + for i = 0; i < len(lbc); i++ { + content := strings.Split(lbc[i], "=\"") + label.Name = content[0] + if i == len(lbc)-1 { + label.Value = content[1][:len(content[1])-2] + } else { + + label.Value = content[1][:len(content[1])-1] + } + lbs = append(lbs, &label) + } + tse.Labels = lbs + ts = append(ts, &tse) + req.Timeseries = ts + } + + } + +} + +func TAOSstrCmp(a string, b string) bool { + //return if a locates before b in a dictrionary. + for i := 0; i < len(a) && i < len(b); i++ { + if int(a[i]-'0') > int(b[i]-'0') { + return false + } else if int(a[i]-'0') < int(b[i]-'0') { + return true + } + } + if len(a) > len(b) { + return false + } else { + return true + } +} + +func OrderInsertS(s string, l *list.List) { + e := l.Front() + if e == nil { + l.PushFront(s) + return + } + + for e = l.Front(); e != nil; e = e.Next() { + str := e.Value.(string) + + if TAOSstrCmp(str, s) { + continue + } else { + l.InsertBefore(s, e) + return + } + } + l.PushBack(s) + return +} diff --git a/bailongma/blm_telegraf/Dockerfile b/bailongma/blm_telegraf/Dockerfile new file mode 100644 index 0000000000000000000000000000000000000000..208bbdd34589c4b645f056dfd631d9cc5e4d2c26 --- /dev/null +++ b/bailongma/blm_telegraf/Dockerfile @@ -0,0 +1,40 @@ +## Builder image +FROM tdengine/tdengine:dev as builder1 +FROM tdengine/bailongma:dev as builder + + +COPY server.go /root/blm_telegraf/ + +WORKDIR /root/blm_telegraf + + +RUN go build +# # build community version +# RUN cmake .. -DVERSION=lite && cmake --build . + +## Target image + + +FROM centos:7 + +WORKDIR /root + +# COPY --from=builder /root/build/build/lib/libtaos.so /usr/lib/libtaos.so.1 +# RUN ln -s /usr/lib/libtaos.so.1 /usr/lib/libtaos.so +COPY --from=builder1 /usr/include/taos.h /usr/include/ +COPY --from=builder1 /usr/lib/libtaos.so /usr/lib/libtaos.so +RUN ln -s /usr/lib/libtaos.so /usr/lib/libtaos.so.1 + + +COPY --from=builder /root/blm_telegraf/blm_telegraf . + +#COPY community/packaging/cfg/taos.cfg /etc/taos/ + +ENV LD_LIBRARY_PATH="$LD_LIBRARY_PATH:/usr/lib" +ENV LANG=en_US.UTF-8 +ENV LANGUAGE=en_US:en +ENV LC_ALL=en_US.UTF-8 +EXPOSE 10202 +VOLUME [ "/var/lib/taos", "/var/log/taos" ] + +ENTRYPOINT [ "/root/blm_telegraf","--host" ] \ No newline at end of file diff --git a/bailongma/blm_telegraf/README.md b/bailongma/blm_telegraf/README.md new file mode 100644 index 0000000000000000000000000000000000000000..b9641a29cb054fad39aec2ffb6513bcbff9eac23 --- /dev/null +++ b/bailongma/blm_telegraf/README.md @@ -0,0 +1,153 @@ +# API for Telegraf + +This is an API to support Telegraf writing data into TDengine. + +## prerequisite + +before running the software, you need to install the `golang-1.10` or later version in your environment and install [TDengine][] so the program can use the lib of TDengine. + +To use it: + +``` +go build +``` +During the go build process, there maybe some errors arised because of lacking some needed packages. You can use `go get` the package to solve it +``` +go get github.com/taosdata/TDengine/src/connector/go/src/taosSql + +``` +After successful build, there will be a blm_telegraf in the same directory. + +## Running in background + +Using following command to run the program in background + +``` +nohup ./blm_telegraf --host 112.102.3.69:0 --batch-size 200 --http-workers 2 --sql-workers 2 --dbname telegraf --port 1234 > /dev/null 2>&1 & +``` +The API url is `http://ipaddress:port/telegraf` + +There are several options can be set: + +``` +--host +set the host of TDengine, IP:port, for example "192.168.0.1:0" + +--batch-size +set the size of how many records in one SQL cmd line writing into TDengine. There is a limitation that TDengine could only accept SQL line small than 64000 bytes, so usually the batch size should not exceed 800. Default is 10. + +--http-workers +set the number of workers who process the HTTP request. default is 10 + +--sql-workers +set the number of workers who process the database request. default is 10 + +--dbname +set the database name in TDengine, if not exists, a database will be created after this dbname. default is "telegraf". + +--dbuser +set the user name that have the right to access the TDengine. default is "root" + +--dbpassword +set the password of dbuser. default is "taosdata" + +--port +set the port that prometheus configuration remote_write. as showed above, in the prometheus.yaml + +``` + +## Configure the Telegraf + +To write into blm_telegraf API, you should configure the telegraf as below +In the telegraf configuration file, output plugin part: + +1. telegraf out put plugin setup: +Set the url to the blm_telegraf API +Set the data format as "json" +Set the json timstamp units as "1ms" +```toml +[[outputs.http]] +# ## URL is the address to send metrics to +url = "http://114.116.124.178:8081/telegraf" + +data_format = "json" +json_timestamp_units = "1ms" + +``` +In the Agent part, the hostname should be unique among all the telegraf which report to the TDengine. + +```toml +# Configuration for telegraf agent +[agent] + ## Default data collection interval for all inputs + interval = "5s" +... + + ## Override default hostname, if empty use os.Hostname() + hostname = "testhost1" + +``` +Then start telegraf: + +```sh + telegraf --config xxx.conf +``` +Then you can check the TDengine if there is super table and tables. + +## Check the TDengine tables and datas + +Use the taos client shell to query the result. +``` +Welcome to the TDengine shell from linux, client version:1.6.4.0 server version:1.6.4.0 +Copyright (c) 2017 by TAOS Data, Inc. All rights reserved. + +This is the trial version and will expire at 2019-12-11 14:25:31. + +taos> use prometheus; +Database changed. + +taos> show stables; + name | created_time |columns| tags | tables | +==================================================================================================================== +system | 19-11-22 21:48:10.205| 2| 3| 12| +system_str | 19-11-22 21:48:10.205| 2| 3| 2| +cpu | 19-11-22 21:48:10.225| 2| 4| 200| +cpu_str | 19-11-22 21:48:10.226| 2| 4| 0| +processes | 19-11-22 21:48:10.230| 2| 3| 16| +processes_str | 19-11-22 21:48:10.230| 2| 3| 0| +disk | 19-11-22 21:48:10.233| 2| 7| 357| +disk_str | 19-11-22 21:48:10.234| 2| 7| 0| +diskio | 19-11-22 21:48:10.247| 2| 4| 72| +diskio_str | 19-11-22 21:48:10.248| 2| 4| 0| +swap | 19-11-22 21:48:10.254| 2| 3| 7| +swap_str | 19-11-22 21:48:10.255| 2| 3| 0| +mem | 19-11-22 21:48:10.272| 2| 3| 61| +mem_str | 19-11-22 21:48:10.272| 2| 3| 0| +Query OK, 14 row(s) in set (0.000733s) + + + +taos> select * from mem; + +...... + + 19-11-23 14:19:11.000| 0.000000000|testhost1 |1.202.240.226 |huge_pages_free | + 19-11-23 14:19:16.000| 0.000000000|testhost1 |1.202.240.226 |huge_pages_free | + 19-11-23 14:19:21.000| 0.000000000|testhost1 |1.202.240.226 |huge_pages_free | + 19-11-23 14:19:26.000| 0.000000000|testhost1 |1.202.240.226 |huge_pages_free |Query OK, 3029 row(s) in set (0.060828s) + + +``` + +## Support Kubernates liveness probe +The blm_telegraf support the liveness probe. + +When the service is running, GET the url`http://ip:port/health` will return 200 OK response which means the service is running healthy. If no response, means the service is dead and need to restart it. + + +## Limitations + +The TDengine limits the length of super table name, so if the name of Telegraf measurement name exceeds 60 byte, it will be truncated to first 60 bytes. And the length of tags name is limited within 50 byte. + + +[TDengine]:https://www.github.com/Taosdata/TDengine \ No newline at end of file diff --git "a/bailongma/blm_telegraf/Telegraf\345\206\231\345\205\245TDengine\347\232\204API.md" "b/bailongma/blm_telegraf/Telegraf\345\206\231\345\205\245TDengine\347\232\204API.md" new file mode 100644 index 0000000000000000000000000000000000000000..e7b9d8668a92dc23bd166e2d280c7cd0d436ff38 --- /dev/null +++ "b/bailongma/blm_telegraf/Telegraf\345\206\231\345\205\245TDengine\347\232\204API.md" @@ -0,0 +1,134 @@ +# Telegraf写入TDengine的API程序 + +刚刚开发完Telegraf写入TDengine的APi程序，本文总结一下程序的设计思路，并在此基础上提出一种schemaless的写入TDengine的适配方法和对应的查询思路。 + +## Telegraf数据分析 +Telegraf采集节点的数据后，按照数据的格式为measurement加上一系列的tags，再加上一系列的fields和timestamp，组成一条记录发出。 + +``` +cpu,cpu=cpu-total,host=liutaodeMacBook-Pro.local usage_irq=0,usage_guest=0,usage_guest_nice=0,usage_iowait=0,usage_softirq=0,usage_steal=0,usage_user=10.55527763881941,usage_system=3.5767883941970986,usage_idle=85.86793396698349,usage_nice=0 1571663200000000000 +``` +上面是一条按照influxdb格式输出的记录，第一个字段是measurement, 然后接着两个tags, tags后面的空格来作为tags和fields的分隔；fields和timestamp之间也是用空格分隔。 + +```json +{ + "fields":{ + "usage_guest":0, + "usage_guest_nice":0, + "usage_idle":87.73726273726274, + "usage_iowait":0, + "usage_irq":0, + "usage_nice":0, + "usage_softirq":0, + "usage_steal":0, + "usage_system":2.6973026973026974, + "usage_user":9.565434565434565 + }, + "name":"cpu", + "tags":{ + "cpu":"cpu-total", + "host":"liutaodeMacBook-Pro.local" + }, + "timestamp":1571665100 +} +``` +如果按json格式输出，则一条json格式的记录如上。 +上面的数据个看上去跟TDengine的记录格式十分类似，很自然的我们可以把name作为超级表名，tags作为tags，fields作为values，timestamp作为timestamp，对应上TDengine的数据格式，照理说应该非常好写入TDengine。但实际过程中发现telegraf在输出数据时，经常会遇到一个问题，就是name，tags格式一样的情况下，fields的格式不一样，fields里的名字，数量，都可能变化。这种变化不是随意变化，可能是两三种组合的变化。比如如下情况 +``` +swap,host=testhost1 out=0i,in=0i 1574663615000000000 +swap,host=testhost1 total=4294967296i,used=3473670144i,free=821297152i,used_percent=80.877685546875 1574663615000000000 +``` +同一个时间点来的两条记录，name都是swap，tag都是host，但fields却完全不相同。 + +再比如 +``` +system,host=testhost1 uptime_format="5 days, 1:07" 1574663615000000000 +system,host=testhost1 uptime=436070i 1574663615000000000 +system,host=testhost1 load15=5.9521484375,n_cpus=4i,n_users=6i,load1=3.17138671875,load5=6.462890625 1574663615000000000 +``` +同一时间点来的三条记录，name都是system，tags都是host，但fields完全不同。 + +如果以name作为TDengine的超级表名，就会面临到表格的结构发生变化。并且，由于influxdb是schemaless的设计，他们能够很好的处理这种变化，不管fields如何变化，都能顺利写入。因此，很难保证telegraf后续产生的数据，fields发生会怎么变化。如何设计TDengine的存储表结构，是一个问题。 + +## TDengine的表结构设计思路 +面对这种数据来源，我们一般可以用两种设计思路： +### 提前设计好超级表，包含所有fields, 这种就是schema方式 +一种，是在对数据的行为有充分的了解后，提前设计好TDengine的表格式，将所有可能变化的fields都放到values中，提前创建好超级表；然后在插入数据时，把同一时间戳的所有fields都收集齐后，再组装成一条TDengine的SQL记录，写入TDengine。这种，是我们当前通常用到的方法，可以成为schema方式的写入。这种方法的优点是，比较符合我们TDengine的设计假设，values有多列，写入性能相对高一些。但也有明显的缺点，需要提前对数据做大量的分析，确定每个测量的格式，手动来写schema配置文件或手动在TDengine客户端创建超级表，对于节点很多的监控，这种设计会带来较大的工作量。 + +### 不提前设计超级表，设计一种建表规则，根据来的数据自动写入，类似influxdb的schemaless的方式 +另外一种，就是根据收到的数据自动建表，只要符合name，tags，fields，timestamp的格式，都能顺利写的创建TDengine的表，并写入TDengine。本程序就是采用这种思路，将每一个fields单独拆开，和name，tags组合起来，形成一个单列的表。这样的超级表符合任何fields，对于任意fields都可以顺利写入。下面将沿着这个设计思路继续展开。 + +## 超级表 +本程序以收到的原始数据name作为超级表名，原始数据中的tags作为tags，同时，额外增加两个tag，一个是发来请求的源IP，用来区分设备；另一个是field，这个tag的值是原始数据中fields的名称，用来表明这个超级表存的是哪个指标。以上面的system这个原始数据为例，则超级表结构为 +```toml +stablename = system +tags = ['host','srcip','field'] +values = ['timestamp','value'] +``` +其中，tags的类型都为binary(50),长度超过50的标签值都截断为50；field这个标签则的可能值则为 +```toml +field : ['uptime_format','uptime','load15','n_cpus','n_users','load1'，'load5'] +``` +## value的类型 +由于无法预知数据的类型，以及简化程序实现，我们将value的类型分成两类，一类是数值型，统一用double来存储；一类是字符串，统一用binary(256)的类型来存。由于所有field都要用同一个超级表来存，因此我一开始就为每个name创建了两个超级表，一个是数值型的超级表，表名就是name；另一个是字符串型的超级表，表名是name加上_str后缀。然后根据field的数据类型，如果是数值型，就用数值型的超级表来创建表；如果是字符串型的，就用name_str的超级表来创建表。 +因此，超级表创建的时候会创建两倍的数据量 +``` + name | created_time |columns| tags | tables | +==================================================================================================================== +system | 19-11-22 21:48:10.205| 2| 3| 12| +system_str | 19-11-22 21:48:10.205| 2| 3| 2| +cpu | 19-11-22 21:48:10.225| 2| 4| 200| +cpu_str | 19-11-22 21:48:10.226| 2| 4| 0| +processes | 19-11-22 21:48:10.230| 2| 3| 16| +processes_str | 19-11-22 21:48:10.230| 2| 3| 0| +disk | 19-11-22 21:48:10.233| 2| 7| 357| +disk_str | 19-11-22 21:48:10.234| 2| 7| 0| +diskio | 19-11-22 21:48:10.247| 2| 4| 72| +diskio_str | 19-11-22 21:48:10.248| 2| 4| 0| +swap | 19-11-22 21:48:10.254| 2| 3| 7| +swap_str | 19-11-22 21:48:10.255| 2| 3| 0| +mem | 19-11-22 21:48:10.272| 2| 3| 61| +mem_str | 19-11-22 21:48:10.272| 2| 3| 0| +Query OK, 14 row(s) in set (0.000588s) + +``` +因此查询的时候，需要根据查询值的类型，选择不同的超级表来查询 +比如，对于数值类型，查询n_cpus值的语句为 +```sql +Select * from system where field = "n_cpus"; +``` + +对于字符串类型，查询uptime_format的值的语句为 +```sql +Select * from system_str where field = "uptime_format"; +``` + +## 表的创建 +对于每个field，程序为它创建了一个表，表名规则如下： +将原始数据的所有tags值加上源ip加上field的名称，组成一个长的字符串，然后进行MD5计算，输出的结果加上MD5_作为前缀，形成表名。 +这种规则，确保了只要数据的tags等特征不变，表就不会发生变化。 +``` +... +md5_b26d30c2e07529ac309d836b3b222f15 | 19-11-24 21:34:35.830| 2|processes | +md5_08147d718d4961368155f90432eab536 | 19-11-22 21:48:10.748| 2|disk | +md5_105158abfca0bbf0d932cc74bfc7e136 | 19-11-24 21:34:35.846| 2|mem | +md5_e6842b5c6b9744b7d5ce3510a4d54c98 | 19-11-24 21:34:35.874| 2|disk | +md5_285fd02686e0bfee76d22505dd29f14c | 19-11-22 21:48:11.509| 2|disk | +md5_9317870bb00109353f6aef58ee2ee9e9 | 19-11-24 21:34:35.919| 2|cpu | +Query OK, 727 row(s) in set (0.020405s) + +``` +因此在数据插入时，只要根据tags值和IP和field的名称，就能计算出表名，直接插入该表。 +查询时，可以用超级表加上tag值和field值来查询，也很清晰便利。 + +因此，数值写入自动生成的sql语句如下： +```sql +insert into md5_285fd02686e0bfee76d22505dd29f14c values(1574663615000,375.2023040); +``` +其中表名md5_285fd02686e0bfee76d22505dd29f14c是自动根据数据特征计算出来的，无需人工输入。 + +而查询则可以通过超级表来查询 + +## Schemaless写入方法 +基于上面的实现，后续我们可以确定一个写入的语法，就可以不用提前设定schema，而根据接收到的数据，自动创建超级表，表，方便的写入TDengine了。 +沿用现有的json格式，或者参考influxdb的语法，确定一个写入的语法，就可以实现influxdb的schemaless写入的能力。 diff --git a/bailongma/blm_telegraf/server.go b/bailongma/blm_telegraf/server.go new file mode 100644 index 0000000000000000000000000000000000000000..2e3707f66f8b0a024aae261cc452e34a1b2119b9 --- /dev/null +++ b/bailongma/blm_telegraf/server.go @@ -0,0 +1,551 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package main + +import ( + "container/list" + "crypto/md5" + "database/sql" + "encoding/json" + "flag" + "fmt" + "io/ioutil" + "log" + "net/http" + "os" + "sort" + "strconv" + "strings" + "sync" + "time" + + _ "github.com/taosdata/driver-go/taosSql" +) + +type metric struct { + Fields map[string]interface{} + Name string + Tags map[string]string + TimeStamp int64 +} + +type Metrics struct { + Metrics []metric + HostIP string +} + +var ( + daemonUrl string + httpworkers int + sqlworkers int + batchSize int + buffersize int + dbname string + dbuser string + dbpassword string + rwport string + debugprt int + taglen int +) + +type nametag struct { + tagmap map[string]string + taglist *list.List +} + +// Global vars +var ( + bufPool sync.Pool + batchChans []chan string //multi table one chan + nodeChans []chan Metrics //multi node one chan + inputDone chan struct{} + //workersGroup sync.WaitGroup + reportTags [][2]string + reportHostname string + taosDriverName string = "taosSql" + IsSTableCreated sync.Map + IsTableCreated sync.Map + taglist *list.List + nametagmap map[string]nametag + tagstr string + blmLog *log.Logger + logNameDefault string = "/var/log/taos/blm_telegraf.log" +) +var scratchBufPool = &sync.Pool{ + New: func() interface{} { + return make([]byte, 0, 1024) + }, +} + +// Parse args: +func init() { + flag.StringVar(&daemonUrl, "host", "", "TDengine host.") + + flag.IntVar(&batchSize, "batch-size", 10, "Batch size (input items).") + flag.IntVar(&httpworkers, "http-workers", 10, "Number of parallel http requests handler .") + flag.IntVar(&sqlworkers, "sql-workers", 10, "Number of parallel sql handler.") + flag.StringVar(&dbname, "dbname", "telegraf", "Database name where to store metrics") + flag.StringVar(&dbuser, "dbuser", "root", "User for host to send result metrics") + flag.StringVar(&dbpassword, "dbpassword", "taosdata", "User password for Host to send result metrics") + flag.StringVar(&rwport, "port", "10202", "remote write port") + flag.IntVar(&debugprt, "debugprt", 0, "if 0 not print, if 1 print the sql") + flag.IntVar(&taglen, "tag-length", 30, "the max length of tag string") + flag.IntVar(&buffersize, "buffersize", 100, "the buffer size of metrics received") + + flag.Parse() + daemonUrl = daemonUrl + ":0" + nametagmap = make(map[string]nametag) + fmt.Print("host: ") + fmt.Print(daemonUrl) + fmt.Print(" port: ") + fmt.Print(rwport) + fmt.Print(" database: ") + fmt.Print(dbname) + tagstr = fmt.Sprintf(" binary(%d)", taglen) + logFile, err := os.OpenFile(logNameDefault, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644) + if err != nil { + fmt.Println(err) + os.Exit(1) + } + blmLog = log.New(logFile, "", log.LstdFlags) + blmLog.SetPrefix("BLM_TLG") + blmLog.SetFlags(log.LstdFlags | log.Lshortfile) + +} + +func main() { + + for i := 0; i < httpworkers; i++ { + nodeChans = append(nodeChans, make(chan Metrics, buffersize)) + } + + createDatabase(dbname) + + for i := 0; i < httpworkers; i++ { + //workersGroup.Add(1) + go NodeProcess(i) + } + + for i := 0; i < sqlworkers; i++ { + batchChans = append(batchChans, make(chan string, batchSize)) + } + + for i := 0; i < sqlworkers; i++ { + //workersGroup.Add(1) + go processBatches(i) + } + + http.HandleFunc("/telegraf", func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusAccepted) + addr := strings.Split(r.RemoteAddr, ":") + idx := TAOShashID([]byte(addr[0])) + + reqBuf, err := ioutil.ReadAll(r.Body) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + r.Body.Close() + var req Metrics + if err := json.Unmarshal(reqBuf, &req); err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + req.HostIP = addr[0] + + nodeChans[idx%httpworkers] <- req + + r.Body.Close() + + }) + http.HandleFunc("/health", func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusNoContent) + }) + + blmLog.Fatal(http.ListenAndServe(":"+rwport, nil)) + +} + +func TAOShashID(ba []byte) int { + var sum int = 0 + + for i := 0; i < len(ba); i++ { + sum += int(ba[i] - '0') + } + + return sum +} + +func TAOSstrCmp(a string, b string) bool { + //return if a locates before b in a dictrionary. + for i := 0; i < len(a) && i < len(b); i++ { + if int(a[i]-'0') > int(b[i]-'0') { + return false + } else if int(a[i]-'0') < int(b[i]-'0') { + return true + } + } + if len(a) > len(b) { + return false + } else { + return true + } +} + +func NodeProcess(workerid int) error { + + for req := range nodeChans[workerid] { + + ProcessReq(req) + + } + + return nil +} + +func OrderInsert(ts int64, l *list.List) { + e := l.Front() + if e == nil { + l.PushFront(ts) + return + } + + for e = l.Front(); e != nil; e = e.Next() { + + if e.Value.(int64) < ts { + continue + } else { + l.InsertBefore(ts, e) + return + } + } +} + +func OrderInsertS(s string, l *list.List) { + e := l.Front() + if e == nil { + l.PushFront(s) + return + } + + for e = l.Front(); e != nil; e = e.Next() { + str := e.Value.(string) + + if TAOSstrCmp(str, s) { + continue + } else { + l.InsertBefore(s, e) + return + } + } + l.PushBack(s) + return +} + +func ProcessReq(req Metrics) error { + + tsmap := make(map[int64]map[string][]metric) + tslist := list.New() + addr := req.HostIP + + var lastTs int64 = 0 + for i := 0; i < len(req.Metrics); i++ { + m := req.Metrics[i] + if tsmap[m.TimeStamp] == nil { + tsmap[m.TimeStamp] = make(map[string][]metric) + } + + mp := tsmap[m.TimeStamp] + mp[m.Name] = append(mp[m.Name], m) + + if lastTs != m.TimeStamp { //there is still some case that will make mistake, when the timestamp is totally out of order. but right now just forget it. + OrderInsert(m.TimeStamp, tslist) + } + lastTs = m.TimeStamp + } + + for e := tslist.Front(); e != nil; e = e.Next() { + + namemap, ok := tsmap[e.Value.(int64)] + if ok { + for _, v := range namemap { + + ProcessData(v, dbname, addr) + } + } else { + info := fmt.Sprintf("ProcessReq: cannot retrieve map") + panic(info) + } + + } + + return nil +} + +func SerilizeTDengine(m metric, dbn string, hostip string, taglist *list.List, db *sql.DB) error { + var tbna []string + + for _, v := range m.Tags { + tbna = append(tbna, v) + } + sort.Strings(tbna) + tbn := strings.Join(tbna, "") // Go map 遍历结果是随机的，必须排下序 + + for k, v := range m.Fields { + s := m.Name + tbn + hostip + k + //fmt.Print(s) + s = "MD5_" + md5V2(s) + _, ok := IsTableCreated.Load(s) + if !ok { + var sqlcmd string + switch v.(type) { + case string: + sqlcmd = "create table if not exists " + s + " using " + m.Name + "_str tags(" + default: + sqlcmd = "create table if not exists " + s + " using " + m.Name + " tags(" + } + + for e := taglist.Front(); e != nil; e = e.Next() { + tagvalue, has := m.Tags[e.Value.(string)] + if len(tagvalue) >= 60 { + tagvalue = tagvalue[:59] + } + if has { + sqlcmd = sqlcmd + "\"" + tagvalue + "\"," + } else { + sqlcmd = sqlcmd + "null," + } + } + sqlcmd = sqlcmd + "\"" + hostip + "\"," + "\"" + k + "\")\n" + execSql(dbn, sqlcmd, db) + IsTableCreated.Store(s, true) + } + idx := TAOShashID([]byte(s)) + sqlcmd := " " + s + " values(" + + tls := strconv.FormatInt(m.TimeStamp, 10) + switch v.(type) { + case string: + sqlcmd = sqlcmd + tls + ",\"" + v.(string) + "\")" + case int64: + sqlcmd = sqlcmd + tls + "," + strconv.FormatInt(v.(int64), 10) + ")" + case float64: + sqlcmd = sqlcmd + tls + "," + strconv.FormatFloat(v.(float64), 'E', -1, 64) + ")" + default: + panic("Checktable error value type") + } + batchChans[idx%sqlworkers] <- sqlcmd + //execSql(dbn,sqlcmd) + } + return nil +} + +func ProcessData(ts []metric, dbn string, hostip string) error { + db, err := sql.Open(taosDriverName, dbuser+":"+dbpassword+"@/tcp("+daemonUrl+")/"+dbname) + if err != nil { + blmLog.Fatalf("Open database error: %s\n", err) + } + defer db.Close() + schema, ok := IsSTableCreated.Load(ts[0].Name) + + if !ok { + var nt nametag + nt.taglist = list.New() + nt.tagmap = make(map[string]string) + IsSTableCreated.Store(ts[0].Name, nt) + tagmap := nt.tagmap + taglist := nt.taglist + + for i := 0; i < len(ts); i++ { + + for k, _ := range ts[i].Tags { + _, ok := tagmap[k] + if !ok { + taglist.PushBack(k) + tagmap[k] = "y" + } + } + } + var sqlcmd string + sqlcmd = "create table if not exists " + ts[0].Name + " (ts timestamp, value double) tags(" + sqlcmd1 := "create table if not exists " + ts[0].Name + "_str (ts timestamp, value binary(256)) tags(" + for e := taglist.Front(); e != nil; e = e.Next() { + sqlcmd = sqlcmd + e.Value.(string) + tagstr + "," + sqlcmd1 = sqlcmd1 + e.Value.(string) + tagstr + "," + } + sqlcmd = sqlcmd + "srcip binary(20), field binary(40))\n" + sqlcmd1 = sqlcmd1 + "srcip binary(20), field binary(40))\n" + execSql(dbn, sqlcmd, db) + execSql(dbn, sqlcmd1, db) + for i := 0; i < len(ts); i++ { + SerilizeTDengine(ts[i], dbn, hostip, taglist, db) + } + + return nil + } + nt := schema.(nametag) + tagmap := nt.tagmap + taglist := nt.taglist + + var sqlcmd, sqlcmd1 string + for i := 0; i < len(ts); i++ { + + for k, _ := range ts[i].Tags { + _, ok := tagmap[k] + if !ok { + sqlcmd = sqlcmd + "alter table " + ts[0].Name + " add tag " + k + tagstr + "\n" + sqlcmd1 = sqlcmd1 + "alter table " + ts[0].Name + "_str add tag " + k + tagstr + "\n" + taglist.PushBack(k) + tagmap[k] = "y" + + } + } + + } + execSql(dbn, sqlcmd, db) + execSql(dbn, sqlcmd1, db) + + for i := 0; i < len(ts); i++ { + SerilizeTDengine(ts[i], dbn, hostip, taglist, db) + } + return nil +} + +func createDatabase(dbname string) { + db, err := sql.Open(taosDriverName, dbuser+":"+dbpassword+"@/tcp("+daemonUrl+")/") + if err != nil { + log.Fatalf("Open database error: %s\n", err) + } + defer db.Close() + sqlcmd := fmt.Sprintf("create database if not exists %s", dbname) + _, err = db.Exec(sqlcmd) + sqlcmd = fmt.Sprintf("use %s", dbname) + _, err = db.Exec(sqlcmd) + checkErr(err) + return +} + +func execSql(dbname string, sqlcmd string, db *sql.DB) { + if len(sqlcmd) < 1 { + return + } + _, err := db.Exec(sqlcmd) + if err != nil { + var count int = 2 + for { + if err != nil && count > 0 { + <-time.After(time.Second * 1) + _, err = db.Exec(sqlcmd) + count-- + } else { + if err != nil { + blmLog.Printf("execSql Error: %s sqlcmd: %s\n", err, sqlcmd) + return + } + break + } + + } + } + return +} + +func checkErr(err error) { + if err != nil { + blmLog.Println(err) + } +} + +func md5V2(str string) string { + data := []byte(str) + has := md5.Sum(data) + md5str := fmt.Sprintf("%x", has) + return md5str +} + +func processBatches(iworker int) { + var i int + db, err := sql.Open(taosDriverName, dbuser+":"+dbpassword+"@/tcp("+daemonUrl+")/"+dbname) + if err != nil { + blmLog.Printf("processBatches Open database error: %s\n", err) + var count int = 5 + for { + if err != nil && count > 0 { + <-time.After(time.Second * 1) + _, err = sql.Open(taosDriverName, dbuser+":"+dbpassword+"@/tcp("+daemonUrl+")/"+dbname) + count-- + } else { + if err != nil { + blmLog.Printf("processBatches Error: %s open database\n", err) + return + } + break + } + } + } + defer db.Close() + sqlcmd := make([]string, batchSize+1) + i = 0 + sqlcmd[i] = "Insert into" + i++ + + for onepoint := range batchChans[iworker] { + sqlcmd[i] = onepoint + i++ + if i > batchSize { + i = 1 + _, err := db.Exec(strings.Join(sqlcmd, "")) + if err != nil { + + var count int = 2 + for { + if err != nil && count > 0 { + <-time.After(time.Second * 1) + _, err = db.Exec(strings.Join(sqlcmd, "")) + count-- + } else { + if err != nil { + blmLog.Printf("Error: %s sqlcmd: %s\n", err, strings.Join(sqlcmd, "")) + } + break + } + + } + } + } + } + if i > 1 { + i = 1 + _, err := db.Exec(strings.Join(sqlcmd, "")) + if err != nil { + var count int = 2 + for { + if err != nil && count > 0 { + <-time.After(time.Second * 1) + _, err = db.Exec(strings.Join(sqlcmd, "")) + count-- + } else { + if err != nil { + blmLog.Printf("Error: %s sqlcmd: %s\n", err, strings.Join(sqlcmd, "")) + } + break + } + } + } + } + + //workersGroup.Done() +} diff --git a/bailongma/go.mod b/bailongma/go.mod new file mode 100644 index 0000000000000000000000000000000000000000..bcf6546542c30b69ea6076c8d4a495824def16d0 --- /dev/null +++ b/bailongma/go.mod @@ -0,0 +1,14 @@ +module github.com/taosdata/TDengine/bailongma + +go 1.14 + +require ( + github.com/gogo/protobuf v1.3.1 + github.com/golang/snappy v0.0.1 + github.com/grpc-ecosystem/grpc-gateway v1.14.5 // indirect + github.com/prometheus/common v0.9.1 + github.com/prometheus/prometheus v2.5.0+incompatible + github.com/taosdata/driver-go v0.0.0-20200311072652-8c58c512b6ac + google.golang.org/genproto v0.0.0-20200507105951-43844f6eee31 // indirect + google.golang.org/grpc v1.29.1 // indirect +) diff --git a/bailongma/go.sum b/bailongma/go.sum new file mode 100644 index 0000000000000000000000000000000000000000..30d61ca351c41a6110814cc52cf230bbeafb7383 --- /dev/null +++ b/bailongma/go.sum @@ -0,0 +1,130 @@ +cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= +github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= +github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/antihax/optional v0.0.0-20180407024304-ca021399b1a6/go.mod h1:V8iCPQYkqmusNa815XgQio277wI47sdRh1dUOLdyC6Q= +github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= +github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= +github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= +github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= +github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= +github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= +github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= +github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= +github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= +github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= +github.com/gogo/protobuf v1.3.1 h1:DqDEcV5aeaTmdFBePNpYsp3FlcVH/2ISVVM9Qf8PSls= +github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.3 h1:gyjaxf+svBWX08ZjK86iN9geUJF0H6gp2IRKX6Nf6/I= +github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= +github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4= +github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= +github.com/grpc-ecosystem/grpc-gateway v1.14.5 h1:aiLxiiVzAXb7wb3lAmubA69IokWOoUNe+E7TdGKh8yw= +github.com/grpc-ecosystem/grpc-gateway v1.14.5/go.mod h1:UJ0EZAp832vCd54Wev9N1BMKEyvcZ5+IM0AwDrnlkEc= +github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= +github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= +github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= +github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= +github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= +github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= +github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= +github.com/prometheus/common v0.9.1 h1:KOMtN28tlbam3/7ZKEYKHhKoJZYYj3gMH4uc62x7X7U= +github.com/prometheus/common v0.9.1/go.mod h1:yhUN8i9wzaXS3w1O07YhxHEBxD+W35wd8bs7vj7HSQ4= +github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= +github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= +github.com/prometheus/prometheus v1.8.2 h1:PAL466mnJw1VolZPm1OarpdUpqukUy/eX4tagia17DM= +github.com/prometheus/prometheus v2.5.0+incompatible h1:7QPitgO2kOFG8ecuRn9O/4L9+10He72rVRJvMXrE9Hg= +github.com/prometheus/prometheus v2.5.0+incompatible/go.mod h1:oAIUtOny2rjMX0OWN5vPR5/q/twIROJvdqnQKDdil/s= +github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= +github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= +github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/taosdata/TDengine v0.0.0-20200511045913-fead5b90383d h1:er7tnj+w7vOo1mLnOL6imwYJdKsvuR2d6/GS8Dyt6oc= +github.com/taosdata/driver-go v0.0.0-20200311072652-8c58c512b6ac h1:uZplMwObJj8mfgI4ZvYPNHRn+fNz2leiMPqShsjtEEc= +github.com/taosdata/driver-go v0.0.0-20200311072652-8c58c512b6ac/go.mod h1:TuMZDpnBrjNO07rneM2C5qMYFqIro4aupL2cUOGGo/I= +golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= +golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190613194153-d28f0bde5980 h1:dfGZHvZk057jK2MCeWus/TowKpJ8y4AmooUzdBSR9GU= +golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20191002035440-2ec189313ef0 h1:2mqDk8w/o6UmeUCu5Qiq2y7iMf6anbx+YA8d1JFoFrs= +golang.org/x/net v0.0.0-20191002035440-2ec189313ef0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190422165155-953cdadca894 h1:Cz4ceDQGXuKRnVBDTS23GTn/pU5OE2C0WrNTOYK1Uuc= +golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= +golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20190927181202-20e1ac93f88c/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= +google.golang.org/genproto v0.0.0-20200507105951-43844f6eee31 h1:Bz1qTn2YRWV+9OKJtxHJiQKCiXIdf+kwuKXdt9cBxyU= +google.golang.org/genproto v0.0.0-20200507105951-43844f6eee31/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= +google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +google.golang.org/grpc v1.24.0/go.mod h1:XDChyiUovWa60DnaeDeZmSW86xtLtjtZbwvSiRnRtcA= +google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= +google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.29.1 h1:EC2SB8S04d2r73uptxphDSUG+kTKVgjRPF+N3xpxRB4= +google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= +gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/deps/dirent/CMakeLists.txt b/deps/dirent/CMakeLists.txt new file mode 100644 index 0000000000000000000000000000000000000000..d4f8b6baf529a5779a2a5b22ecbfc6810ef80c68 --- /dev/null +++ b/deps/dirent/CMakeLists.txt @@ -0,0 +1,53 @@ +cmake_minimum_required (VERSION 2.8.11) +project (dirent LANGUAGES C CXX) + +# Initialize C and C++ compilers +enable_language (C CXX) + +# Compile in debug mode by default +if (NOT CMAKE_BUILD_TYPE) + set (CMAKE_BUILD_TYPE Debug + CACHE STRING + "Type of build: None Debug Release RelWithDebInfo MinSizeRel." + FORCE + ) +endif (NOT CMAKE_BUILD_TYPE) + +# Use the include directory only on Windows +if (WIN32) + include_directories (${CMAKE_SOURCE_DIR}/include) +endif (WIN32) + +# Install dirent.h +if (WIN32) + install (FILES include/dirent.h DESTINATION include) +endif (WIN32) + +# Add distclean target +add_custom_target (distclean + COMMAND ${CMAKE_BUILD_TOOL} clean + COMMAND ${CMAKE_COMMAND} -P ${CMAKE_SOURCE_DIR}/distclean.cmake +) + +# Build example programs +add_executable (find examples/find.c) +add_executable (ls examples/ls.c) +add_executable (locate examples/locate.c) +add_executable (updatedb examples/updatedb.c) +add_executable (scandir examples/scandir.c) +add_executable (cat examples/cat.c) + +# Build test programs +include (CTest) +add_custom_target (check COMMAND ${CMAKE_CTEST_COMMAND} --output-on-failure -C ${CMAKE_CFG_INTDIR}) +function (add_test_executable TEST_NAME) + add_executable (${TEST_NAME} EXCLUDE_FROM_ALL ${ARGN}) + add_test (NAME ${TEST_NAME} WORKING_DIRECTORY ${CMAKE_SOURCE_DIR} COMMAND $) + add_dependencies (check ${TEST_NAME}) +endfunction (add_test_executable) + +add_test_executable (t-compile tests/t-compile.c) +add_test_executable (t-dirent tests/t-dirent.c) +add_test_executable (t-scandir tests/t-scandir.c) +add_test_executable (t-unicode tests/t-unicode.c) +add_test_executable (t-cplusplus tests/t-cplusplus.cpp) diff --git a/deps/dirent/ChangeLog b/deps/dirent/ChangeLog new file mode 100644 index 0000000000000000000000000000000000000000..ce7975ec5e201533d733f30c26505635f700c286 --- /dev/null +++ b/deps/dirent/ChangeLog @@ -0,0 +1,129 @@ +2018-05-08 Toni Rönkkö + + * Version 1.23.2: fixes bad scandir prototype. + +2017-08-27 Toni Rönkkö + + * Version 1.23: support readdir_r and scandir functions. + +2017-07-18 Toni Rönkkö + + * Created release branches v1.22 and v1.21 to Git. Published version + 1.22 at softagalleria.net. + +2016-09-11 Toni Rönkkö + + * Version 1.22: added support for CMake. Thanks to Paul Fultz II. + +2014-09-25 Toni Rönkkö + + * Version 1.21: compiles correctly under Open Watcom. Thanks to + Virgil Banowetz for a patch! + +2014-04-07 Toni Rönkkö + + * Version 1.20.1: the zip file from the previous version did not open + correctly with Microsoft's compressed folders. Thanks to Alexandre + for info! + +2014-03-17 Toni Ronkko + + * Version 1.20: dirent.h compiles correctly in 64-bit architecture. + Thanks to Aaron Simmons! + +2014-03-03 Toni Ronkko + + * Version 1.13.2: define DT_LNK for compatibility with Unix + programs. Thanks to Joel Bruick for suggestion! + +2013-01-27 Toni Ronkko + + * Version 1.13.1: patch from Edward Berner fixes set_errno() on + Windows NT 4.0. + + * Revised wcstombs() and mbstowcs() wrappers to make sure that they do + not write past their target string. + + * PATH_MAX from windows.h includes zero terminator so there is no + need to add one extra byte to variables and structures. + +2012-12-12 Toni Ronkko + + * Version 1.13: use the traditional 8+3 file naming scheme if a file + name cannot be represented in the default ANSI code page. Now + compiles again with MSVC 6.0. Thanks to Konstantin Khomoutov for + testing. + +2012-10-01 Toni Ronkko + + * Version 1.12.1: renamed wide-character DIR structure _wDIR to + _WDIR (with capital W) in order to maintain compatibility with MingW. + +2012-09-30 Toni Ronkko + + * Version 1.12: define PATH_MAX and NAME_MAX. Added wide-character + variants _wDIR, _wdirent, _wopendir(), _wreaddir(), _wclosedir() and + _wrewinddir(). Thanks to Edgar Buerkle and Jan Nijtmans for ideas + and code. + + * Now avoiding windows.h. This allows dirent.h to be integrated + more easily into programs using winsock. Thanks to Fernando + Azaldegui. + +2011-03-15 Toni Ronkko + + * Version 1.11: defined FILE_ATTRIBUTE_DEVICE for MSVC 6.0. + +2010-08-11 Toni Ronkko + + * Version 1.10: added d_type and d_namlen fields to dirent structure. + The former is especially useful for determining whether directory + entry represents a file or a directory. For more information, see + http://www.delorie.com/gnu/docs/glibc/libc_270.html + + * Improved conformance to the standards. For example, errno is now + set properly on failure and assert() is never used. Thanks to Peter + Brockam for suggestions. + + * Fixed a bug in rewinddir(): when using relative directory names, + change of working directory no longer causes rewinddir() to fail. + +2009-12-15 John Cunningham + + * Version 1.9: added rewinddir member function + +2008-01-18 Toni Ronkko + + * Version 1.8: Using FindFirstFileA and WIN32_FIND_DATAA to avoid + converting string between multi-byte and unicode representations. + This makes the code simpler and also allows the code to be compiled + under MingW. Thanks to Azriel Fasten for the suggestion. + +2007-03-04 Toni Ronkko + + * Bug fix: due to the strncpy_s() function this file only compiled in + Visual Studio 2005. Using the new string functions only when the + compiler version allows. + +2006-11-02 Toni Ronkko + + * Major update: removed support for Watcom C, MS-DOS and Turbo C to + simplify the file, updated the code to compile cleanly on Visual + Studio 2005 with both unicode and multi-byte character strings, + removed rewinddir() as it had a bug. + +2006-08-20 Toni Ronkko + + * Removed all remarks about MSVC 1.0, which is antiqued now. + Simplified comments by removing SGML tags. + +2002-05-14 Toni Ronkko + + * Embedded the function definitions directly to the header so that no + source modules need to be included in the Visual Studio project. + Removed all the dependencies to other projects so that this header + file can be used independently. + +1998-05-28 Toni Ronkko + + * First version. diff --git a/deps/dirent/LICENSE b/deps/dirent/LICENSE new file mode 100644 index 0000000000000000000000000000000000000000..c5740cd7ec44568ee31990d0788f1395dd9cc035 --- /dev/null +++ b/deps/dirent/LICENSE @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 1998-2019 Toni Ronkko + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/deps/dirent/README.md b/deps/dirent/README.md new file mode 100644 index 0000000000000000000000000000000000000000..c90647347b2f6031e87cc65f1c2be2436e33f43f --- /dev/null +++ b/deps/dirent/README.md @@ -0,0 +1,96 @@ +# Dirent +Dirent is a C/C++ programming interface that allows programmers to retrieve +information about files and directories under Linux/UNIX. This project +provides Linux compatible Dirent interface for Microsoft Windows. + + +# Installation + +Download the latest Dirent installation package from +[GitHub](https://github.com/tronkko/dirent/releases) and +unpack the installation file with 7-zip, for example. The installation +package contains dirent.h file as well as a few example programs and +tests. + + +## Install Dirent for All Programs + +To make dirent.h available for all C/C++ programs, simply copy the +``include/dirent.h`` file to the system include directory. System include +directory contains header files such as assert.h and windows.h. In Visual +Studio 2008, for example, the system include may be found at +``C:\Program Files\Microsoft Visual Studio 9.0\VC\include``. + +Everything you need is included in the single dirent.h file, and you can +start using Dirent immediately -- there is no need to add files to your +Visual Studio project. + + +## Embed Dirent into Your Own Project + +If you wish to distribute dirent.h alongside with your own source code, then +copy ``include/dirent.h`` file to a new sub-directory within your project and +add that directory to include path on Windows while omitting the directory +under Linux/UNIX. This allows your project to be compiled against native +dirent.h on Linux/UNIX while substituting the functionality on Microsoft +Windows. + + +## Examples + +The installation package contains four example programs: + +Program | Purpose +-------- | ----------------------------------------------------------------- +ls | List files in a directory, e.g. ls "c:\Program Files" +find | Find files in subdirectories, e.g. find "c:\Program Files\CMake" +updatedb | Build database of files in a drive, e.g. updatedb c:\ +locate | Locate a file from database, e.g. locate notepad.exe + +To build the example programs, first install [CMake](https://cmake.org/). +Then, with CMake installed, open command prompt and create a temporary +directory ``c:\temp\dirent`` for the build files as + +``` +c:\ +mkdir temp +mkdir temp\dirent +cd temp\dirent +``` + +Generate build files as + +``` +cmake d:\dirent +``` + +where ``d:\dirent`` is the root directory of the Dirent package (containing +this README.md and LICENSE file). + +Once CMake is finished, open Visual Studio, load the generated dirent.sln file +from the build directory and build the solution. Once the build completes, run +the example programs from the command prompt as + +``` +cd Debug +ls . +find . +updatedb c:\ +locate cmd.exe +``` + + +# Copying + +Dirent may be freely distributed under the MIT license. See the +[LICENSE](LICENSE) file for details. + + +# Alternatives to Dirent + +I ported Dirent to Microsoft Windows in 1998 when only a few alternatives +were available. However, the situation has changed since then and nowadays +both [Cygwin](http://www.cygwin.com) and [MingW](http://www.mingw.org) +allow you to compile a great number of UNIX programs in Microsoft Windows. +They both provide a full dirent API as well as many other UNIX APIs. MingW +can even be used for commercial applications! diff --git a/deps/dirent/distclean.cmake b/deps/dirent/distclean.cmake new file mode 100644 index 0000000000000000000000000000000000000000..e6bde39227142b509b52a3934a83c7b61c9d00cb --- /dev/null +++ b/deps/dirent/distclean.cmake @@ -0,0 +1,62 @@ +# Remove CMake generated temporary files +set (cmake_generated + ${CMAKE_BINARY_DIR}/ALL_BUILD.vcxproj + ${CMAKE_BINARY_DIR}/ALL_BUILD.vcxproj.filters + ${CMAKE_BINARY_DIR}/CMakeCache.txt + ${CMAKE_BINARY_DIR}/CMakeFiles + ${CMAKE_BINARY_DIR}/CTestTestfile.cmake + ${CMAKE_BINARY_DIR}/Continuous.vcxproj + ${CMAKE_BINARY_DIR}/Continuous.vcxproj.filters + ${CMAKE_BINARY_DIR}/DartConfiguration.tcl + ${CMAKE_BINARY_DIR}/Debug + ${CMAKE_BINARY_DIR}/Experimental.vcxproj + ${CMAKE_BINARY_DIR}/Experimental.vcxproj.filters + ${CMAKE_BINARY_DIR}/INSTALL.vcxproj + ${CMAKE_BINARY_DIR}/INSTALL.vcxproj.filters + ${CMAKE_BINARY_DIR}/Makefile + ${CMAKE_BINARY_DIR}/Nightly.vcxproj + ${CMAKE_BINARY_DIR}/Nightly.vcxproj.filters + ${CMAKE_BINARY_DIR}/NightlyMemoryCheck.vcxproj + ${CMAKE_BINARY_DIR}/NightlyMemoryCheck.vcxproj.filters + ${CMAKE_BINARY_DIR}/RUN_TESTS.vcxproj + ${CMAKE_BINARY_DIR}/RUN_TESTS.vcxproj.filters + ${CMAKE_BINARY_DIR}/Testing + ${CMAKE_BINARY_DIR}/Win32 + ${CMAKE_BINARY_DIR}/ZERO_CHECK.vcxproj + ${CMAKE_BINARY_DIR}/ZERO_CHECK.vcxproj.filters + ${CMAKE_BINARY_DIR}/check.vcxproj + ${CMAKE_BINARY_DIR}/check.vcxproj.filters + ${CMAKE_BINARY_DIR}/cmake_install.cmake + ${CMAKE_BINARY_DIR}/dirent.sln + ${CMAKE_BINARY_DIR}/distclean.vcxproj + ${CMAKE_BINARY_DIR}/distclean.vcxproj.filters + ${CMAKE_BINARY_DIR}/find + ${CMAKE_BINARY_DIR}/find.dir + ${CMAKE_BINARY_DIR}/find.vcxproj + ${CMAKE_BINARY_DIR}/find.vcxproj.filters + ${CMAKE_BINARY_DIR}/locate + ${CMAKE_BINARY_DIR}/locate.dir + ${CMAKE_BINARY_DIR}/locate.vcxproj + ${CMAKE_BINARY_DIR}/locate.vcxproj.filters + ${CMAKE_BINARY_DIR}/ls + ${CMAKE_BINARY_DIR}/ls.dir + ${CMAKE_BINARY_DIR}/ls.vcxproj + ${CMAKE_BINARY_DIR}/ls.vcxproj.filters + ${CMAKE_BINARY_DIR}/t-compile + ${CMAKE_BINARY_DIR}/t-compile.dir + ${CMAKE_BINARY_DIR}/t-compile.vcxproj + ${CMAKE_BINARY_DIR}/t-compile.vcxproj.filters + ${CMAKE_BINARY_DIR}/t-dirent + ${CMAKE_BINARY_DIR}/t-dirent.dir + ${CMAKE_BINARY_DIR}/t-dirent.vcxproj + ${CMAKE_BINARY_DIR}/t-dirent.vcxproj.filters + ${CMAKE_BINARY_DIR}/updatedb + ${CMAKE_BINARY_DIR}/updatedb.dir + ${CMAKE_BINARY_DIR}/updatedb.vcxproj + ${CMAKE_BINARY_DIR}/updatedb.vcxproj.filters +) +foreach (file ${cmake_generated}) + if (EXISTS ${file}) + file (REMOVE_RECURSE ${file}) + endif() +endforeach (file) diff --git a/deps/dirent/examples/cat.c b/deps/dirent/examples/cat.c new file mode 100644 index 0000000000000000000000000000000000000000..7067bfcbdece1cac4c3d9915d9be4a50acc8e60c --- /dev/null +++ b/deps/dirent/examples/cat.c @@ -0,0 +1,85 @@ +/* + * Output contents of a file. + * + * Compile this file with Visual Studio and run the produced command in + * console with a file name argument. For example, command + * + * cat include\dirent.h + * + * will output the dirent.h to screen. + * + * Copyright (C) 1998-2019 Toni Ronkko + * This file is part of dirent. Dirent may be freely distributed + * under the MIT license. For all details and documentation, see + * https://github.com/tronkko/dirent + */ +#define _CRT_SECURE_NO_WARNINGS +#include +#include +#include +#include +#include +#include + +static void output_file (const char *fn); + + +int +main( + int argc, char *argv[]) +{ + int i; + + /* Select default locale */ + setlocale (LC_ALL, ""); + + /* Require at least one file */ + if (argc == 1) { + fprintf (stderr, "Usage: cat filename\n"); + return EXIT_FAILURE; + } + + /* For each file name argument in command line */ + i = 1; + while (i < argc) { + output_file (argv[i]); + i++; + } + return EXIT_SUCCESS; +} + +/* + * Output file to screen + */ +static void +output_file( + const char *fn) +{ + FILE *fp; + + /* Open file */ + fp = fopen (fn, "r"); + if (fp != NULL) { + size_t n; + char buffer[4096]; + + /* Output file to screen */ + do { + + /* Read some bytes from file */ + n = fread (buffer, 1, 4096, fp); + + /* Output bytes to screen */ + fwrite (buffer, 1, n, stdout); + + } while (n != 0); + + /* Close file */ + fclose (fp); + + } else { + /* Could not open directory */ + fprintf (stderr, "Cannot open %s (%s)\n", fn, strerror (errno)); + exit (EXIT_FAILURE); + } +} diff --git a/deps/dirent/examples/find.c b/deps/dirent/examples/find.c new file mode 100644 index 0000000000000000000000000000000000000000..8073d050180aa1d994a0fca49db2defd5a7f2ba1 --- /dev/null +++ b/deps/dirent/examples/find.c @@ -0,0 +1,147 @@ +/* + * An example demonstrating recursive directory traversal. + * + * Compile this file with Visual Studio and run the produced command in + * console with a directory name argument. For example, command + * + * find "C:\Program Files" + * + * will output thousands of file names such as + * + * c:\Program Files/7-Zip/7-zip.chm + * c:\Program Files/7-Zip/7-zip.dll + * c:\Program Files/7-Zip/7z.dll + * c:\Program Files/Adobe/Reader 10.0/Reader/logsession.dll + * c:\Program Files/Adobe/Reader 10.0/Reader/LogTransport2.exe + * c:\Program Files/Windows NT/Accessories/wordpad.exe + * c:\Program Files/Windows NT/Accessories/write.wpc + * + * The find command provided by this file is only an example: the command does + * not provide options to restrict the output to certain files as the Linux + * version does. + * + * Copyright (C) 1998-2019 Toni Ronkko + * This file is part of dirent. Dirent may be freely distributed + * under the MIT license. For all details and documentation, see + * https://github.com/tronkko/dirent + */ +#define _CRT_SECURE_NO_WARNINGS +#include +#include +#include +#include +#include +#include + +static int find_directory (const char *dirname); + + +int +main( + int argc, char *argv[]) +{ + int i; + int ok; + + /* Select default locale */ + setlocale (LC_ALL, ""); + + /* For each directory in command line */ + i = 1; + while (i < argc) { + ok = find_directory (argv[i]); + if (!ok) { + exit (EXIT_FAILURE); + } + i++; + } + + /* List current working directory if no arguments on command line */ + if (argc == 1) { + find_directory ("."); + } + return EXIT_SUCCESS; +} + +/* Find files and subdirectories recursively */ +static int +find_directory( + const char *dirname) +{ + DIR *dir; + char buffer[PATH_MAX + 2]; + char *p = buffer; + const char *src; + char *end = &buffer[PATH_MAX]; + int ok; + + /* Copy directory name to buffer */ + src = dirname; + while (p < end && *src != '\0') { + *p++ = *src++; + } + *p = '\0'; + + /* Open directory stream */ + dir = opendir (dirname); + if (dir != NULL) { + struct dirent *ent; + + /* Print all files and directories within the directory */ + while ((ent = readdir (dir)) != NULL) { + char *q = p; + char c; + + /* Get final character of directory name */ + if (buffer < q) { + c = q[-1]; + } else { + c = ':'; + } + + /* Append directory separator if not already there */ + if (c != ':' && c != '/' && c != '\\') { + *q++ = '/'; + } + + /* Append file name */ + src = ent->d_name; + while (q < end && *src != '\0') { + *q++ = *src++; + } + *q = '\0'; + + /* Decide what to do with the directory entry */ + switch (ent->d_type) { + case DT_LNK: + case DT_REG: + /* Output file name with directory */ + printf ("%s\n", buffer); + break; + + case DT_DIR: + /* Scan sub-directory recursively */ + if (strcmp (ent->d_name, ".") != 0 + && strcmp (ent->d_name, "..") != 0) { + find_directory (buffer); + } + break; + + default: + /* Ignore device entries */ + /*NOP*/; + } + + } + + closedir (dir); + ok = 1; + + } else { + /* Could not open directory */ + fprintf (stderr, "Cannot open %s (%s)\n", dirname, strerror (errno)); + ok = 0; + } + + return ok; +} diff --git a/deps/dirent/examples/locate.c b/deps/dirent/examples/locate.c new file mode 100644 index 0000000000000000000000000000000000000000..c5cf0f471f3c72e9718380b98c028008203082bb --- /dev/null +++ b/deps/dirent/examples/locate.c @@ -0,0 +1,280 @@ +/* + * A file look-up utility to complement updatedb + * + * Compile and run updatedb command first to create locate.db file. Then, + * compile this program with Visual Studio and run the program in console with + * a file name argument. For example, the command + * + * locate autoexec + * + * might output something like + * + * c:/AUTOEXEC.BAT + * c:/WINDOWS/repair/autoexec.nt + * c:/WINDOWS/system32/AUTOEXEC.NT + * + * Be ware that this file uses wide-character API which is not compatible + * with Linux or other major Unixes. + * + * Copyright (C) 1998-2019 Toni Ronkko + * This file is part of dirent. Dirent may be freely distributed + * under the MIT license. For all details and documentation, see + * https://github.com/tronkko/dirent + */ +#include +#include +#include +#include +#ifdef WIN32 +# include +# include +#endif +#include + +/* File name and location of database file */ +#define DB_LOCATION L"locate.db" + + +/* Forward-decl */ +static int db_locate (const wchar_t *pattern); +static int db_match (const wchar_t *fn, const wchar_t *pattern); +static void db_open (void); +static void db_close (void); +static int db_read (wchar_t *buffer, size_t max); + + +/* Module local variables */ +static FILE *db = NULL; + + +int +main( + int argc, char *argv[]) +{ +#ifdef WIN32 + int i; + + /* Prepare for unicode output */ + _setmode (_fileno (stdout), _O_U16TEXT); + + /* For each pattern in command line */ + i = 1; + while (i < argc) { + wchar_t buffer[PATH_MAX + 1]; + errno_t error; + size_t n; + int count = 0; + + /* Convert ith argument to wide-character string */ + error = mbstowcs_s (&n, buffer, PATH_MAX, argv[i], _TRUNCATE); + if (!error) { + /* Find files matching pattern */ + count = db_locate (buffer); + + /* Output warning if string is not found */ + if (count == 0) { + wprintf (L"%s not found\n", buffer); + } + } + + + i++; + } + + if (argc < 2) { + wprintf (L"Usage: locate pattern\n"); + exit (EXIT_FAILURE); + } +#else + printf ("locate only works on Microsoft Windows\n"); +#endif + + return EXIT_SUCCESS; +} + +/* Match pattern against files in locate.db file */ +static int +db_locate( + const wchar_t *pattern) +{ + int count = 0; + +#ifdef WIN32 + wchar_t buffer[PATH_MAX + 1]; + + /* Open locate.db for read */ + db_open (); + + /* Read one directory and file name at a time from database file */ + while (db_read (buffer, PATH_MAX)) { + /* See if file name in buffer matches the search pattern */ + if (db_match (buffer, pattern)) { + /* Match found => output file name and path */ + wprintf (L"%s\n", buffer); + count++; + } + + } + + db_close (); +#endif + + return count; +} + +/* Match pattern against file name */ +static int +db_match( + const wchar_t *fn, const wchar_t *pattern) +{ + int found = 0; + +#ifdef WIN32 + wchar_t *p; + wchar_t base[PATH_MAX + 1]; + wchar_t patt[PATH_MAX + 1]; + int i; + int done = 0; + + /* Locate zero-terminator from fn */ + p = wcschr (fn, '\0'); + + /* Find base name from buffer */ + while (fn < p && !done) { + switch (p[-1]) { + case ':': + case '/': + case '\\': + /* Final path separator found */ + done = 1; + break; + + default: + /* No path separator yet */ + p--; + } + } + + /* Convert base name to lower case */ + i = 0; + while (i < PATH_MAX && p[i] != '\0') { + base[i] = towlower (p[i]); + i++; + } + base[i] = '\0'; + + /* Convert search pattern to lower case */ + i = 0; + while (i < PATH_MAX && pattern[i] != '\0') { + patt[i] = towlower (pattern[i]); + i++; + } + patt[i] = '\0'; + + /* See if file name matches pattern */ + if (wcsstr (base, patt) != NULL) { + found = 1; + } else { + found = 0; + } +#endif + + return found; +} + +/* + * Read line from locate.db. This function is same as fgetws() except + * that new-line at the end of line is not included. + */ +static int +db_read( + wchar_t *buffer, size_t max) +{ + int ok = 0; + +#ifdef WIN32 + size_t i = 0; + wchar_t c; + int done = 0; + + do { + /* Read wide-character from stream */ + if (db) { + c = fgetwc (db); + } else { + wprintf (L"Database not open\n"); + exit (EXIT_SUCCESS); + } + + /* Determine how to process character */ + switch (c) { + case '\r': + /* Ignore, should be handled by run-time libraries */ + /*NOP*/; + break; + + case '\n': + /* End of string => return file name and true */ + done = 1; + ok = 1; + break; + + case /*EOF*/WEOF: + /* End of file */ + done = 1; + if (i == 0) { + /* No data in buffer => return false to indicate EOF */ + ok = 0; + } else { + /* Data in buffer => return true */ + ok = 1; + } + break; + + default: + /* Store character */ + if (i < max - 1) { + buffer[i++] = c; + } else { + wprintf (L"Buffer too small"); + exit (EXIT_FAILURE); + } + } + } while (!done); + + /* Zero-terminate buffer */ + buffer[i] = '\0'; +#endif + + return ok; +} + +/* Open database file locate.db */ +static void +db_open( + void) +{ +#ifdef WIN32 + if (db == NULL) { + errno_t error; + + /* Open file for writing */ + error = _wfopen_s (&db, DB_LOCATION, L"rt, ccs=UNICODE"); + if (error) { + wprintf (L"Cannot open %s\n", DB_LOCATION); + exit (EXIT_FAILURE); + } + } +#endif +} + +/* Close database file */ +static void +db_close( + void) +{ + if (db) { + fclose (db); + db = NULL; + } +} diff --git a/deps/dirent/examples/ls.c b/deps/dirent/examples/ls.c new file mode 100644 index 0000000000000000000000000000000000000000..d0dcee12f0ae4647588fe2156cfb036e63f05bbf --- /dev/null +++ b/deps/dirent/examples/ls.c @@ -0,0 +1,103 @@ +/* + * An example demonstrating basic directory listing. + * + * Compile this file with Visual Studio and run the produced command in + * console with a directory name argument. For example, command + * + * ls "c:\Program Files" + * + * might output something like + * + * ./ + * ../ + * 7-Zip/ + * Internet Explorer/ + * Microsoft Visual Studio 9.0/ + * Microsoft.NET/ + * Mozilla Firefox/ + * + * The ls command provided by this file is only an example: the command does + * not have any fancy options like "ls -al" in Linux and the command does not + * support file name matching like "ls *.c". + * + * Copyright (C) 1998-2019 Toni Ronkko + * This file is part of dirent. Dirent may be freely distributed + * under the MIT license. For all details and documentation, see + * https://github.com/tronkko/dirent + */ +#define _CRT_SECURE_NO_WARNINGS +#include +#include +#include +#include +#include +#include + +static void list_directory (const char *dirname); + + +int +main( + int argc, char *argv[]) +{ + int i; + + /* Select default locale */ + setlocale (LC_ALL, ""); + + /* For each directory in command line */ + i = 1; + while (i < argc) { + list_directory (argv[i]); + i++; + } + + /* List current working directory if no arguments on command line */ + if (argc == 1) { + list_directory ("."); + } + return EXIT_SUCCESS; +} + +/* + * List files and directories within a directory. + */ +static void +list_directory( + const char *dirname) +{ + DIR *dir; + struct dirent *ent; + + /* Open directory stream */ + dir = opendir (dirname); + if (dir != NULL) { + + /* Print all files and directories within the directory */ + while ((ent = readdir (dir)) != NULL) { + switch (ent->d_type) { + case DT_REG: + printf ("%s\n", ent->d_name); + break; + + case DT_DIR: + printf ("%s/\n", ent->d_name); + break; + + case DT_LNK: + printf ("%s@\n", ent->d_name); + break; + + default: + printf ("%s*\n", ent->d_name); + } + } + + closedir (dir); + + } else { + /* Could not open directory */ + fprintf (stderr, "Cannot open %s (%s)\n", dirname, strerror (errno)); + exit (EXIT_FAILURE); + } +} diff --git a/deps/dirent/examples/scandir.c b/deps/dirent/examples/scandir.c new file mode 100644 index 0000000000000000000000000000000000000000..5c783e95270ba7260d53dde361e198ff7e5f8e29 --- /dev/null +++ b/deps/dirent/examples/scandir.c @@ -0,0 +1,110 @@ +/* + * Example program demonstrating the use of scandir function. + * + * Compile this file with Visual Studio and run the produced command in + * console with a directory name argument. For example, command + * + * scandir "c:\Program Files" + * + * might output something like + * + * ./ + * ../ + * 7-Zip/ + * Internet Explorer/ + * Microsoft Visual Studio 9.0/ + * Microsoft.NET/ + * Mozilla Firefox/ + * + * Copyright (C) 1998-2019 Toni Ronkko + * This file is part of dirent. Dirent may be freely distributed + * under the MIT license. For all details and documentation, see + * https://github.com/tronkko/dirent + */ +#define _CRT_SECURE_NO_WARNINGS +#include +#include +#include +#include +#include +#include + +static void list_directory (const char *dirname); + + +int +main( + int argc, char *argv[]) +{ + int i; + + /* Select default locale */ + setlocale (LC_ALL, ""); + + /* For each directory in command line */ + i = 1; + while (i < argc) { + list_directory (argv[i]); + i++; + } + + /* List current working directory if no arguments on command line */ + if (argc == 1) { + list_directory ("."); + } + return EXIT_SUCCESS; +} + +/* + * List files and directories within a directory. + */ +static void +list_directory( + const char *dirname) +{ + struct dirent **files; + int i; + int n; + + /* Scan files in directory */ + n = scandir (dirname, &files, NULL, alphasort); + if (n >= 0) { + + /* Loop through file names */ + for (i = 0; i < n; i++) { + struct dirent *ent; + + /* Get pointer to file entry */ + ent = files[i]; + + /* Output file name */ + switch (ent->d_type) { + case DT_REG: + printf ("%s\n", ent->d_name); + break; + + case DT_DIR: + printf ("%s/\n", ent->d_name); + break; + + case DT_LNK: + printf ("%s@\n", ent->d_name); + break; + + default: + printf ("%s*\n", ent->d_name); + } + + } + + /* Release file names */ + for (i = 0; i < n; i++) { + free (files[i]); + } + free (files); + + } else { + fprintf (stderr, "Cannot open %s (%s)\n", dirname, strerror (errno)); + exit (EXIT_FAILURE); + } +} diff --git a/deps/dirent/examples/updatedb.c b/deps/dirent/examples/updatedb.c new file mode 100644 index 0000000000000000000000000000000000000000..1642178b9c1f9c1193e340a417746aef503a5f83 --- /dev/null +++ b/deps/dirent/examples/updatedb.c @@ -0,0 +1,232 @@ +/* + * An example demonstrating wide-character functions + * + * Compile this file with Visual Studio and run the produced command in + * console with a directory name argument. For example, command + * + * updatedb C:\ + * + * will produce the file locate.db with one file name per line such as + * + * c:\Program Files/7-Zip/7-zip.chm + * c:\Program Files/7-Zip/7-zip.dll + * c:\Program Files/7-Zip/7z.dll + * c:\Program Files/Adobe/Reader 10.0/Reader/logsession.dll + * c:\Program Files/Adobe/Reader 10.0/Reader/LogTransport2.exe + * c:\Program Files/Windows NT/Accessories/wordpad.exe + * c:\Program Files/Windows NT/Accessories/write.wpc + * + * Be ware that this file uses wide-character API which is not compatible + * with Linux or other major Unixes. + * + * Copyright (C) 1998-2019 Toni Ronkko + * This file is part of dirent. Dirent may be freely distributed + * under the MIT license. For all details and documentation, see + * https://github.com/tronkko/dirent + */ +#include +#include +#include +#include +#ifdef WIN32 +# include +# include +#endif +#include + +/* File name and location of database file */ +#define DB_LOCATION L"locate.db" + + +/* Forward-decl */ +static int update_directory (const wchar_t *dirname); +static void db_open (void); +static void db_close (void); +static void db_store (const wchar_t *dirname); + + +/* Module local variables */ +static FILE *db = NULL; + + +int +main( + int argc, char *argv[]) +{ +#ifdef WIN32 + int i; + int ok; + + /* Prepare for unicode output */ + _setmode (_fileno (stdout), _O_U16TEXT); + + /* Open locate.db */ + db_open (); + + /* For each directory in command line */ + i = 1; + while (i < argc) { + wchar_t buffer[PATH_MAX + 1]; + errno_t error; + size_t n; + + /* Convert ith argument to wide-character string */ + error = mbstowcs_s (&n, buffer, PATH_MAX, argv[i], _TRUNCATE); + if (!error) { + + /* Scan directory for files */ + ok = update_directory (buffer); + if (!ok) { + wprintf (L"Cannot open directory %s\n", buffer); + exit (EXIT_FAILURE); + } + + } + + i++; + } + + /* Use current working directory if no arguments on command line */ + if (argc == 1) { + update_directory (L"."); + } + + db_close (); +#else + printf ("updatedb only works on Microsoft Windows\n"); +#endif + + return EXIT_SUCCESS; +} + +/* Find files recursively */ +static int +update_directory( + const wchar_t *dirname) +{ + int ok = 0; + +#ifdef WIN32 + _WDIR *dir; + wchar_t buffer[PATH_MAX + 2]; + wchar_t *p = buffer; + const wchar_t *src; + wchar_t *end = &buffer[PATH_MAX]; + + /* Copy directory name to buffer */ + src = dirname; + while (p < end && *src != '\0') { + *p++ = *src++; + } + *p = '\0'; + + /* Open directory stream */ + dir = _wopendir (dirname); + if (dir != NULL) { + struct _wdirent *ent; + + /* Print all files and directories within the directory */ + while ((ent = _wreaddir (dir)) != NULL) { + wchar_t *q = p; + wchar_t c; + + /* Get final character of directory name */ + if (buffer < q) { + c = q[-1]; + } else { + c = ':'; + } + + /* Append directory separator if not already there */ + if (c != ':' && c != '/' && c != '\\') { + *q++ = '/'; + } + + /* Append file name */ + src = ent->d_name; + while (q < end && *src != '\0') { + *q++ = *src++; + } + *q = '\0'; + + /* Decide what to do with the directory entry */ + switch (ent->d_type) { + case DT_REG: + /* Store file name */ + db_store (buffer); + break; + + case DT_DIR: + /* Scan sub-directory recursively */ + if (wcscmp (ent->d_name, L".") != 0 + && wcscmp (ent->d_name, L"..") != 0) { + update_directory (buffer); + } + break; + + default: + /* Do not device entries */ + /*NOP*/; + } + + } + + wclosedir (dir); + ok = 1; + + } else { + + /* Cannot open directory */ + ok = 0; + + } +#endif + + return ok; +} + +/* Store file name to locate.db */ +static void +db_store( + const wchar_t *dirname) +{ +#ifdef WIN32 + if (db) { + /* Output line to file */ + fwprintf (db, L"%s\n", dirname); + } else { + wprintf (L"Database not open\n"); + exit (EXIT_FAILURE); + } +#endif +} + +/* Open database file locate.db */ +static void +db_open( + void) +{ +#ifdef WIN32 + if (db == NULL) { + errno_t error; + + /* Open file for writing */ + error = _wfopen_s (&db, DB_LOCATION, L"wt, ccs=UNICODE"); + if (error) { + wprintf (L"Cannot open %s\n", DB_LOCATION); + exit (EXIT_FAILURE); + } + } +#endif +} + +/* Close database file */ +static void +db_close( + void) +{ + if (db) { + fclose (db); + db = NULL; + } +} diff --git a/deps/dirent/include/dirent.h b/deps/dirent/include/dirent.h new file mode 100644 index 0000000000000000000000000000000000000000..c885b398a715cedaf253552936eefefc860c327d --- /dev/null +++ b/deps/dirent/include/dirent.h @@ -0,0 +1,1160 @@ +/* + * Dirent interface for Microsoft Visual Studio + * + * Copyright (C) 1998-2019 Toni Ronkko + * This file is part of dirent. Dirent may be freely distributed + * under the MIT license. For all details and documentation, see + * https://github.com/tronkko/dirent + */ +#ifndef DIRENT_H +#define DIRENT_H + +/* Hide warnings about unreferenced local functions */ +#if defined(__clang__) +# pragma clang diagnostic ignored "-Wunused-function" +#elif defined(_MSC_VER) +# pragma warning(disable:4505) +#elif defined(__GNUC__) +# pragma GCC diagnostic ignored "-Wunused-function" +#endif + +/* + * Include windows.h without Windows Sockets 1.1 to prevent conflicts with + * Windows Sockets 2.0. + */ +#ifndef WIN32_LEAN_AND_MEAN +# define WIN32_LEAN_AND_MEAN +#endif +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +/* Indicates that d_type field is available in dirent structure */ +#define _DIRENT_HAVE_D_TYPE + +/* Indicates that d_namlen field is available in dirent structure */ +#define _DIRENT_HAVE_D_NAMLEN + +/* Entries missing from MSVC 6.0 */ +#if !defined(FILE_ATTRIBUTE_DEVICE) +# define FILE_ATTRIBUTE_DEVICE 0x40 +#endif + +/* File type and permission flags for stat(), general mask */ +#if !defined(S_IFMT) +# define S_IFMT _S_IFMT +#endif + +/* Directory bit */ +#if !defined(S_IFDIR) +# define S_IFDIR _S_IFDIR +#endif + +/* Character device bit */ +#if !defined(S_IFCHR) +# define S_IFCHR _S_IFCHR +#endif + +/* Pipe bit */ +#if !defined(S_IFFIFO) +# define S_IFFIFO _S_IFFIFO +#endif + +/* Regular file bit */ +#if !defined(S_IFREG) +# define S_IFREG _S_IFREG +#endif + +/* Read permission */ +#if !defined(S_IREAD) +# define S_IREAD _S_IREAD +#endif + +/* Write permission */ +#if !defined(S_IWRITE) +# define S_IWRITE _S_IWRITE +#endif + +/* Execute permission */ +#if !defined(S_IEXEC) +# define S_IEXEC _S_IEXEC +#endif + +/* Pipe */ +#if !defined(S_IFIFO) +# define S_IFIFO _S_IFIFO +#endif + +/* Block device */ +#if !defined(S_IFBLK) +# define S_IFBLK 0 +#endif + +/* Link */ +#if !defined(S_IFLNK) +# define S_IFLNK 0 +#endif + +/* Socket */ +#if !defined(S_IFSOCK) +# define S_IFSOCK 0 +#endif + +/* Read user permission */ +#if !defined(S_IRUSR) +# define S_IRUSR S_IREAD +#endif + +/* Write user permission */ +#if !defined(S_IWUSR) +# define S_IWUSR S_IWRITE +#endif + +/* Execute user permission */ +#if !defined(S_IXUSR) +# define S_IXUSR 0 +#endif + +/* Read group permission */ +#if !defined(S_IRGRP) +# define S_IRGRP 0 +#endif + +/* Write group permission */ +#if !defined(S_IWGRP) +# define S_IWGRP 0 +#endif + +/* Execute group permission */ +#if !defined(S_IXGRP) +# define S_IXGRP 0 +#endif + +/* Read others permission */ +#if !defined(S_IROTH) +# define S_IROTH 0 +#endif + +/* Write others permission */ +#if !defined(S_IWOTH) +# define S_IWOTH 0 +#endif + +/* Execute others permission */ +#if !defined(S_IXOTH) +# define S_IXOTH 0 +#endif + +/* Maximum length of file name */ +#if !defined(PATH_MAX) +# define PATH_MAX MAX_PATH +#endif +#if !defined(FILENAME_MAX) +# define FILENAME_MAX MAX_PATH +#endif +#if !defined(NAME_MAX) +# define NAME_MAX FILENAME_MAX +#endif + +/* File type flags for d_type */ +#define DT_UNKNOWN 0 +#define DT_REG S_IFREG +#define DT_DIR S_IFDIR +#define DT_FIFO S_IFIFO +#define DT_SOCK S_IFSOCK +#define DT_CHR S_IFCHR +#define DT_BLK S_IFBLK +#define DT_LNK S_IFLNK + +/* Macros for converting between st_mode and d_type */ +#define IFTODT(mode) ((mode) & S_IFMT) +#define DTTOIF(type) (type) + +/* + * File type macros. Note that block devices, sockets and links cannot be + * distinguished on Windows and the macros S_ISBLK, S_ISSOCK and S_ISLNK are + * only defined for compatibility. These macros should always return false + * on Windows. + */ +#if !defined(S_ISFIFO) +# define S_ISFIFO(mode) (((mode) & S_IFMT) == S_IFIFO) +#endif +#if !defined(S_ISDIR) +# define S_ISDIR(mode) (((mode) & S_IFMT) == S_IFDIR) +#endif +#if !defined(S_ISREG) +# define S_ISREG(mode) (((mode) & S_IFMT) == S_IFREG) +#endif +#if !defined(S_ISLNK) +# define S_ISLNK(mode) (((mode) & S_IFMT) == S_IFLNK) +#endif +#if !defined(S_ISSOCK) +# define S_ISSOCK(mode) (((mode) & S_IFMT) == S_IFSOCK) +#endif +#if !defined(S_ISCHR) +# define S_ISCHR(mode) (((mode) & S_IFMT) == S_IFCHR) +#endif +#if !defined(S_ISBLK) +# define S_ISBLK(mode) (((mode) & S_IFMT) == S_IFBLK) +#endif + +/* Return the exact length of the file name without zero terminator */ +#define _D_EXACT_NAMLEN(p) ((p)->d_namlen) + +/* Return the maximum size of a file name */ +#define _D_ALLOC_NAMLEN(p) ((PATH_MAX)+1) + + +#ifdef __cplusplus +extern "C" { +#endif + + +/* Wide-character version */ +struct _wdirent { + /* Always zero */ + long d_ino; + + /* File position within stream */ + long d_off; + + /* Structure size */ + unsigned short d_reclen; + + /* Length of name without \0 */ + size_t d_namlen; + + /* File type */ + int d_type; + + /* File name */ + wchar_t d_name[PATH_MAX+1]; +}; +typedef struct _wdirent _wdirent; + +struct _WDIR { + /* Current directory entry */ + struct _wdirent ent; + + /* Private file data */ + WIN32_FIND_DATAW data; + + /* True if data is valid */ + int cached; + + /* Win32 search handle */ + HANDLE handle; + + /* Initial directory name */ + wchar_t *patt; +}; +typedef struct _WDIR _WDIR; + +/* Multi-byte character version */ +struct dirent { + /* Always zero */ + long d_ino; + + /* File position within stream */ + long d_off; + + /* Structure size */ + unsigned short d_reclen; + + /* Length of name without \0 */ + size_t d_namlen; + + /* File type */ + int d_type; + + /* File name */ + char d_name[PATH_MAX+1]; +}; +typedef struct dirent dirent; + +struct DIR { + struct dirent ent; + struct _WDIR *wdirp; +}; +typedef struct DIR DIR; + + +/* Dirent functions */ +static DIR *opendir (const char *dirname); +static _WDIR *_wopendir (const wchar_t *dirname); + +static struct dirent *readdir (DIR *dirp); +static struct _wdirent *_wreaddir (_WDIR *dirp); + +static int readdir_r( + DIR *dirp, struct dirent *entry, struct dirent **result); +static int _wreaddir_r( + _WDIR *dirp, struct _wdirent *entry, struct _wdirent **result); + +static int closedir (DIR *dirp); +static int _wclosedir (_WDIR *dirp); + +static void rewinddir (DIR* dirp); +static void _wrewinddir (_WDIR* dirp); + +static int scandir (const char *dirname, struct dirent ***namelist, + int (*filter)(const struct dirent*), + int (*compare)(const struct dirent**, const struct dirent**)); + +static int alphasort (const struct dirent **a, const struct dirent **b); + +static int versionsort (const struct dirent **a, const struct dirent **b); + + +/* For compatibility with Symbian */ +#define wdirent _wdirent +#define WDIR _WDIR +#define wopendir _wopendir +#define wreaddir _wreaddir +#define wclosedir _wclosedir +#define wrewinddir _wrewinddir + + +/* Internal utility functions */ +static WIN32_FIND_DATAW *dirent_first (_WDIR *dirp); +static WIN32_FIND_DATAW *dirent_next (_WDIR *dirp); + +static int dirent_mbstowcs_s( + size_t *pReturnValue, + wchar_t *wcstr, + size_t sizeInWords, + const char *mbstr, + size_t count); + +static int dirent_wcstombs_s( + size_t *pReturnValue, + char *mbstr, + size_t sizeInBytes, + const wchar_t *wcstr, + size_t count); + +static void dirent_set_errno (int error); + + +/* + * Open directory stream DIRNAME for read and return a pointer to the + * internal working area that is used to retrieve individual directory + * entries. + */ +static _WDIR* +_wopendir( + const wchar_t *dirname) +{ + _WDIR *dirp; + DWORD n; + wchar_t *p; + + /* Must have directory name */ + if (dirname == NULL || dirname[0] == '\0') { + dirent_set_errno (ENOENT); + return NULL; + } + + /* Allocate new _WDIR structure */ + dirp = (_WDIR*) malloc (sizeof (struct _WDIR)); + if (!dirp) { + return NULL; + } + + /* Reset _WDIR structure */ + dirp->handle = INVALID_HANDLE_VALUE; + dirp->patt = NULL; + dirp->cached = 0; + + /* + * Compute the length of full path plus zero terminator + * + * Note that on WinRT there's no way to convert relative paths + * into absolute paths, so just assume it is an absolute path. + */ +#if WINAPI_FAMILY_PARTITION(WINAPI_PARTITION_DESKTOP) + /* Desktop */ + n = GetFullPathNameW (dirname, 0, NULL, NULL); +#else + /* WinRT */ + n = wcslen (dirname); +#endif + + /* Allocate room for absolute directory name and search pattern */ + dirp->patt = (wchar_t*) malloc (sizeof (wchar_t) * n + 16); + if (dirp->patt == NULL) { + goto exit_closedir; + } + + /* + * Convert relative directory name to an absolute one. This + * allows rewinddir() to function correctly even when current + * working directory is changed between opendir() and rewinddir(). + * + * Note that on WinRT there's no way to convert relative paths + * into absolute paths, so just assume it is an absolute path. + */ +#if WINAPI_FAMILY_PARTITION(WINAPI_PARTITION_DESKTOP) + /* Desktop */ + n = GetFullPathNameW (dirname, n, dirp->patt, NULL); + if (n <= 0) { + goto exit_closedir; + } +#else + /* WinRT */ + wcsncpy_s (dirp->patt, n+1, dirname, n); +#endif + + /* Append search pattern \* to the directory name */ + p = dirp->patt + n; + switch (p[-1]) { + case '\\': + case '/': + case ':': + /* Directory ends in path separator, e.g. c:\temp\ */ + /*NOP*/; + break; + + default: + /* Directory name doesn't end in path separator */ + *p++ = '\\'; + } + *p++ = '*'; + *p = '\0'; + + /* Open directory stream and retrieve the first entry */ + if (!dirent_first (dirp)) { + goto exit_closedir; + } + + /* Success */ + return dirp; + + /* Failure */ +exit_closedir: + _wclosedir (dirp); + return NULL; +} + +/* + * Read next directory entry. + * + * Returns pointer to static directory entry which may be overwritten by + * subsequent calls to _wreaddir(). + */ +static struct _wdirent* +_wreaddir( + _WDIR *dirp) +{ + struct _wdirent *entry; + + /* + * Read directory entry to buffer. We can safely ignore the return value + * as entry will be set to NULL in case of error. + */ + (void) _wreaddir_r (dirp, &dirp->ent, &entry); + + /* Return pointer to statically allocated directory entry */ + return entry; +} + +/* + * Read next directory entry. + * + * Returns zero on success. If end of directory stream is reached, then sets + * result to NULL and returns zero. + */ +static int +_wreaddir_r( + _WDIR *dirp, + struct _wdirent *entry, + struct _wdirent **result) +{ + WIN32_FIND_DATAW *datap; + + /* Read next directory entry */ + datap = dirent_next (dirp); + if (datap) { + size_t n; + DWORD attr; + + /* + * Copy file name as wide-character string. If the file name is too + * long to fit in to the destination buffer, then truncate file name + * to PATH_MAX characters and zero-terminate the buffer. + */ + n = 0; + while (n < PATH_MAX && datap->cFileName[n] != 0) { + entry->d_name[n] = datap->cFileName[n]; + n++; + } + entry->d_name[n] = 0; + + /* Length of file name excluding zero terminator */ + entry->d_namlen = n; + + /* File type */ + attr = datap->dwFileAttributes; + if ((attr & FILE_ATTRIBUTE_DEVICE) != 0) { + entry->d_type = DT_CHR; + } else if ((attr & FILE_ATTRIBUTE_DIRECTORY) != 0) { + entry->d_type = DT_DIR; + } else { + entry->d_type = DT_REG; + } + + /* Reset dummy fields */ + entry->d_ino = 0; + entry->d_off = 0; + entry->d_reclen = sizeof (struct _wdirent); + + /* Set result address */ + *result = entry; + + } else { + + /* Return NULL to indicate end of directory */ + *result = NULL; + + } + + return /*OK*/0; +} + +/* + * Close directory stream opened by opendir() function. This invalidates the + * DIR structure as well as any directory entry read previously by + * _wreaddir(). + */ +static int +_wclosedir( + _WDIR *dirp) +{ + int ok; + if (dirp) { + + /* Release search handle */ + if (dirp->handle != INVALID_HANDLE_VALUE) { + FindClose (dirp->handle); + } + + /* Release search pattern */ + free (dirp->patt); + + /* Release directory structure */ + free (dirp); + ok = /*success*/0; + + } else { + + /* Invalid directory stream */ + dirent_set_errno (EBADF); + ok = /*failure*/-1; + + } + return ok; +} + +/* + * Rewind directory stream such that _wreaddir() returns the very first + * file name again. + */ +static void +_wrewinddir( + _WDIR* dirp) +{ + if (dirp) { + /* Release existing search handle */ + if (dirp->handle != INVALID_HANDLE_VALUE) { + FindClose (dirp->handle); + } + + /* Open new search handle */ + dirent_first (dirp); + } +} + +/* Get first directory entry (internal) */ +static WIN32_FIND_DATAW* +dirent_first( + _WDIR *dirp) +{ + WIN32_FIND_DATAW *datap; + DWORD error; + + /* Open directory and retrieve the first entry */ + dirp->handle = FindFirstFileExW( + dirp->patt, FindExInfoStandard, &dirp->data, + FindExSearchNameMatch, NULL, 0); + if (dirp->handle != INVALID_HANDLE_VALUE) { + + /* a directory entry is now waiting in memory */ + datap = &dirp->data; + dirp->cached = 1; + + } else { + + /* Failed to open directory: no directory entry in memory */ + dirp->cached = 0; + datap = NULL; + + /* Set error code */ + error = GetLastError (); + switch (error) { + case ERROR_ACCESS_DENIED: + /* No read access to directory */ + dirent_set_errno (EACCES); + break; + + case ERROR_DIRECTORY: + /* Directory name is invalid */ + dirent_set_errno (ENOTDIR); + break; + + case ERROR_PATH_NOT_FOUND: + default: + /* Cannot find the file */ + dirent_set_errno (ENOENT); + } + + } + return datap; +} + +/* + * Get next directory entry (internal). + * + * Returns + */ +static WIN32_FIND_DATAW* +dirent_next( + _WDIR *dirp) +{ + WIN32_FIND_DATAW *p; + + /* Get next directory entry */ + if (dirp->cached != 0) { + + /* A valid directory entry already in memory */ + p = &dirp->data; + dirp->cached = 0; + + } else if (dirp->handle != INVALID_HANDLE_VALUE) { + + /* Get the next directory entry from stream */ + if (FindNextFileW (dirp->handle, &dirp->data) != FALSE) { + /* Got a file */ + p = &dirp->data; + } else { + /* The very last entry has been processed or an error occurred */ + FindClose (dirp->handle); + dirp->handle = INVALID_HANDLE_VALUE; + p = NULL; + } + + } else { + + /* End of directory stream reached */ + p = NULL; + + } + + return p; +} + +/* + * Open directory stream using plain old C-string. + */ +static DIR* +opendir( + const char *dirname) +{ + struct DIR *dirp; + + /* Must have directory name */ + if (dirname == NULL || dirname[0] == '\0') { + dirent_set_errno (ENOENT); + return NULL; + } + + /* Allocate memory for DIR structure */ + dirp = (DIR*) malloc (sizeof (struct DIR)); + if (!dirp) { + return NULL; + } + { + int error; + wchar_t wname[PATH_MAX + 1]; + size_t n; + + /* Convert directory name to wide-character string */ + error = dirent_mbstowcs_s( + &n, wname, PATH_MAX + 1, dirname, PATH_MAX + 1); + if (error) { + /* + * Cannot convert file name to wide-character string. This + * occurs if the string contains invalid multi-byte sequences or + * the output buffer is too small to contain the resulting + * string. + */ + goto exit_free; + } + + + /* Open directory stream using wide-character name */ + dirp->wdirp = _wopendir (wname); + if (!dirp->wdirp) { + goto exit_free; + } + + } + + /* Success */ + return dirp; + + /* Failure */ +exit_free: + free (dirp); + return NULL; +} + +/* + * Read next directory entry. + */ +static struct dirent* +readdir( + DIR *dirp) +{ + struct dirent *entry; + + /* + * Read directory entry to buffer. We can safely ignore the return value + * as entry will be set to NULL in case of error. + */ + (void) readdir_r (dirp, &dirp->ent, &entry); + + /* Return pointer to statically allocated directory entry */ + return entry; +} + +/* + * Read next directory entry into called-allocated buffer. + * + * Returns zero on success. If the end of directory stream is reached, then + * sets result to NULL and returns zero. + */ +static int +readdir_r( + DIR *dirp, + struct dirent *entry, + struct dirent **result) +{ + WIN32_FIND_DATAW *datap; + + /* Read next directory entry */ + datap = dirent_next (dirp->wdirp); + if (datap) { + size_t n; + int error; + + /* Attempt to convert file name to multi-byte string */ + error = dirent_wcstombs_s( + &n, entry->d_name, PATH_MAX + 1, datap->cFileName, PATH_MAX + 1); + + /* + * If the file name cannot be represented by a multi-byte string, + * then attempt to use old 8+3 file name. This allows traditional + * Unix-code to access some file names despite of unicode + * characters, although file names may seem unfamiliar to the user. + * + * Be ware that the code below cannot come up with a short file + * name unless the file system provides one. At least + * VirtualBox shared folders fail to do this. + */ + if (error && datap->cAlternateFileName[0] != '\0') { + error = dirent_wcstombs_s( + &n, entry->d_name, PATH_MAX + 1, + datap->cAlternateFileName, PATH_MAX + 1); + } + + if (!error) { + DWORD attr; + + /* Length of file name excluding zero terminator */ + entry->d_namlen = n - 1; + + /* File attributes */ + attr = datap->dwFileAttributes; + if ((attr & FILE_ATTRIBUTE_DEVICE) != 0) { + entry->d_type = DT_CHR; + } else if ((attr & FILE_ATTRIBUTE_DIRECTORY) != 0) { + entry->d_type = DT_DIR; + } else { + entry->d_type = DT_REG; + } + + /* Reset dummy fields */ + entry->d_ino = 0; + entry->d_off = 0; + entry->d_reclen = sizeof (struct dirent); + + } else { + + /* + * Cannot convert file name to multi-byte string so construct + * an erroneous directory entry and return that. Note that + * we cannot return NULL as that would stop the processing + * of directory entries completely. + */ + entry->d_name[0] = '?'; + entry->d_name[1] = '\0'; + entry->d_namlen = 1; + entry->d_type = DT_UNKNOWN; + entry->d_ino = 0; + entry->d_off = -1; + entry->d_reclen = 0; + + } + + /* Return pointer to directory entry */ + *result = entry; + + } else { + + /* No more directory entries */ + *result = NULL; + + } + + return /*OK*/0; +} + +/* + * Close directory stream. + */ +static int +closedir( + DIR *dirp) +{ + int ok; + if (dirp) { + + /* Close wide-character directory stream */ + ok = _wclosedir (dirp->wdirp); + dirp->wdirp = NULL; + + /* Release multi-byte character version */ + free (dirp); + + } else { + + /* Invalid directory stream */ + dirent_set_errno (EBADF); + ok = /*failure*/-1; + + } + return ok; +} + +/* + * Rewind directory stream to beginning. + */ +static void +rewinddir( + DIR* dirp) +{ + /* Rewind wide-character string directory stream */ + _wrewinddir (dirp->wdirp); +} + +/* + * Scan directory for entries. + */ +static int +scandir( + const char *dirname, + struct dirent ***namelist, + int (*filter)(const struct dirent*), + int (*compare)(const struct dirent**, const struct dirent**)) +{ + struct dirent **files = NULL; + size_t size = 0; + size_t allocated = 0; + const size_t init_size = 1; + DIR *dir = NULL; + struct dirent *entry; + struct dirent *tmp = NULL; + size_t i; + int result = 0; + + /* Open directory stream */ + dir = opendir (dirname); + if (dir) { + + /* Read directory entries to memory */ + while (1) { + + /* Enlarge pointer table to make room for another pointer */ + if (size >= allocated) { + void *p; + size_t num_entries; + + /* Compute number of entries in the enlarged pointer table */ + if (size < init_size) { + /* Allocate initial pointer table */ + num_entries = init_size; + } else { + /* Double the size */ + num_entries = size * 2; + } + + /* Allocate first pointer table or enlarge existing table */ + p = realloc (files, sizeof (void*) * num_entries); + if (p != NULL) { + /* Got the memory */ + files = (dirent**) p; + allocated = num_entries; + } else { + /* Out of memory */ + result = -1; + break; + } + + } + + /* Allocate room for temporary directory entry */ + if (tmp == NULL) { + tmp = (struct dirent*) malloc (sizeof (struct dirent)); + if (tmp == NULL) { + /* Cannot allocate temporary directory entry */ + result = -1; + break; + } + } + + /* Read directory entry to temporary area */ + if (readdir_r (dir, tmp, &entry) == /*OK*/0) { + + /* Did we get an entry? */ + if (entry != NULL) { + int pass; + + /* Determine whether to include the entry in result */ + if (filter) { + /* Let the filter function decide */ + pass = filter (tmp); + } else { + /* No filter function, include everything */ + pass = 1; + } + + if (pass) { + /* Store the temporary entry to pointer table */ + files[size++] = tmp; + tmp = NULL; + + /* Keep up with the number of files */ + result++; + } + + } else { + + /* + * End of directory stream reached => sort entries and + * exit. + */ + qsort (files, size, sizeof (void*), + (int (*) (const void*, const void*)) compare); + break; + + } + + } else { + /* Error reading directory entry */ + result = /*Error*/ -1; + break; + } + + } + + } else { + /* Cannot open directory */ + result = /*Error*/ -1; + } + + /* Release temporary directory entry */ + free (tmp); + + /* Release allocated memory on error */ + if (result < 0) { + for (i = 0; i < size; i++) { + free (files[i]); + } + free (files); + files = NULL; + } + + /* Close directory stream */ + if (dir) { + closedir (dir); + } + + /* Pass pointer table to caller */ + if (namelist) { + *namelist = files; + } + return result; +} + +/* Alphabetical sorting */ +static int +alphasort( + const struct dirent **a, const struct dirent **b) +{ + return strcoll ((*a)->d_name, (*b)->d_name); +} + +/* Sort versions */ +static int +versionsort( + const struct dirent **a, const struct dirent **b) +{ + /* FIXME: implement strverscmp and use that */ + return alphasort (a, b); +} + +/* Convert multi-byte string to wide character string */ +static int +dirent_mbstowcs_s( + size_t *pReturnValue, + wchar_t *wcstr, + size_t sizeInWords, + const char *mbstr, + size_t count) +{ + int error; + +#if defined(_MSC_VER) && _MSC_VER >= 1400 + + /* Microsoft Visual Studio 2005 or later */ + error = mbstowcs_s (pReturnValue, wcstr, sizeInWords, mbstr, count); + +#else + + /* Older Visual Studio or non-Microsoft compiler */ + size_t n; + + /* Convert to wide-character string (or count characters) */ + n = mbstowcs (wcstr, mbstr, sizeInWords); + if (!wcstr || n < count) { + + /* Zero-terminate output buffer */ + if (wcstr && sizeInWords) { + if (n >= sizeInWords) { + n = sizeInWords - 1; + } + wcstr[n] = 0; + } + + /* Length of resulting multi-byte string WITH zero terminator */ + if (pReturnValue) { + *pReturnValue = n + 1; + } + + /* Success */ + error = 0; + + } else { + + /* Could not convert string */ + error = 1; + + } + +#endif + return error; +} + +/* Convert wide-character string to multi-byte string */ +static int +dirent_wcstombs_s( + size_t *pReturnValue, + char *mbstr, + size_t sizeInBytes, /* max size of mbstr */ + const wchar_t *wcstr, + size_t count) +{ + int error; + +#if defined(_MSC_VER) && _MSC_VER >= 1400 + + /* Microsoft Visual Studio 2005 or later */ + error = wcstombs_s (pReturnValue, mbstr, sizeInBytes, wcstr, count); + +#else + + /* Older Visual Studio or non-Microsoft compiler */ + size_t n; + + /* Convert to multi-byte string (or count the number of bytes needed) */ + n = wcstombs (mbstr, wcstr, sizeInBytes); + if (!mbstr || n < count) { + + /* Zero-terminate output buffer */ + if (mbstr && sizeInBytes) { + if (n >= sizeInBytes) { + n = sizeInBytes - 1; + } + mbstr[n] = '\0'; + } + + /* Length of resulting multi-bytes string WITH zero-terminator */ + if (pReturnValue) { + *pReturnValue = n + 1; + } + + /* Success */ + error = 0; + + } else { + + /* Cannot convert string */ + error = 1; + + } + +#endif + return error; +} + +/* Set errno variable */ +static void +dirent_set_errno( + int error) +{ +#if defined(_MSC_VER) && _MSC_VER >= 1400 + + /* Microsoft Visual Studio 2005 and later */ + _set_errno (error); + +#else + + /* Non-Microsoft compiler or older Microsoft compiler */ + errno = error; + +#endif +} + + +#ifdef __cplusplus +} +#endif +#endif /*DIRENT_H*/ diff --git a/deps/dirent/tests/1/dir/readme.txt b/deps/dirent/tests/1/dir/readme.txt new file mode 100644 index 0000000000000000000000000000000000000000..0c582df36a63ea505b15ed49c6d4183694354937 --- /dev/null +++ b/deps/dirent/tests/1/dir/readme.txt @@ -0,0 +1,3 @@ +This file ensures that the directory dir will be created accordingly when +you unzip dirent to your computer. The directory itself is needed by the +test program t-dirent. diff --git a/deps/dirent/tests/1/file b/deps/dirent/tests/1/file new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/deps/dirent/tests/2/Testfile-1.2.3.dat b/deps/dirent/tests/2/Testfile-1.2.3.dat new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/deps/dirent/tests/2/file.txt b/deps/dirent/tests/2/file.txt new file mode 100644 index 0000000000000000000000000000000000000000..980283b0a99bc1740e74ceb1d5906697b64ec1d5 --- /dev/null +++ b/deps/dirent/tests/2/file.txt @@ -0,0 +1 @@ +This dummy file is needed by the test program t-dirent. diff --git a/deps/dirent/tests/3/3zero.dat b/deps/dirent/tests/3/3zero.dat new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/deps/dirent/tests/3/666.dat b/deps/dirent/tests/3/666.dat new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/deps/dirent/tests/3/Qwerty-my-aunt.dat b/deps/dirent/tests/3/Qwerty-my-aunt.dat new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/deps/dirent/tests/3/README.txt b/deps/dirent/tests/3/README.txt new file mode 100644 index 0000000000000000000000000000000000000000..e93188e235c94b73d9e7325369ea5c01a731bc0a --- /dev/null +++ b/deps/dirent/tests/3/README.txt @@ -0,0 +1,2 @@ +This directory contains some random files for the t-scandir test program. The +files are empty and only the file names matter. diff --git a/deps/dirent/tests/3/aaa.dat b/deps/dirent/tests/3/aaa.dat new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/deps/dirent/tests/3/dirent.dat b/deps/dirent/tests/3/dirent.dat new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/deps/dirent/tests/3/empty.dat b/deps/dirent/tests/3/empty.dat new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/deps/dirent/tests/3/sane-1.12.0.dat b/deps/dirent/tests/3/sane-1.12.0.dat new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/deps/dirent/tests/3/sane-1.2.3.dat b/deps/dirent/tests/3/sane-1.2.3.dat new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/deps/dirent/tests/3/sane-1.2.4.dat b/deps/dirent/tests/3/sane-1.2.4.dat new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/deps/dirent/tests/3/zebra.dat b/deps/dirent/tests/3/zebra.dat new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/deps/dirent/tests/t-compile.c b/deps/dirent/tests/t-compile.c new file mode 100644 index 0000000000000000000000000000000000000000..e45d52f332ac948c3b8f1ae39bf399a7219b9ea8 --- /dev/null +++ b/deps/dirent/tests/t-compile.c @@ -0,0 +1,47 @@ +/* + * Test program to make sure that dirent compiles cleanly with winsock. + * + * Copyright (C) 1998-2019 Toni Ronkko + * This file is part of dirent. Dirent may be freely distributed + * under the MIT license. For all details and documentation, see + * https://github.com/tronkko/dirent + */ +#include +#ifdef WIN32 +# include +# include +#endif +#include +#include +#include + +int +main( + int argc, char *argv[]) +{ + struct dirent *dirp = NULL; + + (void) argc; + (void) argv; + +#ifdef _DIRENT_HAVE_D_TYPE + printf ("Has d_type\n"); +#endif +#ifdef _DIRENT_HAVE_D_NAMLEN + printf ("Has d_namlen\n"); +#endif +#ifdef _D_EXACT_NAMLEN + printf ("Has _D_EXACT_NAMLEN\n"); +#endif +#ifdef _D_ALLOC_NAMLEN + printf ("Has _D_ALLOC_NAMLEN\n"); +#endif +#ifdef _D_ALLOC_NAMLEN + printf ("Has _D_ALLOC_NAMLEN\n"); +#endif + printf ("Length of d_name with terminator: %d\n", + (int) sizeof (dirp->d_name)); + + printf ("OK\n"); + return EXIT_SUCCESS; +} diff --git a/deps/dirent/tests/t-cplusplus.cpp b/deps/dirent/tests/t-cplusplus.cpp new file mode 100644 index 0000000000000000000000000000000000000000..81d04d0bc2b4b1bfffc666e19252e38df88dabda --- /dev/null +++ b/deps/dirent/tests/t-cplusplus.cpp @@ -0,0 +1,160 @@ +/* + * Test program to make sure that dirent compiles cleanly with C++ + * + * Copyright (C) 1998-2019 Toni Ronkko + * This file is part of dirent. Dirent may be freely distributed + * under the MIT license. For all details and documentation, see + * https://github.com/tronkko/dirent + */ +#include +#include +#include +#include +using namespace std; + + +/* Filter and sort functions */ +static int only_readme (const struct dirent *entry); + + +int +main( + int argc, char *argv[]) +{ + (void) argc; + (void) argv; + + /* Basic directory retrieval */ + { + DIR *dir; + struct dirent *ent; + int found = 0; + + /* Open directory */ + dir = opendir ("tests/1"); + if (dir == NULL) { + cerr << "Directory tests/1 not found" << endl; + abort (); + } + + /* Read entries */ + while ((ent = readdir (dir)) != NULL) { + + /* Check each file */ + if (strcmp (ent->d_name, ".") == 0) { + /* Directory itself */ +#ifdef _DIRENT_HAVE_D_TYPE + assert (ent->d_type == DT_DIR); +#endif +#ifdef _DIRENT_HAVE_D_NAMLEN + assert (ent->d_namlen == 1); +#endif +#ifdef _D_EXACT_NAMLEN + assert (_D_EXACT_NAMLEN(ent) == 1); +#endif +#ifdef _D_ALLOC_NAMLEN + assert (_D_ALLOC_NAMLEN(ent) > 1); +#endif + found += 1; + + } else if (strcmp (ent->d_name, "..") == 0) { + /* Parent directory */ +#ifdef _DIRENT_HAVE_D_TYPE + assert (ent->d_type == DT_DIR); +#endif +#ifdef _DIRENT_HAVE_D_NAMLEN + assert (ent->d_namlen == 2); +#endif +#ifdef _D_EXACT_NAMLEN + assert (_D_EXACT_NAMLEN(ent) == 2); +#endif +#ifdef _D_ALLOC_NAMLEN + assert (_D_ALLOC_NAMLEN(ent) > 2); +#endif + found += 2; + + } else if (strcmp (ent->d_name, "file") == 0) { + /* Regular file */ +#ifdef _DIRENT_HAVE_D_TYPE + assert (ent->d_type == DT_REG); +#endif +#ifdef _DIRENT_HAVE_D_NAMLEN + assert (ent->d_namlen == 4); +#endif +#ifdef _D_EXACT_NAMLEN + assert (_D_EXACT_NAMLEN(ent) == 4); +#endif +#ifdef _D_ALLOC_NAMLEN + assert (_D_ALLOC_NAMLEN(ent) > 4); +#endif + found += 4; + + } else if (strcmp (ent->d_name, "dir") == 0) { + /* Just a directory */ +#ifdef _DIRENT_HAVE_D_TYPE + assert (ent->d_type == DT_DIR); +#endif +#ifdef _DIRENT_HAVE_D_NAMLEN + assert (ent->d_namlen == 3); +#endif +#ifdef _D_EXACT_NAMLEN + assert (_D_EXACT_NAMLEN(ent) == 3); +#endif +#ifdef _D_ALLOC_NAMLEN + assert (_D_ALLOC_NAMLEN(ent) > 3); +#endif + found += 8; + + } else { + /* Other file */ + cerr << "Unexpected file " << ent->d_name << endl; + abort (); + } + + } + + /* Make sure that all files were found */ + assert (found == 0xf); + + closedir (dir); + } + + /* Basic scan with simple filter function */ + { + struct dirent **files; + int n; + int i; + + /* Read directory entries */ + n = scandir ("tests/3", &files, only_readme, alphasort); + assert (n == 1); + + /* Make sure that the filter works */ + assert (strcmp (files[0]->d_name, "README.txt") == 0); + + /* Release file names */ + for (i = 0; i < n; i++) { + free (files[i]); + } + free (files); + } + + + cout << "OK" << endl; + return EXIT_SUCCESS; +} + +/* Only pass README.txt file */ +static int +only_readme (const struct dirent *entry) +{ + int pass; + + if (strcmp (entry->d_name, "README.txt") == 0) { + pass = 1; + } else { + pass = 0; + } + + return pass; +} diff --git a/deps/dirent/tests/t-dirent.c b/deps/dirent/tests/t-dirent.c new file mode 100644 index 0000000000000000000000000000000000000000..053c93a6de89d7b89b2d0d7bf7e05997a0247881 --- /dev/null +++ b/deps/dirent/tests/t-dirent.c @@ -0,0 +1,633 @@ +/* + * A test program to make sure that dirent works correctly. + * + * Copyright (C) 1998-2019 Toni Ronkko + * This file is part of dirent. Dirent may be freely distributed + * under the MIT license. For all details and documentation, see + * https://github.com/tronkko/dirent + */ +#include +#include +#include +#ifdef _MSC_VER +# include +# define chdir(x) _chdir(x) +#else +# include +#endif +#include +#include +#include + +#undef NDEBUG +#include + + +int +main( + int argc, char *argv[]) +{ + (void) argc; + (void) argv; + + /* File type macros */ + { + assert (DTTOIF(DT_REG) == S_IFREG); + assert (DTTOIF(DT_DIR) == S_IFDIR); + assert (DTTOIF(DT_FIFO) == S_IFIFO); + assert (DTTOIF(DT_SOCK) == S_IFSOCK); + assert (DTTOIF(DT_CHR) == S_IFCHR); + assert (DTTOIF(DT_BLK) == S_IFBLK); + + assert (IFTODT(S_IFREG) == DT_REG); + assert (IFTODT(S_IFDIR) == DT_DIR); + assert (IFTODT(S_IFIFO) == DT_FIFO); + assert (IFTODT(S_IFSOCK) == DT_SOCK); + assert (IFTODT(S_IFCHR) == DT_CHR); + assert (IFTODT(S_IFBLK) == DT_BLK); + } + + /* Basic directory retrieval */ + { + DIR *dir; + struct dirent *ent; + int found = 0; + + /* Open directory */ + dir = opendir ("tests/1"); + if (dir == NULL) { + fprintf (stderr, "Directory tests/1 not found\n"); + abort (); + } + + /* Read entries */ + while ((ent = readdir (dir)) != NULL) { + + /* Check each file */ + if (strcmp (ent->d_name, ".") == 0) { + /* Directory itself */ +#ifdef _DIRENT_HAVE_D_TYPE + assert (ent->d_type == DT_DIR); +#endif +#ifdef _DIRENT_HAVE_D_NAMLEN + assert (ent->d_namlen == 1); +#endif +#ifdef _D_EXACT_NAMLEN + assert (_D_EXACT_NAMLEN(ent) == 1); +#endif +#ifdef _D_ALLOC_NAMLEN + assert (_D_ALLOC_NAMLEN(ent) > 1); +#endif + found += 1; + + } else if (strcmp (ent->d_name, "..") == 0) { + /* Parent directory */ +#ifdef _DIRENT_HAVE_D_TYPE + assert (ent->d_type == DT_DIR); +#endif +#ifdef _DIRENT_HAVE_D_NAMLEN + assert (ent->d_namlen == 2); +#endif +#ifdef _D_EXACT_NAMLEN + assert (_D_EXACT_NAMLEN(ent) == 2); +#endif +#ifdef _D_ALLOC_NAMLEN + assert (_D_ALLOC_NAMLEN(ent) > 2); +#endif + found += 2; + + } else if (strcmp (ent->d_name, "file") == 0) { + /* Regular file */ +#ifdef _DIRENT_HAVE_D_TYPE + assert (ent->d_type == DT_REG); +#endif +#ifdef _DIRENT_HAVE_D_NAMLEN + assert (ent->d_namlen == 4); +#endif +#ifdef _D_EXACT_NAMLEN + assert (_D_EXACT_NAMLEN(ent) == 4); +#endif +#ifdef _D_ALLOC_NAMLEN + assert (_D_ALLOC_NAMLEN(ent) > 4); +#endif + found += 4; + + } else if (strcmp (ent->d_name, "dir") == 0) { + /* Just a directory */ +#ifdef _DIRENT_HAVE_D_TYPE + assert (ent->d_type == DT_DIR); +#endif +#ifdef _DIRENT_HAVE_D_NAMLEN + assert (ent->d_namlen == 3); +#endif +#ifdef _D_EXACT_NAMLEN + assert (_D_EXACT_NAMLEN(ent) == 3); +#endif +#ifdef _D_ALLOC_NAMLEN + assert (_D_ALLOC_NAMLEN(ent) > 3); +#endif + found += 8; + + } else { + /* Other file */ + fprintf (stderr, "Unexpected file %s\n", ent->d_name); + abort (); + } + + } + + /* Make sure that all files were found */ + assert (found == 0xf); + + closedir (dir); + } + + /* Function opendir() fails if directory doesn't exist */ + { + DIR *dir; + + /* Open directory */ + dir = opendir ("tests/invalid"); + assert (dir == NULL); + assert (errno == ENOENT); + } + + /* Function opendir() fails if pathname is really a file */ + { + DIR *dir; + + /* Open directory */ + dir = opendir ("tests/1/file"); + assert (dir == NULL); + assert (errno == ENOTDIR); + } + + /* Function opendir() fails if pathname is a zero-length string */ + { + DIR *dir; + + /* Open directory */ + dir = opendir (""); + assert (dir == NULL); + assert (errno == ENOENT); + } + + /* Rewind of directory stream */ + { + DIR *dir; + struct dirent *ent; + int found = 0; + + /* Open directory */ + dir = opendir ("tests/1"); + assert (dir != NULL); + + /* Read entries */ + while ((ent = readdir (dir)) != NULL) { + + /* Check each file */ + if (strcmp (ent->d_name, ".") == 0) { + /* Directory itself */ + found += 1; + + } else if (strcmp (ent->d_name, "..") == 0) { + /* Parent directory */ + found += 2; + + } else if (strcmp (ent->d_name, "file") == 0) { + /* Regular file */ + found += 4; + + } else if (strcmp (ent->d_name, "dir") == 0) { + /* Just a directory */ + found += 8; + + } else { + /* Other file */ + fprintf (stderr, "Unexpected file %s\n", ent->d_name); + abort (); + } + + } + + /* Make sure that all files were found */ + assert (found == 0xf); + + /* Rewind stream and read entries again */ + rewinddir (dir); + found = 0; + + /* Read entries */ + while ((ent = readdir (dir)) != NULL) { + + /* Check each file */ + if (strcmp (ent->d_name, ".") == 0) { + /* Directory itself */ + found += 1; + + } else if (strcmp (ent->d_name, "..") == 0) { + /* Parent directory */ + found += 2; + + } else if (strcmp (ent->d_name, "file") == 0) { + /* Regular file */ + found += 4; + + } else if (strcmp (ent->d_name, "dir") == 0) { + /* Just a directory */ + found += 8; + + } else { + /* Other file */ + fprintf (stderr, "Unexpected file %s\n", ent->d_name); + abort (); + } + + } + + /* Make sure that all files were found */ + assert (found == 0xf); + + closedir (dir); + } + + /* Rewind with intervening change of working directory */ + { + DIR *dir; + struct dirent *ent; + int found = 0; + int errorcode; + + /* Open directory */ + dir = opendir ("tests/1"); + assert (dir != NULL); + + /* Read entries */ + while ((ent = readdir (dir)) != NULL) { + + /* Check each file */ + if (strcmp (ent->d_name, ".") == 0) { + /* Directory itself */ + found += 1; + + } else if (strcmp (ent->d_name, "..") == 0) { + /* Parent directory */ + found += 2; + + } else if (strcmp (ent->d_name, "file") == 0) { + /* Regular file */ + found += 4; + + } else if (strcmp (ent->d_name, "dir") == 0) { + /* Just a directory */ + found += 8; + + } else { + /* Other file */ + fprintf (stderr, "Unexpected file %s\n", ent->d_name); + abort (); + } + + } + + /* Make sure that all files were found */ + assert (found == 0xf); + + /* Change working directory */ + errorcode = chdir ("tests"); + assert (errorcode == 0); + + /* Rewind stream and read entries again */ + rewinddir (dir); + found = 0; + + /* Read entries */ + while ((ent = readdir (dir)) != NULL) { + + /* Check each file */ + if (strcmp (ent->d_name, ".") == 0) { + /* Directory itself */ + found += 1; + + } else if (strcmp (ent->d_name, "..") == 0) { + /* Parent directory */ + found += 2; + + } else if (strcmp (ent->d_name, "file") == 0) { + /* Regular file */ + found += 4; + + } else if (strcmp (ent->d_name, "dir") == 0) { + /* Just a directory */ + found += 8; + + } else { + /* Other file */ + fprintf (stderr, "Unexpected file %s\n", ent->d_name); + abort (); + } + + } + + /* Make sure that all files were found */ + assert (found == 0xf); + + /* Restore working directory */ + errorcode = chdir (".."); + assert (errorcode == 0); + + closedir (dir); + } + + /* Long file name */ + { + DIR *dir; + struct dirent *ent; + int found = 0; + + /* Open directory */ + dir = opendir ("tests/2"); + if (dir == NULL) { + fprintf (stderr, "Directory tests/2 not found\n"); + abort (); + } + + /* Read entries */ + while ((ent = readdir (dir)) != NULL) { + + /* Check each file */ + if (strcmp (ent->d_name, ".") == 0) { + /* Directory itself */ + found += 1; + + } else if (strcmp (ent->d_name, "..") == 0) { + /* Parent directory */ + found += 2; + + } else if (strcmp (ent->d_name, "file.txt") == 0) { + /* Regular 8+3 filename */ +#ifdef _DIRENT_HAVE_D_TYPE + assert (ent->d_type == DT_REG); +#endif +#ifdef _DIRENT_HAVE_D_NAMLEN + assert (ent->d_namlen == 8); +#endif +#ifdef _D_EXACT_NAMLEN + assert (_D_EXACT_NAMLEN(ent) == 8); +#endif +#ifdef _D_ALLOC_NAMLEN + assert (_D_ALLOC_NAMLEN(ent) > 8); +#endif + found += 4; + + } else if (strcmp (ent->d_name, "Testfile-1.2.3.dat") == 0) { + /* Long file name with multiple dots */ +#ifdef _DIRENT_HAVE_D_TYPE + assert (ent->d_type == DT_REG); +#endif +#ifdef _DIRENT_HAVE_D_NAMLEN + assert (ent->d_namlen == 18); +#endif +#ifdef _D_EXACT_NAMLEN + assert (_D_EXACT_NAMLEN(ent) == 18); +#endif +#ifdef _D_ALLOC_NAMLEN + assert (_D_ALLOC_NAMLEN(ent) > 18); +#endif + found += 8; + + } else { + /* Other file */ + fprintf (stderr, "Unexpected file %s\n", ent->d_name); + abort (); + } + + } + + /* Make sure that all files were found */ + assert (found == 0xf); + + closedir (dir); + } + + /* Basic directory retrieval with readdir_r */ + { + DIR *dir; + struct dirent ent[10]; + struct dirent *entry; + size_t i = 0; + size_t n = 0; + int found = 0; + + /* Open directory */ + dir = opendir ("tests/1"); + if (dir == NULL) { + fprintf (stderr, "Directory tests/1 not found\n"); + abort (); + } + + /* Read entries to table */ + while (readdir_r (dir, &ent[n], &entry) == /*OK*/0 && entry != 0) { + n++; + assert (n <= 4); + } + + /* Make sure that we got all the files from directory */ + assert (n == 4); + + /* Check entries in memory */ + for (i = 0; i < 4; i++) { + entry = &ent[i]; + + /* Check each file */ + if (strcmp (entry->d_name, ".") == 0) { + /* Directory itself */ +#ifdef _DIRENT_HAVE_D_TYPE + assert (entry->d_type == DT_DIR); +#endif +#ifdef _DIRENT_HAVE_D_NAMLEN + assert (entry->d_namlen == 1); +#endif +#ifdef _D_EXACT_NAMLEN + assert (_D_EXACT_NAMLEN(entry) == 1); +#endif +#ifdef _D_ALLOC_NAMLEN + assert (_D_ALLOC_NAMLEN(entry) > 1); +#endif + found += 1; + + } else if (strcmp (entry->d_name, "..") == 0) { + /* Parent directory */ +#ifdef _DIRENT_HAVE_D_TYPE + assert (entry->d_type == DT_DIR); +#endif +#ifdef _DIRENT_HAVE_D_NAMLEN + assert (entry->d_namlen == 2); +#endif +#ifdef _D_EXACT_NAMLEN + assert (_D_EXACT_NAMLEN(entry) == 2); +#endif +#ifdef _D_ALLOC_NAMLEN + assert (_D_ALLOC_NAMLEN(entry) > 2); +#endif + found += 2; + + } else if (strcmp (entry->d_name, "file") == 0) { + /* Regular file */ +#ifdef _DIRENT_HAVE_D_TYPE + assert (entry->d_type == DT_REG); +#endif +#ifdef _DIRENT_HAVE_D_NAMLEN + assert (entry->d_namlen == 4); +#endif +#ifdef _D_EXACT_NAMLEN + assert (_D_EXACT_NAMLEN(entry) == 4); +#endif +#ifdef _D_ALLOC_NAMLEN + assert (_D_ALLOC_NAMLEN(entry) > 4); +#endif + found += 4; + + } else if (strcmp (entry->d_name, "dir") == 0) { + /* Just a directory */ +#ifdef _DIRENT_HAVE_D_TYPE + assert (entry->d_type == DT_DIR); +#endif +#ifdef _DIRENT_HAVE_D_NAMLEN + assert (entry->d_namlen == 3); +#endif +#ifdef _D_EXACT_NAMLEN + assert (_D_EXACT_NAMLEN(entry) == 3); +#endif +#ifdef _D_ALLOC_NAMLEN + assert (_D_ALLOC_NAMLEN(entry) > 3); +#endif + found += 8; + + } else { + /* Other file */ + fprintf (stderr, "Unexpected file %s\n", entry->d_name); + abort (); + } + + } + + /* Make sure that all files were found */ + assert (found == 0xf); + + closedir (dir); + } + + /* Basic directory retrieval with _wreaddir_r */ +#ifdef WIN32 + { + _WDIR *dir; + struct _wdirent ent[10]; + struct _wdirent *entry; + size_t i = 0; + size_t n = 0; + int found = 0; + + /* Open directory */ + dir = _wopendir (L"tests/1"); + if (dir == NULL) { + fprintf (stderr, "Directory tests/1 not found\n"); + abort (); + } + + /* Read entries to table */ + while (_wreaddir_r (dir, &ent[n], &entry) == /*OK*/0 && entry != 0) { + n++; + assert (n <= 4); + } + + /* Make sure that we got all the files from directory */ + assert (n == 4); + + /* Check entries in memory */ + for (i = 0; i < 4; i++) { + entry = &ent[i]; + + /* Check each file */ + if (wcscmp (entry->d_name, L".") == 0) { + /* Directory itself */ +#ifdef _DIRENT_HAVE_D_TYPE + assert (entry->d_type == DT_DIR); +#endif +#ifdef _DIRENT_HAVE_D_NAMLEN + assert (entry->d_namlen == 1); +#endif +#ifdef _D_EXACT_NAMLEN + assert (_D_EXACT_NAMLEN(entry) == 1); +#endif +#ifdef _D_ALLOC_NAMLEN + assert (_D_ALLOC_NAMLEN(entry) > 1); +#endif + found += 1; + + } else if (wcscmp (entry->d_name, L"..") == 0) { + /* Parent directory */ +#ifdef _DIRENT_HAVE_D_TYPE + assert (entry->d_type == DT_DIR); +#endif +#ifdef _DIRENT_HAVE_D_NAMLEN + assert (entry->d_namlen == 2); +#endif +#ifdef _D_EXACT_NAMLEN + assert (_D_EXACT_NAMLEN(entry) == 2); +#endif +#ifdef _D_ALLOC_NAMLEN + assert (_D_ALLOC_NAMLEN(entry) > 2); +#endif + found += 2; + + } else if (wcscmp (entry->d_name, L"file") == 0) { + /* Regular file */ +#ifdef _DIRENT_HAVE_D_TYPE + assert (entry->d_type == DT_REG); +#endif +#ifdef _DIRENT_HAVE_D_NAMLEN + assert (entry->d_namlen == 4); +#endif +#ifdef _D_EXACT_NAMLEN + assert (_D_EXACT_NAMLEN(entry) == 4); +#endif +#ifdef _D_ALLOC_NAMLEN + assert (_D_ALLOC_NAMLEN(entry) > 4); +#endif + found += 4; + + } else if (wcscmp (entry->d_name, L"dir") == 0) { + /* Just a directory */ +#ifdef _DIRENT_HAVE_D_TYPE + assert (entry->d_type == DT_DIR); +#endif +#ifdef _DIRENT_HAVE_D_NAMLEN + assert (entry->d_namlen == 3); +#endif +#ifdef _D_EXACT_NAMLEN + assert (_D_EXACT_NAMLEN(entry) == 3); +#endif +#ifdef _D_ALLOC_NAMLEN + assert (_D_ALLOC_NAMLEN(entry) > 3); +#endif + found += 8; + + } else { + /* Other file */ + fprintf (stderr, "Unexpected file\n"); + abort (); + } + + } + + /* Make sure that all files were found */ + assert (found == 0xf); + + _wclosedir (dir); + } +#endif + + printf ("OK\n"); + return EXIT_SUCCESS; +} diff --git a/deps/dirent/tests/t-scandir.c b/deps/dirent/tests/t-scandir.c new file mode 100644 index 0000000000000000000000000000000000000000..491c962fe5aea540e27e4a5e9c35f7504b23fd1f --- /dev/null +++ b/deps/dirent/tests/t-scandir.c @@ -0,0 +1,264 @@ +/* + * Make sure that scandir function works OK. + * + * Copyright (C) 1998-2019 Toni Ronkko + * This file is part of dirent. Dirent may be freely distributed + * under the MIT license. For all details and documentation, see + * https://github.com/tronkko/dirent + */ + +/* Silence warning about fopen being insecure */ +#define _CRT_SECURE_NO_WARNINGS + +#include +#include +#include +#include +#include +#include +#include +#include + +#undef NDEBUG +#include + + +/* Filter and sort functions */ +static int only_readme (const struct dirent *entry); +static int no_directories (const struct dirent *entry); +static int reverse_alpha (const struct dirent **a, const struct dirent **b); + + +int +main( + int argc, char *argv[]) +{ + struct dirent **files; + int i; + int n; + + (void) argc; + (void) argv; + + /* Initialize random number generator */ + srand ((unsigned) time (NULL)); + + /* Basic scan with simple filter function */ + { + /* Read directory entries */ + n = scandir ("tests/3", &files, only_readme, alphasort); + assert (n == 1); + + /* Make sure that the filter works */ + assert (strcmp (files[0]->d_name, "README.txt") == 0); + + /* Release file names */ + for (i = 0; i < n; i++) { + free (files[i]); + } + free (files); + } + + /* Basic scan with default sorting function */ + { + /* Read directory entries in alphabetic order */ + n = scandir ("tests/3", &files, NULL, alphasort); + assert (n == 13); + + /* Make sure that we got all the file names in the proper order */ + assert (strcmp (files[0]->d_name, ".") == 0); + assert (strcmp (files[1]->d_name, "..") == 0); + assert (strcmp (files[2]->d_name, "3zero.dat") == 0); + assert (strcmp (files[3]->d_name, "666.dat") == 0); + assert (strcmp (files[4]->d_name, "Qwerty-my-aunt.dat") == 0); + assert (strcmp (files[5]->d_name, "README.txt") == 0); + assert (strcmp (files[6]->d_name, "aaa.dat") == 0); + assert (strcmp (files[7]->d_name, "dirent.dat") == 0); + assert (strcmp (files[8]->d_name, "empty.dat") == 0); + assert (strcmp (files[9]->d_name, "sane-1.12.0.dat") == 0); + assert (strcmp (files[10]->d_name, "sane-1.2.3.dat") == 0); + assert (strcmp (files[11]->d_name, "sane-1.2.4.dat") == 0); + assert (strcmp (files[12]->d_name, "zebra.dat") == 0); + + /* Release file names */ + for (i = 0; i < n; i++) { + free (files[i]); + } + free (files); + } + + /* Custom filter AND sort function */ + { + /* Read directory entries in alphabetic order */ + n = scandir ("tests/3", &files, no_directories, reverse_alpha); + assert (n == 11); + + /* Make sure that we got all the FILE names in the REVERSE order */ + assert (strcmp (files[0]->d_name, "zebra.dat") == 0); + assert (strcmp (files[1]->d_name, "sane-1.2.4.dat") == 0); + assert (strcmp (files[2]->d_name, "sane-1.2.3.dat") == 0); + assert (strcmp (files[3]->d_name, "sane-1.12.0.dat") == 0); + assert (strcmp (files[4]->d_name, "empty.dat") == 0); + assert (strcmp (files[5]->d_name, "dirent.dat") == 0); + assert (strcmp (files[6]->d_name, "aaa.dat") == 0); + assert (strcmp (files[7]->d_name, "README.txt") == 0); + assert (strcmp (files[8]->d_name, "Qwerty-my-aunt.dat") == 0); + assert (strcmp (files[9]->d_name, "666.dat") == 0); + assert (strcmp (files[10]->d_name, "3zero.dat") == 0); + + /* Release file names */ + for (i = 0; i < n; i++) { + free (files[i]); + } + free (files); + } + + /* Trying to read from non-existent directory leads to an error */ + { + files = NULL; + n = scandir ("tests/invalid", &files, NULL, alphasort); + assert (n == -1); + assert (files == NULL); + assert (errno == ENOENT); + } + + /* Trying to open file as a directory produces ENOTDIR error */ + { + files = NULL; + n = scandir ("tests/3/666.dat", &files, NULL, alphasort); + assert (n == -1); + assert (files == NULL); + assert (errno == ENOTDIR); + } + + /* Scan large directory */ + { + char dirname[PATH_MAX+1]; + int i, j; + int ok; + + /* Copy name of temporary directory to variable dirname */ +#ifdef WIN32 + i = GetTempPathA (PATH_MAX, dirname); + assert (i > 0); +#else + strcpy (dirname, "/tmp/"); + i = strlen (dirname); +#endif + + /* Append random characters to dirname */ + for (j = 0; j < 10; j++) { + char c; + + /* Generate random character */ + c = "abcdefghijklmnopqrstuvwxyz"[rand() % 26]; + + /* Append character to dirname */ + assert (i < PATH_MAX); + dirname[i++] = c; + } + + /* Terminate directory name */ + assert (i < PATH_MAX); + dirname[i] = '\0'; + + /* Create directory */ +#ifdef WIN32 + ok = CreateDirectoryA (dirname, NULL); + assert (ok); +#else + ok = mkdir (dirname, 0700); + assert (ok == /*success*/0); +#endif + + /* Create one thousand files */ + assert (i + 5 < PATH_MAX); + for (j = 0; j < 1000; j++) { + FILE *fp; + + /* Construct file name */ + dirname[i] = '/'; + dirname[i+1] = 'z'; + dirname[i+2] = '0' + ((j / 100) % 10); + dirname[i+3] = '0' + ((j / 10) % 10); + dirname[i+4] = '0' + (j % 10); + dirname[i+5] = '\0'; + + /* Create file */ + fp = fopen (dirname, "w"); + assert (fp != NULL); + fclose (fp); + + } + + /* Cut out the file name part */ + dirname[i] = '\0'; + + /* Scan directory */ + n = scandir (dirname, &files, no_directories, alphasort); + assert (n == 1000); + + /* Make sure that all 1000 files are read back */ + for (j = 0; j < n; j++) { + char match[100]; + + /* Construct file name */ + match[0] = 'z'; + match[1] = '0' + ((j / 100) % 10); + match[2] = '0' + ((j / 10) % 10); + match[3] = '0' + (j % 10); + match[4] = '\0'; + + /* Make sure that file name matches that on the disk */ + assert (strcmp (files[j]->d_name, match) == 0); + + } + + /* Release file names */ + for (j = 0; j < n; j++) { + free (files[j]); + } + free (files); + } + + printf ("OK\n"); + return EXIT_SUCCESS; +} + +/* Only pass README.txt file */ +static int +only_readme (const struct dirent *entry) +{ + int pass; + + if (strcmp (entry->d_name, "README.txt") == 0) { + pass = 1; + } else { + pass = 0; + } + + return pass; +} + +/* Filter out directories */ +static int +no_directories (const struct dirent *entry) +{ + int pass; + + if (entry->d_type != DT_DIR) { + pass = 1; + } else { + pass = 0; + } + + return pass; +} + +/* Sort in reverse direction */ +static int +reverse_alpha( + const struct dirent **a, const struct dirent **b) +{ + return strcoll ((*b)->d_name, (*a)->d_name); +} diff --git a/deps/dirent/tests/t-unicode.c b/deps/dirent/tests/t-unicode.c new file mode 100644 index 0000000000000000000000000000000000000000..1a65d0bb279ae27b07c49cb65cb024a98b259445 --- /dev/null +++ b/deps/dirent/tests/t-unicode.c @@ -0,0 +1,401 @@ +/* + * Test program to try unicode file names. + * + * Copyright (C) 1998-2019 Toni Ronkko + * This file is part of dirent. Dirent may be freely distributed + * under the MIT license. For all details and documentation, see + * https://github.com/tronkko/dirent + */ + +/* Silence warning about fopen being insecure */ +#define _CRT_SECURE_NO_WARNINGS + +#include +#include +#include +#include +#include +#include +#include + +#undef NDEBUG +#include + + +int +main( + int argc, char *argv[]) +{ +#ifdef WIN32 + wchar_t wpath[MAX_PATH+1]; + char path[MAX_PATH+1]; + DWORD i, j, k, x; + BOOL ok; + HANDLE fh; + _WDIR *wdir; + struct _wdirent *wentry; + DIR *dir; + struct dirent *entry; + char buffer[100]; + FILE *fp; + int counter = 0; + + (void) argc; + (void) argv; + + + /* Initialize random number generator */ + srand (((int) time (NULL)) * 257 + ((int) GetCurrentProcessId ())); + + /* Set current locale */ + if (argc > 1) { + printf ("Locale %s\n", argv[1]); + setlocale (LC_ALL, argv[1]); + } else { + setlocale (LC_ALL, ""); + } + + + /****** CREATE FILE WITH UNICODE FILE NAME ******/ + + /* Get path to temporary directory (wide-character and ascii) */ + i = GetTempPathW (MAX_PATH, wpath); + assert (i > 0); + j = GetTempPathA (MAX_PATH, path); + assert (j > 0); + + /* Append random directory name */ + for (k = 0; k < 10; k++) { + char c; + + /* Generate random character */ + c = "abcdefghijklmnopqrstuvwxyz"[rand() % 26]; + + /* Append character to paths */ + assert (i < MAX_PATH && j < MAX_PATH); + wpath[i++] = c; + path[j++] = c; + } + + /* Terminate paths */ + assert (i < MAX_PATH && j < MAX_PATH); + wpath[i] = '\0'; + path[j] = '\0'; + + /* Remember the end of directory name */ + k = i; + + /* Create directory using unicode */ + ok = CreateDirectoryW (wpath, NULL); + if (!ok) { + DWORD e = GetLastError (); + wprintf (L"Cannot create directory %ls (code %u)\n", wpath, e); + abort (); + } + + /* Overwrite zero terminator with path separator */ + assert (i < MAX_PATH && j < MAX_PATH); + wpath[i++] = '\\'; + + /* Append a few unicode characters */ + assert (i < MAX_PATH); + wpath[i++] = 0x6d4b; + assert (i < MAX_PATH); + wpath[i++] = 0x8bd5; + + /* Terminate string */ + assert (i < MAX_PATH); + wpath[i] = '\0'; + + /* Create file with unicode */ + fh = CreateFileW( + wpath, + /* Access */ GENERIC_READ | GENERIC_WRITE, + /* Share mode */ 0, + /* Security attributes */ NULL, + /* Creation disposition */ CREATE_NEW, + /* Attributes */ FILE_ATTRIBUTE_NORMAL, + /* Template files */ NULL + ); + assert (fh != INVALID_HANDLE_VALUE); + + /* Write some data to file */ + ok = WriteFile( + /* File handle */ fh, + /* Pointer to data */ "hep\n", + /* Number of bytes to write */ 4, + /* Number of bytes written */ NULL, + /* Overlapped */ NULL + ); + assert (ok); + + /* Close file */ + ok = CloseHandle (fh); + assert (ok); + + + /****** MAKE SURE THAT UNICODE FILE NAME CAN BE READ BY _WREADDIR ******/ + + /* Zero terminate wide-character path and open directory stream */ + wpath[k] = '\0'; + wdir = _wopendir (wpath); + if (wdir == NULL) { + wprintf (L"Cannot open directory %ls\n", wpath); + abort (); + } + + /* Read through entries */ + counter = 0; + while ((wentry = _wreaddir (wdir)) != NULL) { + + /* Skip pseudo directories */ + if (wcscmp (wentry->d_name, L".") == 0) { + continue; + } + if (wcscmp (wentry->d_name, L"..") == 0) { + continue; + } + + /* Found a file */ + counter++; + assert (wentry->d_type == DT_REG); + + /* Append file name to path */ + i = k; + assert (i < MAX_PATH); + wpath[i++] = '\\'; + x = 0; + while (wentry->d_name[x] != '\0') { + assert (i < MAX_PATH); + wpath[i++] = wentry->d_name[x++]; + } + assert (i < MAX_PATH); + wpath[i] = '\0'; + + /* Open file for read */ + fh = CreateFileW( + wpath, + /* Access */ GENERIC_READ, + /* Share mode */ 0, + /* Security attributes */ NULL, + /* Creation disposition */ OPEN_EXISTING, + /* Attributes */ FILE_ATTRIBUTE_NORMAL, + /* Template files */ NULL + ); + assert (fh != INVALID_HANDLE_VALUE); + + /* Read data from file */ + ok = ReadFile( + /* File handle */ fh, + /* Output buffer */ buffer, + /* Maximum number of bytes to read */ sizeof (buffer) - 1, + /* Number of bytes actually read */ &x, + /* Overlapped */ NULL + ); + assert (ok); + + /* Make sure that we got the file contents right */ + assert (x == 4); + assert (buffer[0] == 'h'); + assert (buffer[1] == 'e'); + assert (buffer[2] == 'p'); + assert (buffer[3] == '\n'); + + /* Close file */ + ok = CloseHandle (fh); + assert (ok); + + } + assert (counter == 1); + + /* Close directory */ + _wclosedir (wdir); + + + /****** MAKE SURE THAT UNICODE FILE NAME CAN BE READ BY READDIR ******/ + + /* Zero terminate ascii path and open directory stream */ + k = j; + path[k] = '\0'; + dir = opendir (path); + if (dir == NULL) { + fprintf (stderr, "Cannot open directory %s\n", path); + abort (); + } + + /* Read through entries */ + counter = 0; + while ((entry = readdir (dir)) != NULL) { + + /* Skip pseudo directories */ + if (strcmp (entry->d_name, ".") == 0) { + continue; + } + if (strcmp (entry->d_name, "..") == 0) { + continue; + } + + /* Found a file */ + counter++; + assert (entry->d_type == DT_REG); + + /* Append file name to path */ + j = k; + assert (j < MAX_PATH); + path[j++] = '\\'; + x = 0; + while (entry->d_name[x] != '\0') { + assert (j < MAX_PATH); + path[j++] = entry->d_name[x++]; + } + assert (j < MAX_PATH); + path[j] = '\0'; + + /* Open file for read */ + fp = fopen (path, "r"); + if (!fp) { + fprintf (stderr, "Cannot open file %s\n", path); + abort (); + } + + /* Read data from file */ + if (fgets (buffer, sizeof (buffer), fp) == NULL) { + fprintf (stderr, "Cannot read file %s\n", path); + abort (); + } + + /* Make sure that we got the file contents right */ + assert (buffer[0] == 'h'); + assert (buffer[1] == 'e'); + assert (buffer[2] == 'p'); + assert (buffer[3] == '\n'); + assert (buffer[4] == '\0'); + + /* Close file */ + fclose (fp); + + } + assert (counter == 1); + + /* Close directory */ + closedir (dir); + + + /****** CREATE FILE WITH UTF-8 ******/ + + /* Append UTF-8 file name (åäö.txt) to path */ + j = k; + path[j++] = '\\'; + path[j++] = 0xc3; + path[j++] = 0xa5; + path[j++] = 0xc3; + path[j++] = 0xa4; + path[j++] = 0xc3; + path[j++] = 0xb6; + path[j++] = 0x2e; + path[j++] = 0x74; + path[j++] = 0x78; + path[j++] = 0x74; + assert (j < MAX_PATH); + path[j] = '\0'; + + /* + * Create file. + * + * Be ware that the code below creates a different file depending on the + * current locale! For example, if the current locale is + * english_us.65001, then the file name will be "åäö.txt" (7 characters). + * However, if the current locale is english_us.1252, then the file name + * will be "ÃċÃĊö.txt" (10 characters). + */ + printf ("Creating %s\n", path); + fp = fopen (path, "w"); + if (!fp) { + fprintf (stderr, "Cannot open file %s\n", path); + abort (); + } + fputs ("hep\n", fp); + fclose (fp); + + /* Open directory again */ + path[k] = '\0'; + dir = opendir (path); + if (dir == NULL) { + fprintf (stderr, "Cannot open directory %s\n", path); + abort (); + } + + /* Read through entries */ + counter = 0; + while ((entry = readdir (dir)) != NULL) { + + /* Skip pseudo directories */ + if (strcmp (entry->d_name, ".") == 0) { + continue; + } + if (strcmp (entry->d_name, "..") == 0) { + continue; + } + + /* Found a file */ + counter++; + assert (entry->d_type == DT_REG); + + /* Append file name to path */ + j = k; + assert (j < MAX_PATH); + path[j++] = '\\'; + x = 0; + while (entry->d_name[x] != '\0') { + assert (j < MAX_PATH); + path[j++] = entry->d_name[x++]; + } + assert (j < MAX_PATH); + path[j] = '\0'; + + /* Print file name for debugging */ + printf ("Opening \"%s\" hex ", path + k + 1); + x = 0; + while (entry->d_name[x] != '\0') { + printf ("0x%02x ", (unsigned) (entry->d_name[x++] & 0xff)); + } + printf ("\n"); + + /* Open file for read */ + fp = fopen (path, "r"); + if (!fp) { + fprintf (stderr, "Cannot open file %s\n", path); + abort (); + } + + /* Read data from file */ + if (fgets (buffer, sizeof (buffer), fp) == NULL) { + fprintf (stderr, "Cannot read file %s\n", path); + abort (); + } + + /* Make sure that we got the file contents right */ + assert (buffer[0] == 'h'); + assert (buffer[1] == 'e'); + assert (buffer[2] == 'p'); + assert (buffer[3] == '\n'); + assert (buffer[4] == '\0'); + + /* Close file */ + fclose (fp); + + } + assert (counter == 2); + + /* Close directory */ + closedir (dir); + +#else + + /* Linux */ + (void) argc; + (void) argv; + +#endif + return EXIT_SUCCESS; +} diff --git a/deps/iconv/CMakeLists.txt b/deps/iconv/CMakeLists.txt index 944cb400e0c80a92f27aed159a4886ed25d9af6d..b2a5dce903793dcdb228ebe39e8f4f41171015aa 100644 --- a/deps/iconv/CMakeLists.txt +++ b/deps/iconv/CMakeLists.txt @@ -1,7 +1,7 @@ CMAKE_MINIMUM_REQUIRED(VERSION 2.8) PROJECT(TDengine) -IF (TD_WINDOWS_64) +IF (TD_WINDOWS_64 OR TD_WINDOWS_32) LIST(APPEND SRC iconv.c) LIST(APPEND SRC localcharset.c) INCLUDE_DIRECTORIES(.) diff --git a/deps/pthread/CMakeLists.txt b/deps/pthread/CMakeLists.txt index a218d05bab036685b5a3e2e86a75cc5af26eaee5..4526d4a6512184a05f6025673b04eb962ce9aefb 100644 --- a/deps/pthread/CMakeLists.txt +++ b/deps/pthread/CMakeLists.txt @@ -1,7 +1,7 @@ CMAKE_MINIMUM_REQUIRED(VERSION 2.8) PROJECT(TDengine) -IF (TD_WINDOWS_64) +IF (TD_WINDOWS_64 OR TD_WINDOWS_32) INCLUDE_DIRECTORIES(.) LIST(APPEND SRC pthread.c) ADD_LIBRARY(pthread ${SRC}) diff --git a/deps/regex/CMakeLists.txt b/deps/regex/CMakeLists.txt index 8f8396caa45f56ff5759b1f2cc55ae28086be6ab..5a6dd9d7d8e9e5b0ac337962bc3664a9646df853 100644 --- a/deps/regex/CMakeLists.txt +++ b/deps/regex/CMakeLists.txt @@ -1,7 +1,7 @@ CMAKE_MINIMUM_REQUIRED(VERSION 2.8) PROJECT(TDengine) -IF (TD_WINDOWS_64) +IF (TD_WINDOWS_64 OR TD_WINDOWS_32) INCLUDE_DIRECTORIES(inc .) LIST(APPEND SRC regex.c) ADD_LIBRARY(regex ${SRC}) diff --git a/src/client/CMakeLists.txt b/src/client/CMakeLists.txt index 92d6b61eb2473c790c967a4a0091e233de84b8fa..d0336a752c2d9f8d1649569ee034a211e567fc1a 100644 --- a/src/client/CMakeLists.txt +++ b/src/client/CMakeLists.txt @@ -31,7 +31,7 @@ IF ((TD_LINUX_64) OR (TD_LINUX_32 AND TD_ARM)) MESSAGE(STATUS "build version ${VERSION_INFO}") SET_TARGET_PROPERTIES(taos PROPERTIES VERSION ${VERSION_INFO} SOVERSION 1) -ELSEIF (TD_WINDOWS_64) +ELSEIF (TD_WINDOWS_64 OR TD_WINDOWS_32) INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/jni/windows) INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/jni/windows/win32) INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/pthread) diff --git a/src/client/jni/com_taosdata_jdbc_TSDBJNIConnector.h b/src/client/jni/com_taosdata_jdbc_TSDBJNIConnector.h index 8dbe63d75a73dd18a15bc1da8f99c7b8db774eea..d976b2be3902e6f3dd014781ea1e32b42cd84cc3 100644 --- a/src/client/jni/com_taosdata_jdbc_TSDBJNIConnector.h +++ b/src/client/jni/com_taosdata_jdbc_TSDBJNIConnector.h @@ -142,8 +142,8 @@ JNIEXPORT jlong JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_subscribeImp * Method: consumeImp * Signature: (J)Lcom/taosdata/jdbc/TSDBResultSetRowData; */ -JNIEXPORT jobject JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_consumeImp - (JNIEnv *, jobject, jlong, jint); +JNIEXPORT jlong JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_consumeImp + (JNIEnv *, jobject, jlong); /* * Class: com_taosdata_jdbc_TSDBJNIConnector diff --git a/src/client/src/TSDBJNIConnector.c b/src/client/src/TSDBJNIConnector.c index 228403c79d318d922f5571a9663b3c97bbffbbc8..27b28c8f33a6ce770f17e7a2f881ecb8ee77bd27 100644 --- a/src/client/src/TSDBJNIConnector.c +++ b/src/client/src/TSDBJNIConnector.c @@ -606,7 +606,7 @@ static jobject convert_one_row(JNIEnv *env, TAOS_ROW row, TAOS_FIELD* fields, in return rowobj; } -JNIEXPORT jobject JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_consumeImp(JNIEnv *env, jobject jobj, jlong sub, jint timeout) { +JNIEXPORT jlong JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_consumeImp(JNIEnv *env, jobject jobj, jlong sub) { jniTrace("jobj:%p, in TSDBJNIConnector_consumeImp, sub:%ld", jobj, sub); jniGetGlobalMethod(env); @@ -616,38 +616,14 @@ JNIEXPORT jobject JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_consumeImp(JNI int64_t start = taosGetTimestampMs(); int count = 0; - while (true) { - TAOS_RES * res = taos_consume(tsub); - if (res == NULL) { - jniError("jobj:%p, tsub:%p, taos_consume returns NULL", jobj, tsub); - return NULL; - } - - TAOS_FIELD *fields = taos_fetch_fields(res); - int num_fields = taos_num_fields(res); - while (true) { - TAOS_ROW row = taos_fetch_row(res); - if (row == NULL) { - break; - } - jobject rowobj = convert_one_row(env, row, fields, num_fields); - (*env)->CallBooleanMethod(env, rows, g_arrayListAddFp, rowobj); - count++; - } + TAOS_RES *res = taos_consume(tsub); - if (count > 0) { - break; - } - if (timeout == -1) { - continue; - } - if (((int)(taosGetTimestampMs() - start)) >= timeout) { - jniTrace("jobj:%p, sub:%ld, timeout", jobj, sub); - break; - } + if (res == NULL) { + jniError("jobj:%p, tsub:%p, taos_consume returns NULL", jobj, tsub); + return NULL; } - return rows; + return res; } JNIEXPORT void JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_unsubscribeImp(JNIEnv *env, jobject jobj, jlong sub, jboolean keepProgress) { diff --git a/src/client/src/tscAsync.c b/src/client/src/tscAsync.c index 34ecc25c59507f836078164130440f89627e86e7..b3c9ef5f217db81e9f63824d4c6c6787ce760a87 100644 --- a/src/client/src/tscAsync.c +++ b/src/client/src/tscAsync.c @@ -26,6 +26,7 @@ #include "tutil.h" #include "tnote.h" +extern void tscSetNextLaunchTimer(SSqlStream *pStream, SSqlObj *pSql); static void tscProcessFetchRow(SSchedMsg *pMsg); static void tscAsyncQueryRowsForNextVnode(void *param, TAOS_RES *tres, int numOfRows); @@ -494,6 +495,11 @@ void tscMeterMetaCallBack(void *param, TAOS_RES *res, int code) { if ((pQueryInfo->type & TSDB_QUERY_TYPE_STABLE_SUBQUERY) == TSDB_QUERY_TYPE_STABLE_SUBQUERY) { SMeterMetaInfo* pMeterMetaInfo = tscGetMeterMetaInfoFromQueryInfo(pQueryInfo, 0); + if(pMeterMetaInfo->pMeterMeta == NULL) { + code = tscGetMeterMeta(pSql, pMeterMetaInfo); + assert(code == TSDB_CODE_SUCCESS); + } + assert(pMeterMetaInfo->pMeterMeta->numOfTags != 0 && pMeterMetaInfo->vnodeIndex >= 0 && pSql->param != NULL); SRetrieveSupport *trs = (SRetrieveSupport *)pSql->param; @@ -504,11 +510,6 @@ void tscMeterMetaCallBack(void *param, TAOS_RES *res, int code) { tscTrace("%p get metricMeta during super table query successfully", pSql); - code = tscGetMeterMeta(pSql, pMeterMetaInfo); - pRes->code = code; - - if (code == TSDB_CODE_ACTION_IN_PROGRESS) return; - code = tscGetMetricMeta(pSql, 0); pRes->code = code; @@ -553,13 +554,27 @@ void tscMeterMetaCallBack(void *param, TAOS_RES *res, int code) { } if (pSql->pStream) { - tscTrace("%p stream:%p meta is updated, start new query, command:%d", pSql, pSql->pStream, pSql->cmd.command); /* * NOTE: * transfer the sql function for super table query before get meter/metric meta, * since in callback functions, only tscProcessSql(pStream->pSql) is executed! */ SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex); + + SMeterMetaInfo *pMeterMetaInfo = tscGetMeterMetaInfoFromQueryInfo(pQueryInfo, 0); + if ((UTIL_METER_IS_SUPERTABLE(pMeterMetaInfo) + && ( pMeterMetaInfo->pMeterMeta == NULL + || pMeterMetaInfo->pMetricMeta == NULL + || pMeterMetaInfo->pMetricMeta->numOfMeters == 0 + || pMeterMetaInfo->pMetricMeta->numOfVnodes == 0)) + || (!(UTIL_METER_IS_SUPERTABLE(pMeterMetaInfo)) && (pMeterMetaInfo->pMeterMeta == NULL))) { + tscTrace("%p stream:%p meta is updated, but no table, clear meter meta and set next launch new query, command:%d", pSql, pSql->pStream, pSql->cmd.command); + tscClearMeterMetaInfo(pMeterMetaInfo, false); + tscSetNextLaunchTimer(pSql->pStream, pSql); + return; + } + + tscTrace("%p stream:%p meta is updated, start new query, command:%d", pSql, pSql->pStream, pSql->cmd.command); tscTansformSQLFunctionForSTableQuery(pQueryInfo); tscIncStreamExecutionCount(pSql->pStream); diff --git a/src/client/src/tscFunctionImpl.c b/src/client/src/tscFunctionImpl.c index fdadced66762d8bc2759b552c5aca49abe3a2583..36e9bec35dad45e44bd1a5d5444ad986fd73031b 100644 --- a/src/client/src/tscFunctionImpl.c +++ b/src/client/src/tscFunctionImpl.c @@ -3669,7 +3669,7 @@ int WCSPatternMatch(const wchar_t *patterStr, const wchar_t *str, size_t size, c wchar_t accept[3] = {towupper(c), towlower(c), 0}; while (1) { - size_t n = wcsspn(str, accept); + size_t n = wcscspn(str, accept); str += n; if (str[0] == 0 || (n >= size - 1)) { @@ -3678,7 +3678,7 @@ int WCSPatternMatch(const wchar_t *patterStr, const wchar_t *str, size_t size, c str++; - int32_t ret = WCSPatternMatch(&patterStr[i], str, wcslen(str), pInfo); + int32_t ret = WCSPatternMatch(&patterStr[i], str, twcslen(str), pInfo); if (ret != TSDB_PATTERN_NOMATCH) { return ret; } diff --git a/src/client/src/tscLocal.c b/src/client/src/tscLocal.c index 7208dc4d72269069a1e3aa5bcc6e8122954f09c1..091eba28a2175614112cb60cc571eac57bd270a3 100644 --- a/src/client/src/tscLocal.c +++ b/src/client/src/tscLocal.c @@ -434,6 +434,19 @@ static void tscProcessServStatus(SSqlObj *pSql) { if (pObj->pHb->res.code == TSDB_CODE_NETWORK_UNAVAIL) { pSql->res.code = TSDB_CODE_NETWORK_UNAVAIL; return; + } else { + int32_t* data = (int32_t*) pObj->pHb->res.data; + if (data != NULL) { + int32_t totalDnode = data[0]; + int32_t onlineDnode = data[1]; + assert(onlineDnode <= totalDnode); + + if (onlineDnode < totalDnode) { + pSql->res.code = TSDB_CODE_NETWORK_UNAVAIL; + return; + } + } + } } else { if (pSql->res.code == TSDB_CODE_NETWORK_UNAVAIL) { diff --git a/src/client/src/tscParseInsert.c b/src/client/src/tscParseInsert.c index cb2ef50fb4f16487a04c82895b71e973fc64b9be..d47b1387100daa8df0c753346ea0479a2279fc62 100644 --- a/src/client/src/tscParseInsert.c +++ b/src/client/src/tscParseInsert.c @@ -1442,6 +1442,9 @@ static int tscInsertDataFromFile(SSqlObj *pSql, FILE *fp, char *tmpTokenBuf) { pTableDataBlock->size = sizeof(SShellSubmitBlock); pTableDataBlock->rowSize = pMeterMeta->rowSize; + code = tscAllocateMemIfNeed(pTableDataBlock, rowSize, &maxRows); + if (TSDB_CODE_SUCCESS != code) return -1; + numOfRows += pSql->res.numOfRows; pSql->res.numOfRows = 0; count = 0; diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index bfaae0595d50e52e5a7f2aa809b74a5c030f4e2b..fc05cdbda2b3fd918d1843d33387d2ded5d3b9d2 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -282,6 +282,7 @@ int32_t tscToSQLCmd(SSqlObj* pSql, struct SSqlInfo* pInfo) { const char* msg2 = "name too long"; SCreateDBInfo* pCreateDB = &(pInfo->pDCLInfo->dbOpt); + pCmd->existsCheck = pInfo->pDCLInfo->existsCheck; if (tscValidateName(&pCreateDB->dbname) != TSDB_CODE_SUCCESS) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1); } @@ -1949,6 +1950,7 @@ static int16_t doGetColumnIndex(SQueryInfo* pQueryInfo, int32_t index, SSQLToken if (strncasecmp(pSchema[i].name, pToken->z, pToken->n) == 0) { columnIndex = i; + break; } } @@ -2595,7 +2597,7 @@ static int32_t doExtractColumnFilterInfo(SQueryInfo* pQueryInfo, SColumnFilterIn tVariantDump(&pRight->val, (char*)pColumnFilter->pz, colType); - size_t len = wcslen((wchar_t*)pColumnFilter->pz); + size_t len = twcslen((wchar_t*)pColumnFilter->pz); pColumnFilter->len = len * TSDB_NCHAR_SIZE; } else { tVariantDump(&pRight->val, (char*)&pColumnFilter->lowerBndd, colType); @@ -4659,6 +4661,7 @@ int32_t parseLimitClause(SQueryInfo* pQueryInfo, int32_t clauseIndex, SQuerySQL* if (pMeterMetaInfo->pMeterMeta == NULL || pMetricMeta == NULL || pMetricMeta->numOfMeters == 0) { tscTrace("%p no table in metricmeta, no output result", pSql); pQueryInfo->command = TSDB_SQL_RETRIEVE_EMPTY_RESULT; + pSql->res.qhandle = 0x1; // to pass the qhandle check; } // keep original limitation value in globalLimit diff --git a/src/client/src/tscSQLParserImpl.c b/src/client/src/tscSQLParserImpl.c index 44001fb31267150326e051c6e56ad52f4ad432f0..8446b6608ff2937d9a4da71b35ece976af85eac6 100644 --- a/src/client/src/tscSQLParserImpl.c +++ b/src/client/src/tscSQLParserImpl.c @@ -48,6 +48,7 @@ int32_t tSQLParse(SSqlInfo *pSQLInfo, const char *pStr) { Parse(pParser, 0, t0, pSQLInfo); goto abort_parse; } + case TK_QUESTION: case TK_ILLEGAL: { snprintf(pSQLInfo->pzErrMsg, tListLen(pSQLInfo->pzErrMsg), "unrecognized token: \"%s\"", t0.z); pSQLInfo->valid = false; @@ -815,6 +816,7 @@ void setCreateDBSQL(SSqlInfo *pInfo, int32_t type, SSQLToken *pToken, SCreateDBI pInfo->pDCLInfo->dbOpt.dbname = *pToken; tTokenListAppend(pInfo->pDCLInfo, pIgExists); + pInfo->pDCLInfo->existsCheck = (pIgExists->n == 1); } void setCreateAcctSQL(SSqlInfo *pInfo, int32_t type, SSQLToken *pName, SSQLToken *pPwd, SCreateAcctSQL *pAcctInfo) { diff --git a/src/client/src/tscSecondaryMerge.c b/src/client/src/tscSecondaryMerge.c index f537e7d187cc3e8afb4ad65e4768cc1f6c59e02e..bacae4788f4b7e31fa90aba7f3a7ac9b930d9ef9 100644 --- a/src/client/src/tscSecondaryMerge.c +++ b/src/client/src/tscSecondaryMerge.c @@ -231,13 +231,13 @@ void tscCreateLocalReducer(tExtMemBuffer **pMemBuffer, int32_t numOfBuffer, tOrd idx += 1; } } - assert(idx >= pReducer->numOfBuffer); + if (idx == 0) { free(pReducer); return; } - pReducer->numOfBuffer = idx; + pReducer->numOfBuffer = idx; // the actual entries that has result for merge SCompareParam *param = malloc(sizeof(SCompareParam)); param->pLocalData = pReducer->pLocalDataSrc; diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index 8bff57c59badd6d89474eaa09b3403363d03f6de..12c76a1897245f74a95b63600dcff470241e06f7 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -106,12 +106,12 @@ static int32_t tscGetMgmtConnMaxRetryTimes() { return tscMgmtIpList.numOfIps * factor; } -void tscProcessHeartBeatRsp(void *param, TAOS_RES *tres, int code) { +int32_t tscProcessHeartBeatRsp(void *param, TAOS_RES *tres, int code) { STscObj *pObj = (STscObj *)param; - if (pObj == NULL) return; + if (pObj == NULL) return TSDB_CODE_APP_ERROR; if (pObj != pObj->signature) { tscError("heart beat msg, pObj:%p, signature:%p invalid", pObj, pObj->signature); - return; + return TSDB_CODE_APP_ERROR; } SSqlObj *pSql = pObj->pHb; @@ -128,11 +128,19 @@ void tscProcessHeartBeatRsp(void *param, TAOS_RES *tres, int code) { if (pRsp->queryId) tscKillQuery(pObj, pRsp->queryId); if (pRsp->streamId) tscKillStream(pObj, pRsp->streamId); } + + if (pRes->data == NULL) { + pRes->data = calloc(2, sizeof(int32_t)); + } + + ((int32_t*)pRes->data)[0] = htonl(pRsp->totalDnodes); + ((int32_t*)pRes->data)[1] = htonl(pRsp->onlineDnodes); } else { tscTrace("heart beat failed, code:%d", code); } taosTmrReset(tscProcessActivityTimer, tsShellActivityTimer * 500, pObj, tscTmr, &pObj->pTimer); + return code; } void tscProcessActivityTimer(void *handle, void *tmrId) { @@ -710,15 +718,21 @@ int32_t tscLaunchJoinSubquery(SSqlObj *pSql, int16_t tableIndex, SJoinSubquerySu int doProcessSql(SSqlObj *pSql) { SSqlCmd *pCmd = &pSql->cmd; SSqlRes *pRes = &pSql->res; + int32_t code = TSDB_CODE_SUCCESS; void *asyncFp = pSql->fp; if (pCmd->command == TSDB_SQL_SELECT || pCmd->command == TSDB_SQL_FETCH || pCmd->command == TSDB_SQL_RETRIEVE || pCmd->command == TSDB_SQL_INSERT || pCmd->command == TSDB_SQL_CONNECT || pCmd->command == TSDB_SQL_HB || pCmd->command == TSDB_SQL_META || pCmd->command == TSDB_SQL_METRIC) { - tscBuildMsg[pCmd->command](pSql, NULL); + code = tscBuildMsg[pCmd->command](pSql, NULL); } - int32_t code = tscSendMsgToServer(pSql); + if (code != TSDB_CODE_SUCCESS) { + pRes->code = code; + return code; + } + + code = tscSendMsgToServer(pSql); if (asyncFp) { if (code != TSDB_CODE_SUCCESS) { @@ -1005,7 +1019,13 @@ int tscLaunchSTableSubqueries(SSqlObj *pSql) { SRetrieveSupport* pSupport = pSub->param; tscTrace("%p sub:%p launch subquery, orderOfSub:%d.", pSql, pSub, pSupport->subqueryIndex); - tscProcessSql(pSub); + int code = tscProcessSql(pSub); + if (code != TSDB_CODE_SUCCESS) { + tscLocalReducerEnvDestroy(pMemoryBuf, pDesc, pModel, numOfSubQueries); + doCleanupSubqueries(pSql, i, pState); + pRes->code = code; + return pRes->code; + } } return TSDB_CODE_SUCCESS; @@ -2713,7 +2733,7 @@ int tscBuildMultiMeterMetaMsg(SSqlObj *pSql, SSqlInfo *pInfo) { tscTrace("%p build load multi-metermeta msg completed, numOfMeters:%d, msg size:%d", pSql, pCmd->count, pCmd->payloadLen); - return pCmd->payloadLen; + return TSDB_CODE_SUCCESS; } static int32_t tscEstimateMetricMetaMsgSize(SSqlCmd *pCmd) { @@ -2940,7 +2960,7 @@ int tscBuildHeartBeatMsg(SSqlObj *pSql, SSqlInfo *pInfo) { pCmd->msgType = TSDB_MSG_TYPE_HEARTBEAT; assert(msgLen + minMsgSize() <= size); - return msgLen; + return TSDB_CODE_SUCCESS; } int tscProcessMeterMetaRsp(SSqlObj *pSql) { @@ -3700,7 +3720,8 @@ int tscGetMetricMeta(SSqlObj *pSql, int32_t clauseIndex) { for (int32_t i = 0; i < pQueryInfo->numOfTables; ++i) { SMeterMetaInfo *pMMInfo = tscGetMeterMetaInfoFromQueryInfo(pQueryInfo, i); - SMeterMeta *pMeterMeta = taosGetDataFromCache(tscCacheHandle, pMMInfo->name); + SMeterMeta *pMeterMeta = (SMeterMeta *)taosGetDataFromExists(tscCacheHandle, pQueryInfo->pMeterInfo[i]->pMeterMeta); + assert(pMeterMeta != NULL); tscAddMeterMetaInfo(pNewQueryInfo, pMMInfo->name, pMeterMeta, NULL, pMMInfo->numOfTags, pMMInfo->tagColumnIndex); } diff --git a/src/client/src/tscSql.c b/src/client/src/tscSql.c index 1421fe43ea8f945b0627b1ebff19ad0be5ed4f7d..f94c7fb13da25f2bfa9f5bc82768904956466a7d 100644 --- a/src/client/src/tscSql.c +++ b/src/client/src/tscSql.c @@ -321,8 +321,8 @@ TAOS_FIELD *taos_fetch_fields(TAOS_RES *res) { if (pSql == NULL || pSql->signature != pSql) return 0; SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(&pSql->cmd, 0); - if(NULL == pQueryInfo) { - return NULL; + if(NULL == pQueryInfo){ + return NULL; } return pQueryInfo->fieldsInfo.pFields; } @@ -756,6 +756,7 @@ TAOS_ROW taos_fetch_row(TAOS_RES *res) { } int taos_fetch_block(TAOS_RES *res, TAOS_ROW *rows) { +#if 0 SSqlObj *pSql = (SSqlObj *)res; SSqlCmd *pCmd = &pSql->cmd; SSqlRes *pRes = &pSql->res; @@ -795,6 +796,11 @@ int taos_fetch_block(TAOS_RES *res, TAOS_ROW *rows) { } return nRows; +#endif + + (*rows) = taos_fetch_row(res); + return ((*rows) != NULL)? 1:0; + } int taos_select_db(TAOS *taos, const char *db) { diff --git a/src/client/src/tscStream.c b/src/client/src/tscStream.c index 59e2f4f85c4c8cffff1dcc85ff2469d9d523eb20..804fb99c864e8af98fbe5f6c9cba26bbce2752cd 100644 --- a/src/client/src/tscStream.c +++ b/src/client/src/tscStream.c @@ -28,7 +28,7 @@ static void tscProcessStreamQueryCallback(void *param, TAOS_RES *tres, int numOfRows); static void tscProcessStreamRetrieveResult(void *param, TAOS_RES *res, int numOfRows); -static void tscSetNextLaunchTimer(SSqlStream *pStream, SSqlObj *pSql); +void tscSetNextLaunchTimer(SSqlStream *pStream, SSqlObj *pSql); static void tscSetRetryTimer(SSqlStream *pStream, SSqlObj *pSql, int64_t timer); static int64_t getDelayValueAfterTimewindowClosed(SSqlStream* pStream, int64_t launchDelay) { @@ -97,6 +97,18 @@ static void tscProcessStreamLaunchQuery(SSchedMsg *pMsg) { return; } + if ((UTIL_METER_IS_SUPERTABLE(pMeterMetaInfo) + && ( pMeterMetaInfo->pMeterMeta == NULL + || pMeterMetaInfo->pMetricMeta == NULL + || pMeterMetaInfo->pMetricMeta->numOfMeters == 0 + || pMeterMetaInfo->pMetricMeta->numOfVnodes == 0)) + || (!(UTIL_METER_IS_SUPERTABLE(pMeterMetaInfo)) && (pMeterMetaInfo->pMeterMeta == NULL))) { + tscTrace("%p no table in metricmeta, no launch query", pSql); + tscClearMeterMetaInfo(pMeterMetaInfo, false); + tscSetNextLaunchTimer(pStream, pSql); + return; + } + tscTrace("%p stream:%p start stream query on:%s", pSql, pStream, pMeterMetaInfo->name); tscProcessSql(pStream->pSql); @@ -323,7 +335,7 @@ static int64_t getLaunchTimeDelay(const SSqlStream* pStream) { } -static void tscSetNextLaunchTimer(SSqlStream *pStream, SSqlObj *pSql) { +void tscSetNextLaunchTimer(SSqlStream *pStream, SSqlObj *pSql) { int64_t timer = 0; SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(&pSql->cmd, 0); diff --git a/src/client/src/tscSyntaxtreefunction.c b/src/client/src/tscSyntaxtreefunction.c index 1d82b0f239572676c204025fe535704c192e4da6..3e263af982c474a478d3f825c761cd24dcdb249f 100644 --- a/src/client/src/tscSyntaxtreefunction.c +++ b/src/client/src/tscSyntaxtreefunction.c @@ -312,7 +312,7 @@ void calc_fn_i32_i32_sub(void *left, void *right, int32_t numLeft, int32_t numRi if (numLeft == numRight) { for (; i >= 0 && i < numRight; i += step, pOutput += 1) { if (isNull((char *)&(pLeft[i]), TSDB_DATA_TYPE_INT) || isNull((char *)&(pRight[i]), TSDB_DATA_TYPE_INT)) { - setNull((char *)&(pOutput[i]), TSDB_DATA_TYPE_DOUBLE, tDataTypeDesc[TSDB_DATA_TYPE_DOUBLE].nSize); + setNull((char *)&(pOutput), TSDB_DATA_TYPE_DOUBLE, tDataTypeDesc[TSDB_DATA_TYPE_DOUBLE].nSize); continue; } *pOutput = (double)pLeft[i] - pRight[i]; diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c index 28e063986c4c403eb34a14a753140264e688f9ca..35daacb3269ae76536fc7580d5e5b9f1be2530b7 100644 --- a/src/client/src/tscUtil.c +++ b/src/client/src/tscUtil.c @@ -457,9 +457,11 @@ void tscFreeSqlObjPartial(SSqlObj* pSql) { pCmd->command = 0; // pSql->sqlstr will be used by tscBuildQueryStreamDesc - pthread_mutex_lock(&pObj->mutex); - tfree(pSql->sqlstr); - pthread_mutex_unlock(&pObj->mutex); + if (pObj->signature == pObj) { + pthread_mutex_lock(&pObj->mutex); + tfree(pSql->sqlstr); + pthread_mutex_unlock(&pObj->mutex); + } tscFreeSqlResult(pSql); tfree(pSql->pSubs); @@ -2024,7 +2026,7 @@ SSqlObj* createSubqueryObj(SSqlObj* pSql, int16_t tableIndex, void (*fp)(), void } // create the fields info from the sql functions - SColumnList columnList = {.num = 1}; + SColumnList columnList = {.num = 0}; // for avg/last/first/histo.. query, the output type is binary not numeric data type for(int32_t k = 0; k < numOfOutputCols; ++k) { @@ -2079,7 +2081,14 @@ SSqlObj* createSubqueryObj(SSqlObj* pSql, int16_t tableIndex, void (*fp)(), void pMeterMetaInfo->tagColumnIndex); } - assert(pFinalInfo->pMeterMeta != NULL && pNewQueryInfo->numOfTables == 1); + if (pFinalInfo->pMeterMeta == NULL) { + tscError("%p new subquery failed for get pMeterMeta is NULL from cache", pSql); + tscFreeSqlObj(pNew); + return NULL; + } + + assert(pNewQueryInfo->numOfTables == 1); + if (UTIL_METER_IS_SUPERTABLE(pMeterMetaInfo)) { assert(pFinalInfo->pMetricMeta != NULL); } diff --git a/src/connector/jdbc/pom.xml b/src/connector/jdbc/pom.xml index 36f1a1010c372b319fab7fceaf13c9c44689dc6b..1db4ca878b74d66b5752d29b5cfcc3f53ae8492c 100755 --- a/src/connector/jdbc/pom.xml +++ b/src/connector/jdbc/pom.xml @@ -37,7 +37,7 @@ UTF-8 - 1.8 + 1.7 3.6.0 diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBConnection.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBConnection.java index 4640f6b446b6d285806408856489c5e950b37a81..062cb63cfd508800d2135494a8d45b6682a16fc1 100644 --- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBConnection.java +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBConnection.java @@ -84,6 +84,14 @@ public class TSDBConnection implements Connection { } } + public TSDBSubscribe createSubscribe() throws SQLException { + if (!this.connector.isClosed()) { + return new TSDBSubscribe(this.connector); + } else { + throw new SQLException(TSDBConstants.FixErrMsg(TSDBConstants.JNI_CONNECTION_NULL)); + } + } + public PreparedStatement prepareStatement(String sql) throws SQLException { if (!this.connector.isClosed()) { return new TSDBPreparedStatement(this.connector, sql); diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBJNIConnector.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBJNIConnector.java index 3adb601822567a6a7c515fa405801024e99a4609..ed428ea15faa0ca5ab38f93f9392c3fc9fa8f31d 100755 --- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBJNIConnector.java +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBJNIConnector.java @@ -22,8 +22,8 @@ public class TSDBJNIConnector { static volatile Boolean isInitialized = false; static { - System.loadLibrary("taos"); System.out.println("java.library.path:" + System.getProperty("java.library.path")); + System.loadLibrary("taos"); } /** @@ -261,31 +261,31 @@ public class TSDBJNIConnector { /** * Subscribe to a table in TSDB */ - public long subscribe(String host, String user, String password, String database, String table, long time, int period) { - return subscribeImp(host, user, password, database, table, time, period); + public long subscribe(String topic, String sql, boolean restart, int period) { + return subscribeImp(this.taos, restart, topic, sql, period); } - private native long subscribeImp(String host, String user, String password, String database, String table, long time, int period); + public native long subscribeImp(long connection, boolean restart, String topic, String sql, int period); /** * Consume a subscribed table */ - public TSDBResultSetRowData consume(long subscription) { + public long consume(long subscription) { return this.consumeImp(subscription); } - private native TSDBResultSetRowData consumeImp(long subscription); + private native long consumeImp(long subscription); /** * Unsubscribe a table * * @param subscription */ - public void unsubscribe(long subscription) { - unsubscribeImp(subscription); + public void unsubscribe(long subscription, boolean isKeep) { + unsubscribeImp(subscription, isKeep); } - private native void unsubscribeImp(long subscription); + private native void unsubscribeImp(long subscription, boolean isKeep); /** * Validate if a create table sql statement is correct without actually creating that table @@ -293,7 +293,7 @@ public class TSDBJNIConnector { public boolean validateCreateTableSql(String sql) { long connection = taos; int res = validateCreateTableSqlImp(connection, sql.getBytes()); - return res != 0 ? false : true; + return res == 0; } private native int validateCreateTableSqlImp(long connection, byte[] sqlBytes); diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBSubscribe.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBSubscribe.java new file mode 100644 index 0000000000000000000000000000000000000000..3b479aafc35a50e85e42d9cda1e93c2d68f8e115 --- /dev/null +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBSubscribe.java @@ -0,0 +1,185 @@ +/*************************************************************************** + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + *****************************************************************************/ +package com.taosdata.jdbc; + +import javax.management.OperationsException; +import java.sql.SQLException; +import java.util.Map; +import java.util.TimerTask; +import java.util.concurrent.*; + +public class TSDBSubscribe { + private TSDBJNIConnector connecter = null; + private static ScheduledExecutorService pool; + private static Map timerTaskMap = new ConcurrentHashMap<>(); + private static Map scheduledMap = new ConcurrentHashMap(); + + private static class TimerInstance { + private static final ScheduledExecutorService instance = Executors.newScheduledThreadPool(1); + } + + public static ScheduledExecutorService getTimerInstance() { + return TimerInstance.instance; + } + + public TSDBSubscribe(TSDBJNIConnector connecter) throws SQLException { + if (null != connecter) { + this.connecter = connecter; + } else { + throw new SQLException(TSDBConstants.FixErrMsg(TSDBConstants.JNI_CONNECTION_NULL)); + } + } + + /** + * sync subscribe + * + * @param topic + * @param sql + * @param restart + * @param period + * @throws SQLException + */ + public long subscribe(String topic, String sql, boolean restart, int period) throws SQLException { + if (this.connecter.isClosed()) { + throw new SQLException(TSDBConstants.FixErrMsg(TSDBConstants.JNI_CONNECTION_NULL)); + } + if (period < 1000) { + throw new SQLException(TSDBConstants.WrapErrMsg(TSDBConstants.INVALID_VARIABLES)); + } + return this.connecter.subscribe(topic, sql, restart, period); + } + + /** + * async subscribe + * + * @param topic + * @param sql + * @param restart + * @param period + * @param callBack + * @throws SQLException + */ + public long subscribe(String topic, String sql, boolean restart, int period, TSDBSubscribeCallBack callBack) throws SQLException { + if (this.connecter.isClosed()) { + throw new SQLException(TSDBConstants.FixErrMsg(TSDBConstants.JNI_CONNECTION_NULL)); + } + final long subscription = this.connecter.subscribe(topic, sql, restart, period); + if (null != callBack) { + pool = getTimerInstance(); + + TSDBTimerTask timerTask = new TSDBTimerTask(subscription, callBack); + + timerTaskMap.put(subscription, timerTask); + + ScheduledFuture scheduledFuture = pool.scheduleAtFixedRate(timerTask, 1, 1000, TimeUnit.MILLISECONDS); + scheduledMap.put(subscription, scheduledFuture); + } + return subscription; + } + + public TSDBResultSet consume(long subscription) throws OperationsException, SQLException { + if (this.connecter.isClosed()) { + throw new SQLException(TSDBConstants.FixErrMsg(TSDBConstants.JNI_CONNECTION_NULL)); + } + if (0 == subscription) { + throw new OperationsException("Invalid use of consume"); + } + long resultSetPointer = this.connecter.consume(subscription); + + if (resultSetPointer == TSDBConstants.JNI_CONNECTION_NULL) { + throw new SQLException(TSDBConstants.FixErrMsg(TSDBConstants.JNI_CONNECTION_NULL)); + } else if (resultSetPointer == TSDBConstants.JNI_NULL_POINTER) { + return null; + } else { + return new TSDBResultSet(this.connecter, resultSetPointer); + } + } + + /** + * cancel subscribe + * + * @param subscription + * @param isKeep + * @throws SQLException + */ + public void unsubscribe(long subscription, boolean isKeep) throws SQLException { + if (this.connecter.isClosed()) { + throw new SQLException(TSDBConstants.FixErrMsg(TSDBConstants.JNI_CONNECTION_NULL)); + } + + if (null != timerTaskMap.get(subscription)) { + synchronized (timerTaskMap.get(subscription)) { + while (1 == timerTaskMap.get(subscription).getState()) { + try { + Thread.sleep(10); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + timerTaskMap.get(subscription).setState(2); + if (!timerTaskMap.isEmpty() && timerTaskMap.containsKey(subscription)) { + timerTaskMap.get(subscription).cancel(); + timerTaskMap.remove(subscription); + scheduledMap.get(subscription).cancel(false); + scheduledMap.remove(subscription); + } + this.connecter.unsubscribe(subscription, isKeep); + } + } else { + this.connecter.unsubscribe(subscription, isKeep); + } + } + + class TSDBTimerTask extends TimerTask { + private long subscription; + private TSDBSubscribeCallBack callBack; + // 0: not running 1: running 2: cancel + private int state = 0; + + public TSDBTimerTask(long subscription, TSDBSubscribeCallBack callBack) { + this.subscription = subscription; + this.callBack = callBack; + } + + public int getState() { + return this.state; + } + + public void setState(int state) { + this.state = state; + } + + @Override + public void run() { + synchronized (this) { + if (2 == state) { + return; + } + + state = 1; + + try { + TSDBResultSet resultSet = consume(subscription); + callBack.invoke(resultSet); + } catch (Exception e) { + this.cancel(); + throw new RuntimeException(e); + } + state = 0; + } + } + } +} + diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBSubscribeCallBack.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBSubscribeCallBack.java new file mode 100644 index 0000000000000000000000000000000000000000..c1b9b02fa85f8e2197f41d75f8f0c25b3f4c416c --- /dev/null +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBSubscribeCallBack.java @@ -0,0 +1,19 @@ +/*************************************************************************** + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + *****************************************************************************/ +package com.taosdata.jdbc; + +public interface TSDBSubscribeCallBack { + void invoke(TSDBResultSet resultSet); +} diff --git a/src/connector/jdbc/src/test/java/TestAsyncTSDBSubscribe.java b/src/connector/jdbc/src/test/java/TestAsyncTSDBSubscribe.java new file mode 100644 index 0000000000000000000000000000000000000000..5b2b6367ec5fe7cb46f0514b59931d8942f9bc74 --- /dev/null +++ b/src/connector/jdbc/src/test/java/TestAsyncTSDBSubscribe.java @@ -0,0 +1,83 @@ +import com.taosdata.jdbc.*; +import org.apache.commons.lang3.StringUtils; + +import java.sql.Connection; +import java.sql.DriverManager; +import java.util.Properties; + +public class TestAsyncTSDBSubscribe { + public static void main(String[] args) { + String usage = "java -cp taos-jdbcdriver-1.0.3_dev-dist.jar com.taosdata.jdbc.TSDBSubscribe -db dbName -topic topicName " + + "-tname tableName -h host"; + if (args.length < 2) { + System.err.println(usage); + return; + } + + String dbName = ""; + String tName = ""; + String host = "localhost"; + String topic = ""; + for (int i = 0; i < args.length; i++) { + if ("-db".equalsIgnoreCase(args[i]) && i < args.length - 1) { + dbName = args[++i]; + } + if ("-tname".equalsIgnoreCase(args[i]) && i < args.length - 1) { + tName = args[++i]; + } + if ("-h".equalsIgnoreCase(args[i]) && i < args.length - 1) { + host = args[++i]; + } + if ("-topic".equalsIgnoreCase(args[i]) && i < args.length - 1) { + topic = args[++i]; + } + } + if (StringUtils.isEmpty(dbName) || StringUtils.isEmpty(tName) || StringUtils.isEmpty(topic)) { + System.err.println(usage); + return; + } + + Connection connection = null; + TSDBSubscribe subscribe = null; + long subscribId = 0; + try { + Class.forName("com.taosdata.jdbc.TSDBDriver"); + Properties properties = new Properties(); + properties.setProperty(TSDBDriver.PROPERTY_KEY_HOST, host); + connection = DriverManager.getConnection("jdbc:TAOS://" + host + ":0/" + dbName + "?user=root&password=taosdata", properties); + String rawSql = "select * from " + tName + ";"; + subscribe = ((TSDBConnection) connection).createSubscribe(); + subscribId = subscribe.subscribe(topic, rawSql, false, 1000, new CallBack("first")); + long subscribId2 = subscribe.subscribe("test", rawSql, false, 1000, new CallBack("second")); + int a = 0; + Thread.sleep(2000); + subscribe.unsubscribe(subscribId, true); + System.err.println("cancel subscribe"); + } catch (Exception e) { + e.printStackTrace(); + } + } + + private static class CallBack implements TSDBSubscribeCallBack { + private String name = ""; + + public CallBack(String name) { + this.name = name; + } + + @Override + public void invoke(TSDBResultSet resultSet) { + try { + while (null !=resultSet && resultSet.next()) { + System.out.print("callback_" + name + ": "); + for (int i = 1; i <= resultSet.getMetaData().getColumnCount(); i++) { + System.out.printf(i + ": " + resultSet.getString(i) + "\t"); + } + System.out.println(); + } + } catch (Exception e) { + e.printStackTrace(); + } + } + } +} diff --git a/src/connector/jdbc/src/test/java/TestPreparedStatement.java b/src/connector/jdbc/src/test/java/TestPreparedStatement.java index 2e2cc0ede6ae88c9b43c4890894378f13a3cc14a..fce2bd1428eead684a7524226c5f16d3e4cc7881 100644 --- a/src/connector/jdbc/src/test/java/TestPreparedStatement.java +++ b/src/connector/jdbc/src/test/java/TestPreparedStatement.java @@ -11,8 +11,14 @@ public class TestPreparedStatement { Class.forName("com.taosdata.jdbc.TSDBDriver"); Properties properties = new Properties(); properties.setProperty(TSDBDriver.PROPERTY_KEY_HOST, "192.168.1.117"); - Connection connection = DriverManager.getConnection("jdbc:TAOS://192.168.1.117:0/?user=root&password=taosdata", properties); + Connection connection = DriverManager.getConnection("jdbc:TAOS://10.211.55.3:0/log?user=root&password=taosdata", properties); + String createSql = "create table t (ts timestamp, speed int);"; + Statement statement = connection.createStatement(); + statement.executeQuery(createSql); String rawSql = "SELECT ts, c1 FROM (select c1, ts from db.tb1) SUB_QRY"; + if (1 < 2) { + return; + } // String[] params = new String[]{"ts", "c1"}; PreparedStatement pstmt = (TSDBPreparedStatement) connection.prepareStatement(rawSql); ResultSet resSet = pstmt.executeQuery(); diff --git a/src/connector/jdbc/src/test/java/TestTSDBResultSetRowData.java b/src/connector/jdbc/src/test/java/TestTSDBResultSetRowData.java new file mode 100644 index 0000000000000000000000000000000000000000..62d4c2ff6118b19fd7559bf8b23e2e3529339c18 --- /dev/null +++ b/src/connector/jdbc/src/test/java/TestTSDBResultSetRowData.java @@ -0,0 +1,35 @@ +import com.taosdata.jdbc.ColumnMetaData; +import com.taosdata.jdbc.DatabaseMetaDataResultSet; +import com.taosdata.jdbc.TSDBResultSetRowData; + +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.List; + +public class TestTSDBResultSetRowData { + public static void main(String[] args) throws SQLException { + DatabaseMetaDataResultSet resultSet = new DatabaseMetaDataResultSet(); + List columnMetaDataList = new ArrayList(1); + ColumnMetaData colMetaData = new ColumnMetaData(); + colMetaData.setColIndex(0); + colMetaData.setColName("TABLE_TYPE"); + colMetaData.setColSize(10); + colMetaData.setColType(8); + columnMetaDataList.add(colMetaData); + + List rowDataList = new ArrayList(2); + TSDBResultSetRowData rowData = new TSDBResultSetRowData(2); + rowData.setString(0, "TABLE"); + rowDataList.add(rowData); + rowData = new TSDBResultSetRowData(2); + rowData.setString(0, "STABLE"); + rowDataList.add(rowData); + + resultSet.setColumnMetaDataList(columnMetaDataList); + resultSet.setRowDataList(rowDataList); + + while (resultSet.next()) { + System.out.println(resultSet.getString(1)); + } + } +} diff --git a/src/connector/jdbc/src/test/java/TestTSDBSubscribe.java b/src/connector/jdbc/src/test/java/TestTSDBSubscribe.java new file mode 100644 index 0000000000000000000000000000000000000000..f12924c8a61a3dbeeec56f7d2c712472d771e8e7 --- /dev/null +++ b/src/connector/jdbc/src/test/java/TestTSDBSubscribe.java @@ -0,0 +1,83 @@ +import com.taosdata.jdbc.TSDBConnection; +import com.taosdata.jdbc.TSDBDriver; +import com.taosdata.jdbc.TSDBResultSet; +import com.taosdata.jdbc.TSDBSubscribe; +import org.apache.commons.lang3.StringUtils; + +import java.sql.Connection; +import java.sql.DriverManager; +import java.util.Properties; + +public class TestTSDBSubscribe { + public static void main(String[] args) throws Exception { + String usage = "java -cp taos-jdbcdriver-1.0.3_dev-dist.jar com.taosdata.jdbc.TSDBSubscribe -db dbName " + + "-topic topicName -tname tableName -h host"; + if (args.length < 2) { + System.err.println(usage); + return; + } + + String dbName = ""; + String tName = ""; + String host = "localhost"; + String topic = ""; + for (int i = 0; i < args.length; i++) { + if ("-db".equalsIgnoreCase(args[i]) && i < args.length - 1) { + dbName = args[++i]; + } + if ("-tname".equalsIgnoreCase(args[i]) && i < args.length - 1) { + tName = args[++i]; + } + if ("-h".equalsIgnoreCase(args[i]) && i < args.length - 1) { + host = args[++i]; + } + if ("-topic".equalsIgnoreCase(args[i]) && i < args.length - 1) { + topic = args[++i]; + } + } + if (StringUtils.isEmpty(dbName) || StringUtils.isEmpty(tName) || StringUtils.isEmpty(topic)) { + System.err.println(usage); + return; + } + + Connection connection = null; + TSDBSubscribe subscribe = null; + long subscribId = 0; + try { + Class.forName("com.taosdata.jdbc.TSDBDriver"); + Properties properties = new Properties(); + properties.setProperty(TSDBDriver.PROPERTY_KEY_HOST, host); + connection = DriverManager.getConnection("jdbc:TAOS://" + host + ":0/" + dbName + "?user=root&password=taosdata" + , properties); + String rawSql = "select * from " + tName + ";"; + subscribe = ((TSDBConnection) connection).createSubscribe(); + subscribId = subscribe.subscribe(topic, rawSql, false, 1000); + int a = 0; + while (true) { + Thread.sleep(900); + TSDBResultSet resSet = subscribe.consume(subscribId); + + while (resSet.next()) { + for (int i = 1; i <= resSet.getMetaData().getColumnCount(); i++) { + System.out.printf(i + ": " + resSet.getString(i) + "\t"); + } + System.out.println("\n======" + a + "=========="); + } + + a++; + if (a >= 10) { + break; + } + } + } catch (Exception e) { + e.printStackTrace(); + } finally { + if (null != subscribe && 0 != subscribId) { + subscribe.unsubscribe(subscribId, true); + } + if (null != connection) { + connection.close(); + } + } + } +} diff --git a/src/connector/python/linux/python2/taos/cinterface.py b/src/connector/python/linux/python2/taos/cinterface.py index 505619436cc1ad5d01a4134aede29477c6f6ae48..d9460efb213bfd3eb35d114676af49d611b8dd96 100644 --- a/src/connector/python/linux/python2/taos/cinterface.py +++ b/src/connector/python/linux/python2/taos/cinterface.py @@ -316,6 +316,9 @@ class CTaosInterface(object): blocks = [None] * len(fields) for i in range(len(fields)): data = ctypes.cast(pblock, ctypes.POINTER(ctypes.c_void_p))[i] + if data == None: + blocks[i] = [None] * num_of_rows + continue if fields[i]['type'] not in _CONVERT_FUNC: raise DatabaseError("Invalid data type returned from database") diff --git a/src/connector/python/linux/python3/taos/cinterface.py b/src/connector/python/linux/python3/taos/cinterface.py index 7fcedc9fe9400cc8db007897906d4568c2eb234f..77001609b61d35c746ef2ef37702aa6fb1460106 100644 --- a/src/connector/python/linux/python3/taos/cinterface.py +++ b/src/connector/python/linux/python3/taos/cinterface.py @@ -316,6 +316,9 @@ class CTaosInterface(object): blocks = [None] * len(fields) for i in range(len(fields)): data = ctypes.cast(pblock, ctypes.POINTER(ctypes.c_void_p))[i] + if data == None: + blocks[i] = [None] * num_of_rows + continue if fields[i]['type'] not in _CONVERT_FUNC: raise DatabaseError("Invalid data type returned from database") diff --git a/src/inc/taoserror.h b/src/inc/taoserror.h index edf0ab24a169c0a24ab1f8e554e7c0282c92a47f..99e42377eef1135871b335d187fecda8c7d8b9cd 100644 --- a/src/inc/taoserror.h +++ b/src/inc/taoserror.h @@ -138,8 +138,9 @@ extern "C" { #define TSDB_CODE_FAILED_TO_LOCK_RESOURCES 117 #define TSDB_CODE_TABLE_ID_MISMATCH 118 #define TSDB_CODE_QUERY_CACHE_ERASED 119 +#define TSDB_CODE_AUTH_BANNED_PERIOD 120 -#define TSDB_CODE_MAX_ERROR_CODE 120 +#define TSDB_CODE_MAX_ERROR_CODE 121 #ifdef __cplusplus } diff --git a/src/inc/taosmsg.h b/src/inc/taosmsg.h index 56b5a020a641fc9187b2a47fdf25a94f9566b5cc..48ee48eae345a7a576b7aaa6eef1b2438a9ed39d 100644 --- a/src/inc/taosmsg.h +++ b/src/inc/taosmsg.h @@ -439,9 +439,15 @@ typedef struct SSqlFuncExprMsg { } SSqlFuncExprMsg; typedef struct SSqlBinaryExprInfo { - struct tSQLBinaryExpr *pBinExpr; /* for binary expression */ + union { + struct tSQLBinaryExpr *pBinExpr; /* for binary expression */ + int64_t resvSpace0; + }; int32_t numOfCols; /* binary expression involves the readed number of columns*/ - SColIndexEx * pReqColumns; /* source column list */ + union { + SColIndexEx * pReqColumns; /* source column list */ + int64_t resvSpace1; + }; } SSqlBinaryExprInfo; typedef struct SSqlFunctionExpr { @@ -481,8 +487,12 @@ typedef struct SColumnInfo { int16_t colId; int16_t type; int16_t bytes; - int16_t numOfFilters; - SColumnFilterInfo *filters; + int16_t numOfFilters; + union { + SColumnFilterInfo *filters; + int64_t resvSpace; + }; + } SColumnInfo; /* @@ -811,6 +821,8 @@ typedef struct { typedef struct { uint32_t queryId; uint32_t streamId; + uint32_t totalDnodes; + uint32_t onlineDnodes; char killConnection; SIpList ipList; } SHeartBeatRsp; diff --git a/src/inc/tglobalcfg.h b/src/inc/tglobalcfg.h index 35cf6a42443ef40135c3937867339c6634c32140..462d488a6ba0c5334f70cd07e2284131df0867f6 100644 --- a/src/inc/tglobalcfg.h +++ b/src/inc/tglobalcfg.h @@ -150,6 +150,7 @@ extern int tsHttpMaxThreads; extern int tsHttpEnableCompress; extern int tsHttpEnableRecordSql; extern int tsTelegrafUseFieldNum; +extern int tsMaxAuthRetry; extern int tsTscEnableRecordSql; extern int tsAnyIp; @@ -255,7 +256,7 @@ extern int tsGlobalConfigNum; extern char * tsCfgStatusStr[]; SGlobalConfig *tsGetConfigOption(const char *option); -#define TSDB_CFG_MAX_NUM 110 +#define TSDB_CFG_MAX_NUM 111 #define TSDB_CFG_PRINT_LEN 23 #define TSDB_CFG_OPTION_LEN 24 #define TSDB_CFG_VALUE_LEN 41 diff --git a/src/inc/trpc.h b/src/inc/trpc.h index 97a0c905f8c3584fee0de46b064f79a78db19f15..fb5cc20908a3b7ac708ad5983762f762bd606570 100644 --- a/src/inc/trpc.h +++ b/src/inc/trpc.h @@ -62,6 +62,7 @@ typedef struct { void (*efp)(int cid); // call back function to process not activated chann int (*afp)(char *meterId, char *spi, char *encrypt, uint8_t *secret, uint8_t *ckey); // call back to retrieve auth info + int (*ufp)(char *user, int32_t *failCount, int32_t *allowTime, bool opSet); // callback to update auth retry info } SRpcInit; typedef struct { diff --git a/src/inc/ttime.h b/src/inc/ttime.h index 34c241cbc0f22afc511660cee475c82d08466599..6302f1a9c67a4ac7e07fc55c1b58c59b7f905e56 100644 --- a/src/inc/ttime.h +++ b/src/inc/ttime.h @@ -44,6 +44,7 @@ int32_t getTimestampInUsFromStr(char* token, int32_t tokenlen, int64_t* ts); int32_t taosParseTime(char* timestr, int64_t* time, int32_t len, int32_t timePrec); void deltaToUtcInitOnce(); +int32_t taosTimeSecToString(time_t ts, char* outstr); #ifdef __cplusplus } #endif diff --git a/src/inc/tutil.h b/src/inc/tutil.h index b66da286973521c1e6cd29db2b2923cfc371be58..2333d8f6b310dc2f73f7ec51f2e801f7f391a91e 100644 --- a/src/inc/tutil.h +++ b/src/inc/tutil.h @@ -130,6 +130,7 @@ extern "C" { #define POW2(x) ((x) * (x)) +size_t twcslen(const wchar_t *wcs); int32_t strdequote(char *src); void strtrim(char *src); diff --git a/src/kit/shell/CMakeLists.txt b/src/kit/shell/CMakeLists.txt index 0f490c58b169284a8892e259513fd42ec8d2cc38..d3b9bbf61d0643150976fcb387495d692a78a255 100644 --- a/src/kit/shell/CMakeLists.txt +++ b/src/kit/shell/CMakeLists.txt @@ -19,7 +19,7 @@ IF ((TD_LINUX_64) OR (TD_LINUX_32 AND TD_ARM)) ENDIF () SET_TARGET_PROPERTIES(shell PROPERTIES OUTPUT_NAME taos) -ELSEIF (TD_WINDOWS_64) +ELSEIF (TD_WINDOWS_64 OR TD_WINDOWS_32) INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/pthread) INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/regex) LIST(APPEND SRC ./src/shellEngine.c) diff --git a/src/kit/shell/src/shellEngine.c b/src/kit/shell/src/shellEngine.c index 27a4aaaa0cd24d84b46003ec3195f130d3ee8c85..4874ceb29a6c5961d05176852ff97c542e97dd91 100644 --- a/src/kit/shell/src/shellEngine.c +++ b/src/kit/shell/src/shellEngine.c @@ -808,7 +808,8 @@ void shellGetGrantInfo(void *con) { if (code == TSDB_CODE_OPS_NOT_SUPPORT) { fprintf(stdout, "Server is Community Edition, version is %s\n\n", taos_get_server_info(con)); } else { - fprintf(stderr, "Failed to check Server Edition, Reason:%d:%s\n\n", taos_errno(con), taos_errstr(con)); + //fprintf(stderr, "Failed to check Server Edition, Reason:%d:%s\n\n", taos_errno(con), taos_errstr(con)); + fprintf(stdout, "Server is Enterprise Edition, version is %s\n\n", taos_get_server_info(con)); } return; } diff --git a/src/kit/taosdump/taosdump.c b/src/kit/taosdump/taosdump.c index 2b41c18407406b32a4495e4a264679c542dd4ddb..8368b1fb34c58b15f3d47c65d5a7667343f522ec 100644 --- a/src/kit/taosdump/taosdump.c +++ b/src/kit/taosdump/taosdump.c @@ -27,6 +27,7 @@ #include #include #include +#include #include "taos.h" #include "taosmsg.h" @@ -162,6 +163,7 @@ static struct argp_option options[] = { {"password", 'p', "PASSWORD", 0, "User password to connect to server. Default is taosdata.", 0}, {"port", 'P', "PORT", 0, "Port to connect", 0}, {"cversion", 'v', "CVERION", 0, "client version", 0}, + {"mysqlFlag", 'q', "MYSQLFLAG", 0, "mysqlFlag, Default is 0", 0}, // input/output file {"outpath", 'o', "OUTPATH", 0, "Output file path.", 1}, {"inpath", 'i', "INPATH", 0, "Input file path.", 1}, @@ -189,6 +191,7 @@ struct arguments { char *password; uint16_t port; char cversion[TSDB_FILENAME_LEN+1]; + uint16_t mysqlFlag; // output file char outpath[TSDB_FILENAME_LEN+1]; char inpath[TSDB_FILENAME_LEN+1]; @@ -236,6 +239,9 @@ static error_t parse_opt(int key, char *arg, struct argp_state *state) { case 'P': arguments->port = atoi(arg); break; + case 'q': + arguments->mysqlFlag = atoi(arg); + break; case 'v': if (wordexp(arg, &full_path, 0) != 0) { fprintf(stderr, "Invalid client vesion %s\n", arg); @@ -341,6 +347,7 @@ struct arguments tsArguments = { "taosdata", 0, "", + 0, // outpath and inpath "", "", @@ -384,7 +391,8 @@ int main(int argc, char *argv[]) { printf("user: %s\n", tsArguments.user); printf("password: %s\n", tsArguments.password); printf("port: %u\n", tsArguments.port); - printf("cversion: %s\n", tsArguments.cversion); + printf("cversion: %s\n", tsArguments.cversion); + printf("mysqlFlag: %d", tsArguments.mysqlFlag); printf("outpath: %s\n", tsArguments.outpath); printf("inpath: %s\n", tsArguments.inpath); printf("encode: %s\n", tsArguments.encode); @@ -918,6 +926,8 @@ void taosDumpCreateDbClause(SDbInfo *dbInfo, bool isDumpProperty, FILE *fp) { dbInfo->ablocks, dbInfo->tblocks, dbInfo->ctime, dbInfo->clog, dbInfo->comp); } + pstr += sprintf(pstr, ";"); + fprintf(fp, "%s\n\n", tmpCommand); free(tmpCommand); } @@ -1236,7 +1246,7 @@ void taosDumpCreateTableClause(STableDef *tableDes, int numOfCols, FILE *fp) { } } - pstr += sprintf(pstr, ")"); + pstr += sprintf(pstr, ");"); fprintf(fp, "%s\n", tmpBuf); @@ -1289,7 +1299,7 @@ void taosDumpCreateMTableClause(STableDef *tableDes, char *metric, int numOfCols /* } */ } - pstr += sprintf(pstr, ")"); + pstr += sprintf(pstr, ");"); fprintf(fp, "%s\n", tmpBuf); free(tmpBuf); @@ -1359,14 +1369,31 @@ int taosDumpTableData(FILE *fp, char *tbname, struct arguments *arguments, TAOS* return -1; } + char sqlStr[8] = "\0"; + if (arguments->mysqlFlag) { + sprintf(sqlStr, "INSERT"); + } else { + sprintf(sqlStr, "IMPORT"); + } + + int rowFlag = 0; count = 0; while ((row = taos_fetch_row(tmpResult)) != NULL) { pstr = tmpBuffer; if (count == 0) { - pstr += sprintf(pstr, "IMPORT INTO %s VALUES (", tbname); - } else { - pstr += sprintf(pstr, "("); + pstr += sprintf(pstr, "%s INTO %s VALUES (", sqlStr, tbname); + } else { + if (arguments->mysqlFlag) { + if (0 == rowFlag) { + pstr += sprintf(pstr, "("); + rowFlag++; + } else { + pstr += sprintf(pstr, ", ("); + } + } else { + pstr += sprintf(pstr, "("); + } } for (int col = 0; col < numFields; col++) { @@ -1410,12 +1437,22 @@ int taosDumpTableData(FILE *fp, char *tbname, struct arguments *arguments, TAOS* pstr += sprintf(pstr, "\'%s\'", tbuf); break; case TSDB_DATA_TYPE_TIMESTAMP: - pstr += sprintf(pstr, "%" PRId64 "", *(int64_t *)row[col]); + if (!arguments->mysqlFlag) { + pstr += sprintf(pstr, "%" PRId64 "", *(int64_t *)row[col]); + } else { + char buf[64] = "\0"; + int64_t ts = *((int64_t *)row[col]); + time_t tt = (time_t)(ts / 1000); + struct tm *ptm = localtime(&tt); + strftime(buf, 64, "%y-%m-%d %H:%M:%S", ptm); + pstr += sprintf(pstr, "\'%s.%03d\'", buf, (int)(ts % 1000)); + } break; default: break; } } + pstr += sprintf(pstr, ") "); totalRows++; @@ -1423,7 +1460,7 @@ int taosDumpTableData(FILE *fp, char *tbname, struct arguments *arguments, TAOS* fprintf(fp, "%s", tmpBuffer); if (count >= arguments->data_batch) { - fprintf(fp, "\n"); + fprintf(fp, ";\n"); count = 0; } //else { //fprintf(fp, "\\\n"); diff --git a/src/os/windows/CMakeLists.txt b/src/os/windows/CMakeLists.txt index dc60b736ea2ca1d223f1f9bbb4206195c63a08ff..6e8695a8a8a96b06144442856640d70ac577e7bc 100644 --- a/src/os/windows/CMakeLists.txt +++ b/src/os/windows/CMakeLists.txt @@ -1,7 +1,7 @@ CMAKE_MINIMUM_REQUIRED(VERSION 2.8) PROJECT(TDengine) -IF (TD_WINDOWS_64) +IF (TD_WINDOWS_64 OR TD_WINDOWS_32) INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/pthread) INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/inc) INCLUDE_DIRECTORIES(inc) diff --git a/src/os/windows/inc/os.h b/src/os/windows/inc/os.h index 3f957e8abd489abdbe7bb009b3d4fa47aadb4467..c26d1cd540f6fd13aec5a895e9ef9673684394d2 100644 --- a/src/os/windows/inc/os.h +++ b/src/os/windows/inc/os.h @@ -142,8 +142,11 @@ extern "C" { #define atomic_exchange_16(ptr, val) _InterlockedExchange16((short volatile*)(ptr), (short)(val)) #define atomic_exchange_32(ptr, val) _InterlockedExchange((long volatile*)(ptr), (long)(val)) #define atomic_exchange_64(ptr, val) _InterlockedExchange64((__int64 volatile*)(ptr), (__int64)(val)) -#define atomic_exchange_ptr(ptr, val) _InterlockedExchangePointer((void* volatile*)(ptr), (void*)(val)) - +#if (_MSC_VER == 1800) + #define atomic_exchange_ptr(ptr, val) InterlockedExchangePointer((void* volatile*)(ptr), (void*)(val)) +#else + #define atomic_exchange_ptr(ptr, val) _InterlockedExchangePointer((void* volatile*)(ptr), (void*)(val)) +#endif #ifdef _TD_GO_DLL_ #define atomic_val_compare_exchange_8 __sync_val_compare_and_swap #else diff --git a/src/os/windows/src/twindows.c b/src/os/windows/src/twindows.c index 30973165dfd03f6179f4fc36a83c4b4a32ed3e55..3f048cde7a6bdfb5aec1cef6c4dfdbf300a1f7c3 100644 --- a/src/os/windows/src/twindows.c +++ b/src/os/windows/src/twindows.c @@ -93,7 +93,11 @@ long interlocked_add_fetch_32(long volatile* ptr, long val) { } __int64 interlocked_add_fetch_64(__int64 volatile* ptr, __int64 val) { +#ifdef _WIN64 return _InterlockedExchangeAdd64(ptr, val) + val; +#else + return _InterlockedExchangeAdd(ptr, val) + val; +#endif } // and @@ -377,9 +381,29 @@ int fsendfile(FILE* out_file, FILE* in_file, int64_t* offset, int32_t count) { return writeLen; } +unsigned char _MyBitScanForward64(unsigned long *ret, uint64_t x) { + unsigned long x0 = (unsigned long)x, top, bottom; + _BitScanForward(&top, (unsigned long)(x >> 32)); + _BitScanForward(&bottom, x0); + *ret = x0 ? bottom : 32 + top; + return x != 0; +} + +unsigned char _MyBitScanReverse64(unsigned long *ret, uint64_t x) { + unsigned long x1 = (unsigned long)(x >> 32), top, bottom; + _BitScanReverse(&top, x1); + _BitScanReverse(&bottom, (unsigned long)x); + *ret = x1 ? top + 32 : bottom; + return x != 0; +} + int32_t BUILDIN_CLZL(uint64_t val) { unsigned long r = 0; +#ifdef _WIN64 _BitScanReverse64(&r, val); +#else + _MyBitScanReverse64(&r, val); +#endif return (int)(r >> 3); } @@ -391,7 +415,11 @@ int32_t BUILDIN_CLZ(uint32_t val) { int32_t BUILDIN_CTZL(uint64_t val) { unsigned long r = 0; +#ifdef _WIN64 _BitScanForward64(&r, val); +#else + _MyBitScanForward64(&r, val); +#endif return (int)(r >> 3); } diff --git a/src/rpc/CMakeLists.txt b/src/rpc/CMakeLists.txt index 8d54d5fda591acd790c4bfab751cec079762e561..e06b462d2c587dd87ffb248cb26ef4c1a00efa40 100644 --- a/src/rpc/CMakeLists.txt +++ b/src/rpc/CMakeLists.txt @@ -12,7 +12,7 @@ ELSEIF (TD_DARWIN_64) LIST(APPEND SRC ./src/trpc.c) LIST(APPEND SRC ./src/tstring.c) LIST(APPEND SRC ./src/tudp.c) -ELSEIF (TD_WINDOWS_64) +ELSEIF (TD_WINDOWS_64 OR TD_WINDOWS_32) INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/pthread) LIST(APPEND SRC ./src/thaship.c) LIST(APPEND SRC ./src/trpc.c) diff --git a/src/rpc/src/trpc.c b/src/rpc/src/trpc.c index 87506861b1a660fa7745012a45120942a2772ec4..c8f58db1fe9b442ced0e3c4634c60c66c17a0df8 100755 --- a/src/rpc/src/trpc.c +++ b/src/rpc/src/trpc.c @@ -30,6 +30,7 @@ #include "tudp.h" #include "tutil.h" #include "lz4.h" +#include "tglobalcfg.h" typedef struct _msg_node { struct _msg_node *next; @@ -101,6 +102,7 @@ typedef struct rpc_server { void *(*fp)(char *, void *ahandle, void *thandle); void (*efp)(int); // FP to report error int (*afp)(char *meterId, char *spi, char *encrypt, uint8_t *secret, uint8_t *ckey); // FP to retrieve auth info + int (*ufp)(char *user, int32_t *failCount, int32_t *allowTime, bool opSet); // FP to update auth fail retry info SRpcChann *channList; } STaosRpc; @@ -408,6 +410,7 @@ void *taosOpenRpc(SRpcInit *pRpc) { pServer->qhandle = pRpc->qhandle; pServer->efp = pRpc->efp; pServer->afp = pRpc->afp; + pServer->ufp = pRpc->ufp; int size = (int)sizeof(SRpcChann) * pRpc->numOfChanns; pServer->channList = (SRpcChann *)malloc((size_t)size); @@ -894,9 +897,11 @@ int taosProcessMsgHeader(STaosHeader *pHeader, SRpcConn **ppConn, STaosRpc *pSer // authentication STaosDigest *pDigest = (STaosDigest *)((char *)pHeader + dataLen - sizeof(STaosDigest)); - int32_t delta; + int32_t delta, authTime,failedCount,authAllowTime; delta = (int32_t)htonl(pDigest->timeStamp); delta -= (int32_t)taosGetTimestampSec(); + authTime = (int32_t)taosGetTimestampSec(); + if (abs(delta) > 900) { tWarn("%s cid:%d sid:%d id:%s, time diff:%d is too big, msg discarded pConn:%p, timestamp:%d", pServer->label, chann, sid, pConn->meterId, delta, pConn, htonl(pDigest->timeStamp)); @@ -904,10 +909,37 @@ int taosProcessMsgHeader(STaosHeader *pHeader, SRpcConn **ppConn, STaosRpc *pSer code = TSDB_CODE_INVALID_TIME_STAMP; goto _exit; } - + + if (pServer->ufp) { + int ret = (*pServer->ufp)(pHeader->meterId,&failedCount,&authAllowTime,false); + if (0 == ret) { + if (authTime < authAllowTime) { + char ipstr[24]; + tinet_ntoa(ipstr, ip); + char timestr[50]; + taosTimeSecToString((time_t)authAllowTime,timestr); + mLError("user:%s login from %s, authentication not allowed until %s", pHeader->meterId, ipstr,timestr); + tTrace("%s cid:%d sid:%d id:%s, auth not allowed because failed authentication exceeds max limit, msg discarded pConn:%p, until %s", pServer->label, chann, sid, + pConn->meterId, pConn, timestr); + code = TSDB_CODE_AUTH_BANNED_PERIOD; + goto _exit; + } + }else { + code = ret; + goto _exit; + } + } + if (taosAuthenticateMsg((uint8_t *)pHeader, dataLen - TSDB_AUTH_LEN, pDigest->auth, pConn->secret) < 0) { char ipstr[24]; tinet_ntoa(ipstr, ip); + failedCount++; + if (failedCount >= tsMaxAuthRetry) { + authAllowTime = authTime + 600;//ban the user for 600 seconds + failedCount = 0; + } + (*pServer->ufp)(pHeader->meterId,&failedCount,&authAllowTime,true); + mLError("user:%s login from %s, authentication failed", pHeader->meterId, ipstr); tError("%s cid:%d sid:%d id:%s, authentication failed, msg discarded pConn:%p", pServer->label, chann, sid, pConn->meterId, pConn); @@ -1152,9 +1184,9 @@ void *taosProcessDataFromPeer(char *data, int dataLen, uint32_t ip, uint16_t por taosTmrReset(taosProcessIdleTimer, pServer->idleTime, pConn, pChann->tmrCtrl, &pConn->pIdleTimer); } - if (code == TSDB_CODE_ALREADY_PROCESSED) { - tTrace("%s cid:%d sid:%d id:%s, %s wont be processed, source:0x%08x dest:0x%08x tranId:%d pConn:%p", pServer->label, - chann, sid, pHeader->meterId, taosMsg[pHeader->msgType], pHeader->sourceId, htonl(pHeader->destId), + if (code == TSDB_CODE_ALREADY_PROCESSED || code == TSDB_CODE_LAST_SESSION_NOT_FINISHED) { + tTrace("%s code:%d, cid:%d sid:%d id:%s, %s wont be processed, source:0x%08x dest:0x%08x tranId:%d pConn:%p", pServer->label, + code, chann, sid, pHeader->meterId, taosMsg[pHeader->msgType], pHeader->sourceId, htonl(pHeader->destId), pHeader->tranId, pConn); free(data); return pConn; diff --git a/src/rpc/src/tstring.c b/src/rpc/src/tstring.c index a254ceecfd1f6ce13b1cc30f8c0c87b6b8edfca9..827942b0e2d44191824e0c694125467715d685f3 100644 --- a/src/rpc/src/tstring.c +++ b/src/rpc/src/tstring.c @@ -244,4 +244,5 @@ char *tsError[] = {"success", "failed to lock resources", "table id/uid mismatch", "client query cache erased", // 119 + "too many authentication failed, try 10 minutes later", //120 }; diff --git a/src/sdb/src/sdbEngine.c b/src/sdb/src/sdbEngine.c index 2bdc64cdecd14f7951f2385a751999a71cd94770..e2bde1eb8b7588691fb1c986eca668bace340728 100644 --- a/src/sdb/src/sdbEngine.c +++ b/src/sdb/src/sdbEngine.c @@ -812,11 +812,10 @@ void sdbResetTable(SSdbTable *pTable) { SRowHead *rowHead = NULL; void * pMetaRow = NULL; int64_t oldId = pTable->id; - //TODO: check - //int oldNumOfRows = pTable->numOfRows; + int oldNumOfRows = pTable->numOfRows; if (sdbOpenSdbFile(pTable) < 0) return; - //pTable->numOfRows = oldNumOfRows; + pTable->numOfRows = oldNumOfRows; total_size = sizeof(SRowHead) + pTable->maxRowSize + sizeof(TSCKSUM); rowHead = (SRowHead *)malloc(total_size); diff --git a/src/system/detail/inc/mgmt.h b/src/system/detail/inc/mgmt.h index 24f9822f1616529801ef2eee8ce3ec3dc5fc6f5d..424b65a0dd3463909105dcabedfa137d3c9838e1 100644 --- a/src/system/detail/inc/mgmt.h +++ b/src/system/detail/inc/mgmt.h @@ -177,6 +177,8 @@ typedef struct _user_obj { char reserved[16]; char updateEnd[1]; struct _user_obj *prev, *next; + int32_t authAllowTime; + int32_t authFailCount; } SUserObj; typedef struct { @@ -272,6 +274,7 @@ int mgmtSendOneFreeVnodeMsg(SVnodeGid *pVnodeGid); int mgmtInitShell(); void mgmtCleanUpShell(); int mgmtRetriveUserAuthInfo(char *user, char *spi, char *encrypt, uint8_t *secret, uint8_t *ckey); +int mgmtGetSetUserAuthFailInfo(char *user, int32_t *failedCount, int32_t *allowTime, bool opSet); // acct API int mgmtInitAccts(); @@ -429,6 +432,8 @@ bool mgmtAddVnode(SVgObj *pVgroup, SDnodeObj *pSrcDnode, SDnodeObj *pDestDnode); void mgmtSetModuleInDnode(SDnodeObj *pDnode, int moduleType); int mgmtUnSetModuleInDnode(SDnodeObj *pDnode, int moduleType); +void mgmtGetDnodeOnlineNum(int32_t *totalDnodes, int32_t *onlineDnodes); + extern int (*mgmtGetMetaFp[])(SMeterMeta *pMeta, SShowObj *pShow, SConnObj *pConn); extern int (*mgmtRetrieveFp[])(SShowObj *pShow, char *data, int rows, SConnObj *pConn); diff --git a/src/system/detail/src/mgmtShell.c b/src/system/detail/src/mgmtShell.c index 2300cdaddcb9cac2ccc2c42b24e3b911c8db14d0..d99b729aa60d643510ec82bbe63d3e7053d485fc 100644 --- a/src/system/detail/src/mgmtShell.c +++ b/src/system/detail/src/mgmtShell.c @@ -87,6 +87,7 @@ int mgmtInitShell() { rpcInit.idleTime = tsShellActivityTimer * 2000; rpcInit.qhandle = mgmtQhandle; rpcInit.afp = mgmtRetriveUserAuthInfo; + rpcInit.ufp = mgmtGetSetUserAuthFailInfo; pShellConn = taosOpenRpc(&rpcInit); if (pShellConn == NULL) { @@ -1184,7 +1185,7 @@ int mgmtProcessHeartBeatMsg(char *cont, int contLen, SConnObj *pConn) { char * pStart, *pMsg; int msgLen; STaosRsp *pRsp; - + mgmtSaveQueryStreamList(cont, contLen, pConn); pStart = taosBuildRspMsgWithSize(pConn->thandle, TSDB_MSG_TYPE_HEARTBEAT_RSP, 128); @@ -1202,6 +1203,10 @@ int mgmtProcessHeartBeatMsg(char *cont, int contLen, SConnObj *pConn) { pConn->streamId = 0; pHBRsp->killConnection = pConn->killConnection; + mgmtGetDnodeOnlineNum(&pHBRsp->totalDnodes, &pHBRsp->onlineDnodes); + pHBRsp->totalDnodes = htonl(pHBRsp->totalDnodes); + pHBRsp->onlineDnodes = htonl(pHBRsp->onlineDnodes); + if (pConn->usePublicIp) { if (pSdbPublicIpList != NULL) { int size = pSdbPublicIpList->numOfIps * 4; @@ -1271,6 +1276,21 @@ int mgmtRetriveUserAuthInfo(char *user, char *spi, char *encrypt, uint8_t *secre return 0; } +int mgmtGetSetUserAuthFailInfo(char *user, int32_t *failCount, int32_t *allowTime, bool opSet) { + SUserObj *pUser = NULL; + + pUser = mgmtGetUser(user); + if (pUser == NULL) return TSDB_CODE_INVALID_USER; + if (opSet) { + pUser->authAllowTime = *allowTime; + pUser->authFailCount = *failCount; + }else { + *allowTime = pUser->authAllowTime; + *failCount = pUser->authFailCount; + } + return 0; +} + int mgmtProcessConnectMsg(char *pMsg, int msgLen, SConnObj *pConn) { STaosRsp * pRsp; SConnectRsp *pConnectRsp; diff --git a/src/system/detail/src/mgmtSupertableQuery.c b/src/system/detail/src/mgmtSupertableQuery.c index b0b73d3a761cb304de96fd3495ec6dc390b1c1f8..b1caacd3a40e5b7a84e63eb83211747da7ac1fd9 100644 --- a/src/system/detail/src/mgmtSupertableQuery.c +++ b/src/system/detail/src/mgmtSupertableQuery.c @@ -577,7 +577,7 @@ static int32_t compareWStrPatternComp(const void* pLeft, const void* pRight) { const wchar_t* pattern = pRight; const wchar_t* str = pLeft; - int32_t ret = WCSPatternMatch(pattern, str, wcslen(str), &pInfo); + int32_t ret = WCSPatternMatch(pattern, str, twcslen(str), &pInfo); return (ret == TSDB_PATTERN_MATCH) ? 0 : 1; } diff --git a/src/system/detail/src/vnodeFile.c b/src/system/detail/src/vnodeFile.c index d1ccbe625cd7e227acf3665e878a1b41c76ce9f1..30d0d96ac00ecdcdce75e86c57b09fce6cd35815 100644 --- a/src/system/detail/src/vnodeFile.c +++ b/src/system/detail/src/vnodeFile.c @@ -1452,7 +1452,8 @@ int vnodeForwardStartPosition(SQuery *pQuery, SCompBlock *pBlock, int32_t slotId int step = QUERY_IS_ASC_QUERY(pQuery) ? 1 : -1; if (pQuery->limit.offset > 0 && pQuery->numOfFilterCols == 0) { - int maxReads = QUERY_IS_ASC_QUERY(pQuery) ? pBlock->numOfPoints - pQuery->pos : pQuery->pos + 1; + assert(pBlock[slotIdx].numOfPoints > pQuery->pos); + int maxReads = QUERY_IS_ASC_QUERY(pQuery) ? pBlock[slotIdx].numOfPoints - pQuery->pos : pQuery->pos + 1; if (pQuery->limit.offset < maxReads) { // start position in current block if (QUERY_IS_ASC_QUERY(pQuery)) { diff --git a/src/system/detail/src/vnodeQueryImpl.c b/src/system/detail/src/vnodeQueryImpl.c index 8931e4d6605f737c710008cfd60bab32aecc297f..9c6230c7e7d202e79fb64a31795105358238734d 100644 --- a/src/system/detail/src/vnodeQueryImpl.c +++ b/src/system/detail/src/vnodeQueryImpl.c @@ -35,6 +35,8 @@ #include "vnodeQueryImpl.h" #include "vnodeStatus.h" +#include + enum { TS_JOIN_TS_EQUAL = 0, TS_JOIN_TS_NOT_EQUALS = 1, @@ -1529,7 +1531,7 @@ static STimeWindow getActiveTimeWindow(SWindowResInfo *pWindowResInfo, int64_t t w.ekey = w.skey + pQuery->intervalTime - 1; } - assert(ts >= w.skey && ts <= w.ekey && w.skey != 0); + assert(ts >= w.skey && ts <= w.ekey/* && w.skey != 0*/); return w; } @@ -1646,7 +1648,7 @@ static void doCheckQueryCompleted(SQueryRuntimeEnv *pRuntimeEnv, TSKEY lastKey, setQueryStatus(pQuery, QUERY_COMPLETED | QUERY_RESBUF_FULL); } else { // set the current index to be the last unclosed window int32_t i = 0; - int64_t skey = 0; + int64_t skey = INT64_MIN; for (i = 0; i < pWindowResInfo->size; ++i) { SWindowResult *pResult = &pWindowResInfo->pResult[i]; @@ -1668,7 +1670,7 @@ static void doCheckQueryCompleted(SQueryRuntimeEnv *pRuntimeEnv, TSKEY lastKey, } // all windows are closed, set the last one to be the skey - if (skey == 0) { + if (skey == INT64_MIN) { assert(i == pWindowResInfo->size); pWindowResInfo->curIndex = pWindowResInfo->size - 1; } else { @@ -1686,7 +1688,7 @@ static void doCheckQueryCompleted(SQueryRuntimeEnv *pRuntimeEnv, TSKEY lastKey, dTrace("QInfo:%p total window:%d, closed:%d", GET_QINFO_ADDR(pQuery), pWindowResInfo->size, n); } - assert(pWindowResInfo->prevSKey != 0); + assert(pWindowResInfo->prevSKey != INT64_MIN); } static int32_t getNumOfRowsInTimeWindow(SQuery *pQuery, SBlockInfo *pBlockInfo, TSKEY *pPrimaryColumn, int32_t startPos, @@ -2798,14 +2800,16 @@ static int32_t rowwiseApplyAllFunctions(SQueryRuntimeEnv *pRuntimeEnv, int32_t * break; } } - - // save the last accessed row of current data block for interpolation - int32_t index = GET_COL_DATA_POS(pQuery, lastIndex, step); - for(int32_t i = 0; i < pQuery->numOfCols; ++i) { - SColumnInfo* pColInfo = &pQuery->colList[i].data; - int32_t s = pColInfo->bytes * index; + + if (lastIndex >= 0) { + // save the last accessed row of current data block for interpolation + int32_t index = GET_COL_DATA_POS(pQuery, lastIndex, step); + for(int32_t i = 0; i < pQuery->numOfCols; ++i) { + SColumnInfo* pColInfo = &pQuery->colList[i].data; + int32_t s = pColInfo->bytes * index; - memcpy(pRuntimeEnv->lastRowInBlock[i], pRuntimeEnv->colDataBuffer[i]->data + s, pColInfo->bytes); + memcpy(pRuntimeEnv->lastRowInBlock[i], pRuntimeEnv->colDataBuffer[i]->data + s, pColInfo->bytes); + } } free(sasArray); @@ -3305,7 +3309,6 @@ static int64_t getOldestKey(int32_t numOfFiles, int64_t fileId, SVnodeCfg *pCfg) } bool isQueryKilled(SQuery *pQuery) { - return false; SQInfo *pQInfo = (SQInfo *)GET_QINFO_ADDR(pQuery); /* @@ -4019,16 +4022,7 @@ bool normalizedFirstQueryRange(bool dataInDisk, bool dataInCache, STableQuerySup *key = nextKey; } - // needs the data before the begin timestamp of query time window - if (nextKey != pQuery->skey) { - if (!pRuntimeEnv->hasTimeWindow) { - pQuery->skey = nextKey; // change the query skey - pQuery->lastKey = pQuery->skey; - } - return true; - } else { - return doGetQueryPos(nextKey, pSupporter, pPointInterpSupporter); - } + return doGetQueryPos(nextKey, pSupporter, pPointInterpSupporter); } // set no data in file @@ -4655,7 +4649,7 @@ static void doSetInterpVal(SQLFunctionCtx *pCtx, TSKEY ts, int16_t type, int32_t len = t + 1 + TSDB_KEYSIZE; pCtx->param[index].pz = calloc(1, len); } else if (type == TSDB_DATA_TYPE_NCHAR) { - t = wcslen((const wchar_t *)data); + t = twcslen((const wchar_t *)data); len = (t + 1) * TSDB_NCHAR_SIZE + TSDB_KEYSIZE; pCtx->param[index].pz = calloc(1, len); diff --git a/src/system/detail/src/vnodeRead.c b/src/system/detail/src/vnodeRead.c index 08b0b1eba6f2dcd18ae6d5c9d716404e7bff08a6..7a92fa235b59b3406bf66123acefd0f4206e207b 100644 --- a/src/system/detail/src/vnodeRead.c +++ b/src/system/detail/src/vnodeRead.c @@ -496,6 +496,9 @@ void vnodeDecRefCount(void *param) { assert(vnodeIsQInfoValid(pQInfo)); int32_t ref = atomic_sub_fetch_32(&pQInfo->refCount, 1); + if (ref < 0) { + return; // avoid two threads dec ref count + } assert(ref >= 0); dTrace("QInfo:%p decrease obj refcount, %d", pQInfo, ref); diff --git a/src/system/detail/src/vnodeStore.c b/src/system/detail/src/vnodeStore.c index 5949b1636d1e5d48991df1ed06f63ca354a79d9a..92e90ebe84c5407cedaf36b4a06443d89c120a98 100644 --- a/src/system/detail/src/vnodeStore.c +++ b/src/system/detail/src/vnodeStore.c @@ -24,6 +24,8 @@ #include "vnodeUtil.h" #include "vnodeStatus.h" +#include + int tsMaxVnode = -1; int tsOpenVnodes = 0; SVnodeObj *vnodeList = NULL; diff --git a/src/system/lite/src/mgmtShell.spec.c b/src/system/lite/src/mgmtShell.spec.c index a1d8e6a34a4e2cc2d7df7c1acc6cdf75a796fc1a..a95ecd2ee2dc6b1816251a2d602432aa7df40b59 100644 --- a/src/system/lite/src/mgmtShell.spec.c +++ b/src/system/lite/src/mgmtShell.spec.c @@ -45,4 +45,9 @@ int mgmtProcessDropAcctMsg(char *pMsg, int msgLen, SConnObj *pConn) { int mgmtProcessCreateAcctMsg(char *pMsg, int msgLen, SConnObj *pConn) { return taosSendSimpleRsp(pConn->thandle, TSDB_MSG_TYPE_CREATE_ACCT_RSP, TSDB_CODE_OPS_NOT_SUPPORT); +} + +void mgmtGetDnodeOnlineNum(int32_t *totalDnodes, int32_t *onlineDnodes) { + *totalDnodes = 1; + *onlineDnodes = 1; } \ No newline at end of file diff --git a/src/util/CMakeLists.txt b/src/util/CMakeLists.txt index d8f74f46f4ef47ea4184c8dc3e915fccd5034a4d..79304d976b26fe89dccc7bd8cef78bd3259853db 100644 --- a/src/util/CMakeLists.txt +++ b/src/util/CMakeLists.txt @@ -28,7 +28,7 @@ IF ((TD_LINUX_64) OR (TD_LINUX_32 AND TD_ARM)) MESSAGE(STATUS "Failed to find iconv, use default encoding method") ENDIF () ENDIF () -ELSEIF (TD_WINDOWS_64) +ELSEIF (TD_WINDOWS_64 OR TD_WINDOWS_32) ADD_DEFINITIONS(-DUSE_LIBICONV) INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/pthread) INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/iconv) diff --git a/src/util/src/tcache.c b/src/util/src/tcache.c index c042c0181f4cf0ed2525e4d57eacb9e68d328615..54c6ea039cfded934e2b9738ff450ff9772eb7ab 100644 --- a/src/util/src/tcache.c +++ b/src/util/src/tcache.c @@ -587,6 +587,8 @@ void *taosAddDataIntoCache(void *handle, char *key, char *pData, int dataSize, i "size:%" PRId64 " bytes, collision:%d", pNode->key, pNode, HASH_INDEX(pNode->hashVal, pObj->capacity), pNode->addTime, pNode->time, pObj->size, pObj->totalSize, pObj->statistics.numOfCollision); + } else { + pError("key:%s failed to added into cache, out of memory", key); } } else { // old data exists, update the node pNode = taosUpdateCacheImpl(pObj, pOldNode, key, keyLen, pData, dataSize, keepTime * 1000L); diff --git a/src/util/src/textbuffer.c b/src/util/src/textbuffer.c index 11f526cec32976865b79503bd82fc205176661bf..0d9d15b6afef66d8075245070fecd6f06509d296 100644 --- a/src/util/src/textbuffer.c +++ b/src/util/src/textbuffer.c @@ -41,11 +41,13 @@ void getTmpfilePath(const char *fileNamePrefix, char *dstPath) { char *tmpDir = "/tmp/"; #endif + int64_t ts = taosGetTimestampUs(); + strcpy(tmpPath, tmpDir); strcat(tmpPath, tdengineTmpFileNamePrefix); strcat(tmpPath, fileNamePrefix); - strcat(tmpPath, "-%llu-%u"); - snprintf(dstPath, MAX_TMPFILE_PATH_LENGTH, tmpPath, taosGetPthreadId(), atomic_add_fetch_32(&tmpFileSerialNum, 1)); + strcat(tmpPath, "-%d-%llu-%u-%llu"); + snprintf(dstPath, MAX_TMPFILE_PATH_LENGTH, tmpPath, getpid(), taosGetPthreadId(), atomic_add_fetch_32(&tmpFileSerialNum, 1), ts); } /* diff --git a/src/util/src/tglobalcfg.c b/src/util/src/tglobalcfg.c index 400bce32e755c0ab683ee0d54c1151219515c7b3..813c66241a820da886392db38775705544879774 100644 --- a/src/util/src/tglobalcfg.c +++ b/src/util/src/tglobalcfg.c @@ -134,6 +134,7 @@ int tsEnableHttpModule = 1; int tsEnableMonitorModule = 1; int tsRestRowLimit = 10240; int tsMaxSQLStringLen = TSDB_MAX_SQL_LEN; +int tsMaxAuthRetry = 5; // the maximum number of results for projection query on super table that are returned from // one virtual node, to order according to timestamp @@ -689,7 +690,11 @@ static void doInitGlobalConfig() { tsInitConfigOption(cfg++, "maxSQLLength", &tsMaxSQLStringLen, TSDB_CFG_VTYPE_INT, TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_CLIENT | TSDB_CFG_CTYPE_B_SHOW, TSDB_MAX_SQL_LEN, TSDB_MAX_ALLOWED_SQL_LEN, 0, TSDB_CFG_UTYPE_BYTE); - + + tsInitConfigOption(cfg++, "maxAuthRetryTime", &tsMaxAuthRetry, TSDB_CFG_VTYPE_INT, + TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_CLIENT | TSDB_CFG_CTYPE_B_SHOW, + 1, 10, 0, TSDB_CFG_UTYPE_BYTE); + tsInitConfigOption(cfg++, "maxNumOfOrderedRes", &tsMaxNumOfOrderedResults, TSDB_CFG_VTYPE_INT, TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_CLIENT | TSDB_CFG_CTYPE_B_SHOW, TSDB_MAX_SQL_LEN, TSDB_MAX_ALLOWED_SQL_LEN, 0, TSDB_CFG_UTYPE_NONE); diff --git a/src/util/src/tnote.c b/src/util/src/tnote.c index 7a133590d2a450d8e8b688bc63515c0ad9e81912..1a1b3a644de580b0fc26bb6aa459635a86ed5204 100644 --- a/src/util/src/tnote.c +++ b/src/util/src/tnote.c @@ -77,7 +77,7 @@ void taosUnLockNote(int fd, taosNoteInfo * pNote) void *taosThreadToOpenNewNote(void *param) { - char name[NOTE_FILE_NAME_LEN]; + char name[NOTE_FILE_NAME_LEN + 16]; taosNoteInfo * pNote = (taosNoteInfo *)param; pNote->taosNoteFlag ^= 1; @@ -170,7 +170,7 @@ void taosGetNoteName(char *fn, taosNoteInfo * pNote) int taosOpenNoteWithMaxLines(char *fn, int maxLines, int maxNoteNum, taosNoteInfo * pNote) { - char name[NOTE_FILE_NAME_LEN] = "\0"; + char name[NOTE_FILE_NAME_LEN + 16] = "\0"; struct stat notestat0, notestat1; int size; diff --git a/src/util/src/ttime.c b/src/util/src/ttime.c index 2656c6deb0461b1bcc794d6cd8698842c8c07386..9e0f92475e4420a147d8f822a54b7ee182d31a97 100644 --- a/src/util/src/ttime.c +++ b/src/util/src/ttime.c @@ -19,6 +19,7 @@ #include #include #include +#include #include "tsdb.h" #include "ttime.h" @@ -158,6 +159,19 @@ int32_t taosParseTime(char* timestr, int64_t* time, int32_t len, int32_t timePre } } +int32_t taosTimeSecToString(time_t ts,char* outstr) { + if (NULL == outstr) { + return 1; + } + struct tm *t; + t = localtime(&ts); + if (NULL == t) return 1; + + sprintf(outstr,"%4d-%02d-%02d %02d:%02d:%02d\n", t->tm_year + 1900, t->tm_mon + 1, t->tm_mday, t->tm_hour, t->tm_min, t->tm_sec); + return 0; + +} + char* forwardToTimeStringEnd(char* str) { int32_t i = 0; int32_t numOfSep = 0; diff --git a/src/util/src/ttypes.c b/src/util/src/ttypes.c index 754793180f326c889ee8ee38730bad84e14a5e92..77d156513284816c22b6b313bdea4ae7aaabdf9e 100644 --- a/src/util/src/ttypes.c +++ b/src/util/src/ttypes.c @@ -233,7 +233,7 @@ int32_t tVariantToString(tVariant *pVar, char *dst) { case TSDB_DATA_TYPE_NCHAR: { dst[0] = '\''; - taosUcs4ToMbs(pVar->wpz, (wcslen(pVar->wpz) + 1) * TSDB_NCHAR_SIZE, dst + 1); + taosUcs4ToMbs(pVar->wpz, (twcslen(pVar->wpz) + 1) * TSDB_NCHAR_SIZE, dst + 1); int32_t len = strlen(dst); dst[len] = '\''; dst[len + 1] = 0; @@ -466,7 +466,7 @@ static int32_t toNchar(tVariant *pVariant, char **pDest, int32_t *pDestSize) { } pVariant->wpz = pWStr; - *pDestSize = wcslen(pVariant->wpz); + *pDestSize = twcslen(pVariant->wpz); // shrink the allocate memory, no need to check here. char* tmp = realloc(pVariant->wpz, (*pDestSize + 1)*TSDB_NCHAR_SIZE); diff --git a/src/util/src/tutil.c b/src/util/src/tutil.c index 21d147d9473fb643df3bdc719aa19699619ea2d7..9236c2860d9e9f180f8c66340172a37b9d1e1cf9 100644 --- a/src/util/src/tutil.c +++ b/src/util/src/tutil.c @@ -27,6 +27,23 @@ #include "tlog.h" #include "taoserror.h" +size_t twcslen(const wchar_t *wcs) { + int *wstr = (int *)wcs; + if (NULL == wstr) { + return 0; + } + + size_t n = 0; + while (1) { + if (0 == *wstr++) { + break; + } + n++; + } + + return n; +} + int32_t strdequote(char *z) { if (z == NULL) { return 0; diff --git a/src/util/src/version.c b/src/util/src/version.c index 265fc7b7dc36ede8806b81eb3bfb1773f97d5840..ea41b3ace885b9d81a55577f5b6c71081269cffa 100644 --- a/src/util/src/version.c +++ b/src/util/src/version.c @@ -1,4 +1,4 @@ -char version[64] = "1.6.6.1"; +char version[64] = "1.6.5.9"; char compatible_version[64] = "1.6.0.0"; char gitinfo[128] = "0b5b412ef0ae2449ece538601a29b899b2b727b9"; char gitinfoOfInternal[128] = "8ae0d83a3610b9b4726373dd3073e4a8f444fb26"; diff --git a/tests/comparisonTest/dataGenerator/com/taosdata/generator/DataGenerator.java b/tests/comparisonTest/dataGenerator/com/taosdata/generator/DataGenerator.java index a7a1f7fc681ebdf33d5252dcb187b855029b372b..7bdc9411f67f0521217003e6dc22acc6581cd58d 100644 --- a/tests/comparisonTest/dataGenerator/com/taosdata/generator/DataGenerator.java +++ b/tests/comparisonTest/dataGenerator/com/taosdata/generator/DataGenerator.java @@ -108,7 +108,6 @@ public class DataGenerator { } private static void getDataInOneFile(String path, int rowsPerDevice, int num, int humidityDistRadius, int tempDistRadius) throws IOException { - DecimalFormat df = new DecimalFormat("0.0000"); long startTime = dataStartTime; FileWriter fw = new FileWriter(new File(path)); @@ -135,13 +134,13 @@ public class DataGenerator { for (int j = 0; j < rowsPerDevice; ++j) { int humidity = (int) humidityDataGen.next(); - double temp = tempDataGen.next(); + int temp = (int) tempDataGen.next(); int deviceGroup = deviceId % 100; StringBuffer sb = new StringBuffer(); sb.append(deviceId).append(" ").append(tagPrefix).append(deviceId).append(" ").append(deviceGroup) .append(" ").append(dataStartTime).append(" ").append(humidity).append(" ") - .append(df.format(temp)); + .append(temp); bw.write(sb.toString()); bw.write("\n"); diff --git a/tests/examples/JDBC/JDBCDemo/src/main/java/TSDBSyncSample.java b/tests/examples/JDBC/JDBCDemo/src/main/java/TSDBSyncSample.java index c093b604da6dc6f815272f99d7fc786dab87928b..d0c167e71460520a346b8b953808db5ae25be214 100644 --- a/tests/examples/JDBC/JDBCDemo/src/main/java/TSDBSyncSample.java +++ b/tests/examples/JDBC/JDBCDemo/src/main/java/TSDBSyncSample.java @@ -4,7 +4,7 @@ public class TSDBSyncSample { private static final String JDBC_PROTOCAL = "jdbc:TAOS://"; private static final String TSDB_DRIVER = "com.taosdata.jdbc.TSDBDriver"; - private String host = "127.0.0.1"; + private String host = "10.211.55.3"; private String user = "root"; private String password = "taosdata"; private int port = 0; diff --git a/tests/examples/JDBC/JDBCDemo/src/main/java/TestAsyncTSDBSubscribeSample.java b/tests/examples/JDBC/JDBCDemo/src/main/java/TestAsyncTSDBSubscribeSample.java new file mode 100644 index 0000000000000000000000000000000000000000..00da0be060ef55ef7d86f065ff69c6d2952b30e2 --- /dev/null +++ b/tests/examples/JDBC/JDBCDemo/src/main/java/TestAsyncTSDBSubscribeSample.java @@ -0,0 +1,58 @@ +import com.taosdata.jdbc.*; +import org.apache.commons.lang3.StringUtils; + +import java.sql.Connection; +import java.sql.DriverManager; +import java.util.Properties; + +public class TestAsyncTSDBSubscribeSample { + public static void main(String[] args) { + String dbName = "log"; + String tName = "dn_10_211_55_3"; + String host = "10.211.55.3"; + String topic = "test"; + + Connection connection = null; + TSDBSubscribe subscribe = null; + long subscribId = 0; + try { + Class.forName("com.taosdata.jdbc.TSDBDriver"); + Properties properties = new Properties(); + properties.setProperty(TSDBDriver.PROPERTY_KEY_HOST, host); + connection = DriverManager.getConnection("jdbc:TAOS://" + host + ":0/" + dbName + "?user=root&password=taosdata", properties); + String rawSql = "select * from " + tName + ";"; + subscribe = ((TSDBConnection) connection).createSubscribe(); + subscribId = subscribe.subscribe(topic, rawSql, false, 1000, new CallBack("first")); + long subscribId2 = subscribe.subscribe("test", rawSql, false, 1000, new CallBack("second")); + int a = 0; + Thread.sleep(2000); + subscribe.unsubscribe(subscribId, true); + System.err.println("cancel subscribe"); + } catch (Exception e) { + e.printStackTrace(); + } + } + + private static class CallBack implements TSDBSubscribeCallBack { + private String name = ""; + + public CallBack(String name) { + this.name = name; + } + + @Override + public void invoke(TSDBResultSet resultSet) { + try { + while (null !=resultSet && resultSet.next()) { + System.out.print("callback_" + name + ": "); + for (int i = 1; i <= resultSet.getMetaData().getColumnCount(); i++) { + System.out.printf(i + ": " + resultSet.getString(i) + "\t"); + } + System.out.println(); + } + } catch (Exception e) { + e.printStackTrace(); + } + } + } +} diff --git a/tests/examples/JDBC/JDBCDemo/src/main/java/TestTSDBSubscribeSample.java b/tests/examples/JDBC/JDBCDemo/src/main/java/TestTSDBSubscribeSample.java new file mode 100644 index 0000000000000000000000000000000000000000..389af33ed050d5e746af9a8047593fe1126bac0f --- /dev/null +++ b/tests/examples/JDBC/JDBCDemo/src/main/java/TestTSDBSubscribeSample.java @@ -0,0 +1,59 @@ +import com.taosdata.jdbc.TSDBConnection; +import com.taosdata.jdbc.TSDBDriver; +import com.taosdata.jdbc.TSDBResultSet; +import com.taosdata.jdbc.TSDBSubscribe; +import org.apache.commons.lang3.StringUtils; + +import java.sql.Connection; +import java.sql.DriverManager; +import java.util.Properties; + +public class TestTSDBSubscribeSample { + public static void main(String[] args) throws Exception { + // use log db + String dbName = "log"; + String tName = "dn_10_211_55_3"; + String host = "10.211.55.3"; + String topic = "test"; + + Connection connection = null; + TSDBSubscribe subscribe = null; + long subscribId = 0; + try { + Class.forName("com.taosdata.jdbc.TSDBDriver"); + Properties properties = new Properties(); + properties.setProperty(TSDBDriver.PROPERTY_KEY_HOST, host); + connection = DriverManager.getConnection("jdbc:TAOS://" + host + ":0/" + dbName + "?user=root&password=taosdata" + , properties); + String rawSql = "select * from " + tName + ";"; + subscribe = ((TSDBConnection) connection).createSubscribe(); + subscribId = subscribe.subscribe(topic, rawSql, false, 1000); + int a = 0; + while (true) { + Thread.sleep(1000); + TSDBResultSet resSet = subscribe.consume(subscribId); + + while (resSet.next()) { + for (int i = 1; i <= resSet.getMetaData().getColumnCount(); i++) { + System.out.printf(i + ": " + resSet.getString(i) + "\t"); + } + System.out.println("\n================"); + } + + a++; + if (a >= 10) { + break; + } + } + } catch (Exception e) { + e.printStackTrace(); + } finally { + if (null != subscribe && 0 != subscribId) { + subscribe.unsubscribe(subscribId, true); + } + if (null != connection) { + connection.close(); + } + } + } +} diff --git a/tests/examples/c/CMakeLists.txt b/tests/examples/c/CMakeLists.txt index 287fca7d410b88d240642a57ec194b3d0c686975..f97075fba8783c5cc72299d6274e83587d9e1fd0 100644 --- a/tests/examples/c/CMakeLists.txt +++ b/tests/examples/c/CMakeLists.txt @@ -1,6 +1,6 @@ PROJECT(TDengine) -IF (TD_WINDOWS_64) +IF (TD_WINDOWS_64 OR TD_WINDOWS_32) INCLUDE_DIRECTORIES(${TD_ROOT_DIR}/deps/pthread) ENDIF ()