Merge remote-tracking branch 'origin/develop' into feature/crash

src/client/src/TSDBJNIConnector.c

@@ -437,13 +437,8 @@ JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_getSchemaMetaData
  * @return
  */
 jstring jniFromNCharToByteArray(JNIEnv *env, char *nchar, int32_t maxBytes) {
-  int len = (int)strlen(nchar);
-  if (len > maxBytes) {  // no terminated symbol exists '\0'
-    len = maxBytes;
-  }
-
-  jbyteArray bytes = (*env)->NewByteArray(env, len);
-  (*env)->SetByteArrayRegion(env, bytes, 0, len, (jbyte *)nchar);
+  jbyteArray bytes = (*env)->NewByteArray(env, maxBytes);
+  (*env)->SetByteArrayRegion(env, bytes, 0, maxBytes, (jbyte *)nchar);
   return bytes;
 }
 
@@ -481,6 +476,8 @@ JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_fetchRowImp(JNIEn
     }
   }
 
+  int32_t* length = taos_fetch_lengths(result);
+
   char tmp[TSDB_MAX_BYTES_PER_ROW] = {0};
 
   for (int i = 0; i < num_fields; i++) {
@@ -515,15 +512,15 @@ JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_fetchRowImp(JNIEn
         (*env)->CallVoidMethod(env, rowobj, g_rowdataSetDoubleFp, i, (jdouble)dv);
       } break;
       case TSDB_DATA_TYPE_BINARY: {
-        strncpy(tmp, row[i], (size_t)fields[i].bytes);  // handle the case that terminated does not exist
+        memcpy(tmp, row[i], length[i]);  // handle the case that terminated does not exist
         (*env)->CallVoidMethod(env, rowobj, g_rowdataSetStringFp, i, (*env)->NewStringUTF(env, tmp));
 
-        memset(tmp, 0, (size_t)fields[i].bytes);
+        memset(tmp, 0, length[i]);
         break;
       }
       case TSDB_DATA_TYPE_NCHAR: {
         (*env)->CallVoidMethod(env, rowobj, g_rowdataSetByteArrayFp, i,
-                               jniFromNCharToByteArray(env, (char *)row[i], fields[i].bytes));
+                               jniFromNCharToByteArray(env, (char *)row[i], length[i]));
         break;
       }
       case TSDB_DATA_TYPE_TIMESTAMP:

src/common/inc/tdataformat.h

@@ -325,8 +325,6 @@ void   tdResetKVRowBuilder(SKVRowBuilder *pBuilder);
 SKVRow tdGetKVRowFromBuilder(SKVRowBuilder *pBuilder);
 
 static FORCE_INLINE int tdAddColToKVRow(SKVRowBuilder *pBuilder, int16_t colId, int8_t type, void *value) {
-  ASSERT(pBuilder->nCols == 0 || colId > pBuilder->pColIdx[pBuilder->nCols - 1].colId);
-
   if (pBuilder->nCols >= pBuilder->tCols) {
     pBuilder->tCols *= 2;
     pBuilder->pColIdx = (SColIdx *)realloc((void *)(pBuilder->pColIdx), sizeof(SColIdx) * pBuilder->tCols);

src/tsdb/src/tsdbFile.c

@@ -26,11 +26,13 @@
 
 const char *tsdbFileSuffix[] = {".head", ".data", ".last", ".stat", ".h", ".d", ".l", ".s"};
 
-static int  tsdbInitFile(SFile *pFile, STsdbRepo *pRepo, int fid, int type);
-static void tsdbDestroyFile(SFile *pFile);
-static int  compFGroup(const void *arg1, const void *arg2);
-static int  keyFGroupCompFunc(const void *key, const void *fgroup);
-static void tsdbInitFileGroup(SFileGroup *pFGroup, STsdbRepo *pRepo);
+static int   tsdbInitFile(SFile *pFile, STsdbRepo *pRepo, int fid, int type);
+static void  tsdbDestroyFile(SFile *pFile);
+static int   compFGroup(const void *arg1, const void *arg2);
+static int   keyFGroupCompFunc(const void *key, const void *fgroup);
+static void  tsdbInitFileGroup(SFileGroup *pFGroup, STsdbRepo *pRepo);
+static TSKEY tsdbGetCurrMinKey(int8_t precision, int32_t keep);
+static int   tsdbGetCurrMinFid(int8_t precision, int32_t keep, int32_t days);
 
 // ---------------- INTERNAL FUNCTIONS ----------------
 STsdbFileH *tsdbNewFileH(STsdbCfg *pCfg) {
@@ -79,9 +81,11 @@ int tsdbOpenFileH(STsdbRepo *pRepo) {
   int     vid = 0;
   regex_t regex1, regex2;
   int     code = 0;
+  char    fname[TSDB_FILENAME_LEN] = "\0";
 
   SFileGroup  fileGroup = {0};
   STsdbFileH *pFileH = pRepo->tsdbFileH;
+  STsdbCfg *  pCfg = &(pRepo->config);
 
   tDataDir = tsdbGetDataDirName(pRepo->rootDir);
   if (tDataDir == NULL) {
@@ -108,6 +112,8 @@ int tsdbOpenFileH(STsdbRepo *pRepo) {
     goto _err;
   }
 
+  int mfid = tsdbGetCurrMinFid(pCfg->precision, pCfg->keep, pCfg->daysPerFile);
+
   struct dirent *dp = NULL;
   while ((dp = readdir(dir)) != NULL) {
     if (strcmp(dp->d_name, ".") == 0 || strcmp(dp->d_name, "..") == 0) continue;
@@ -120,6 +126,14 @@ int tsdbOpenFileH(STsdbRepo *pRepo) {
         continue;
       }
 
+      if (fid < mfid) {
+        for (int type = 0; type < TSDB_FILE_TYPE_MAX; type++) {
+          tsdbGetDataFileName(pRepo->rootDir, pCfg->tsdbId, fid, type, fname);
+          (void)remove(fname);
+        }
+        continue;
+      }
+
       if (tsdbSearchFGroup(pFileH, fid, TD_EQ) != NULL) continue;
       memset((void *)(&fileGroup), 0, sizeof(SFileGroup));
       fileGroup.fileId = fid;
@@ -179,8 +193,18 @@ void tsdbCloseFileH(STsdbRepo *pRepo) {
 
 SFileGroup *tsdbCreateFGroupIfNeed(STsdbRepo *pRepo, char *dataDir, int fid) {
   STsdbFileH *pFileH = pRepo->tsdbFileH;
+  STsdbCfg *  pCfg = &(pRepo->config);
+
+  if (pFileH->nFGroups >= pFileH->maxFGroups) {
+    int mfid = tsdbGetCurrMinFid(pCfg->precision, pCfg->keep, pCfg->daysPerFile);
+    if (pFileH->pFGroup[0].fileId < mfid) {
+      pthread_rwlock_wrlock(&pFileH->fhlock);
+      tsdbRemoveFileGroup(pRepo, &(pFileH->pFGroup[0]));
+      pthread_rwlock_unlock(&pFileH->fhlock);
+    }
+  }
 
-  if (pFileH->nFGroups >= pFileH->maxFGroups) return NULL;
+  ASSERT(pFileH->nFGroups < pFileH->maxFGroups);
 
   SFileGroup  fGroup;
   SFileGroup *pFGroup = &fGroup;
@@ -342,8 +366,7 @@ void tsdbFitRetention(STsdbRepo *pRepo) {
   STsdbFileH *pFileH = pRepo->tsdbFileH;
   SFileGroup *pGroup = pFileH->pFGroup;
 
-  int mfid = (int)(TSDB_KEY_FILEID(taosGetTimestamp(pCfg->precision), pCfg->daysPerFile, pCfg->precision) -
-             TSDB_MAX_FILE(pCfg->keep, pCfg->daysPerFile));
+  int mfid = tsdbGetCurrMinFid(pCfg->precision, pCfg->keep, pCfg->daysPerFile);
 
   pthread_rwlock_wrlock(&(pFileH->fhlock));
 
@@ -547,3 +570,11 @@ static void tsdbInitFileGroup(SFileGroup *pFGroup, STsdbRepo *pRepo) {
     }
   }
 }
+
+static TSKEY tsdbGetCurrMinKey(int8_t precision, int32_t keep) {
+  return (TSKEY)(taosGetTimestamp(precision) - keep * tsMsPerDay[precision]);
+}
+
+static int tsdbGetCurrMinFid(int8_t precision, int32_t keep, int32_t days) {
+  return TSDB_KEY_FILEID(tsdbGetCurrMinKey(precision, keep), days, precision);
+}
\ No newline at end of file

tests/examples/JDBC/springbootdemo/pom.xml

@@ -63,7 +63,7 @@
         <dependency>
             <groupId>com.taosdata.jdbc</groupId>
             <artifactId>taos-jdbcdriver</artifactId>
-            <version>2.0.2</version>
+            <version>2.0.4</version>
         </dependency>
 
         <dependency>
@@ -76,6 +76,24 @@
     </dependencies>
 
     <build>
+        <resources>
+            <resource>
+                <directory>src/main/resources</directory>
+                <includes>
+                    <include>**/*.properties</include>
+                    <include>**/*.xml</include>
+                </includes>
+                <filtering>true</filtering>
+            </resource>
+            <resource>
+                <directory>src/main/java</directory>
+                <includes>
+                    <include>**/*.properties</include>
+                    <include>**/*.xml</include>
+                </includes>
+            </resource>
+
+        </resources>
         <plugins>
             <plugin>
                 <groupId>org.springframework.boot</groupId>

tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/jdbc/springbootdemo/controller/RainStationController.java

+package com.taosdata.jdbc.springbootdemo.controller;
+
+
+import com.taosdata.jdbc.springbootdemo.domain.Rainfall;
+import com.taosdata.jdbc.springbootdemo.service.RainStationService;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.web.bind.annotation.*;
+
+@RestController
+@RequestMapping("/rainstation")
+public class RainStationController {
+
+    @Autowired
+    private RainStationService service;
+
+    @GetMapping("/init")
+    public boolean init() {
+        service.init();
+        service.createTable();
+        return true;
+    }
+
+    @PostMapping("/insert")
+    public int insert(@RequestBody Rainfall rainfall){
+        return service.insert(rainfall);
+    }
+
+}

tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/jdbc/springbootdemo/controller/WeatherController.java

@@ -16,43 +16,47 @@ public class WeatherController {
 
     /**
      * create database and table
+     *
      * @return
      */
     @GetMapping("/init")
-    public boolean init(){
+    public boolean init() {
         return weatherService.init();
     }
 
     /**
      * Pagination Query
+     *
      * @param limit
      * @param offset
      * @return
      */
     @GetMapping("/{limit}/{offset}")
-    public List<Weather> queryWeather(@PathVariable Long limit, @PathVariable Long offset){
+    public List<Weather> queryWeather(@PathVariable Long limit, @PathVariable Long offset) {
         return weatherService.query(limit, offset);
     }
 
     /**
      * upload single weather info
+     *
      * @param temperature
      * @param humidity
      * @return
      */
     @PostMapping("/{temperature}/{humidity}")
-    public int saveWeather(@PathVariable int temperature, @PathVariable float humidity){
+    public int saveWeather(@PathVariable int temperature, @PathVariable float humidity) {
 
         return weatherService.save(temperature, humidity);
     }
 
     /**
      * upload multi weather info
+     *
      * @param weatherList
      * @return
      */
     @PostMapping("/batch")
-    public int batchSaveWeather(@RequestBody List<Weather> weatherList){
+    public int batchSaveWeather(@RequestBody List<Weather> weatherList) {
 
         return weatherService.save(weatherList);
     }

tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/jdbc/springbootdemo/dao/DatabaseMapper.java

+package com.taosdata.jdbc.springbootdemo.dao;
+
+import java.util.Map;
+
+public interface DatabaseMapper {
+
+    int createDatabase(String dbname);
+
+    int dropDatabase(String dbname);
+
+    int creatDatabaseWithParameters(Map<String,String> map);
+
+    int useDatabase(String dbname);
+
+}

tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/jdbc/springbootdemo/dao/DatabaseMapper.xml

+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN" "http://mybatis.org/dtd/mybatis-3-mapper.dtd">
+
+<mapper namespace="com.taosdata.jdbc.springbootdemo.dao.DatabaseMapper">
+
+    <update id="createDatabase" parameterType="java.lang.String">
+        create database if not exists ${dbname}
+    </update>
+
+    <update id="dropDatabase" parameterType="java.lang.String">
+        DROP database if exists ${dbname}
+    </update>
+    
+    
+    <update id="creatDatabaseWithParameters" parameterType="map">
+        CREATE database if not EXISTS ${dbname}
+        <if test="keep != null">
+            KEEP ${keep}
+        </if>
+        <if test="days != null">
+            DAYS ${days}
+        </if>
+        <if test="replica != null">
+            REPLICA ${replica}
+        </if>
+        <if test="cache != null">
+            cache ${cache}
+        </if>
+        <if test="blocks != null">
+            blocks ${blocks}
+        </if>
+        <if test="minrows != null">
+            minrows ${minrows}
+        </if>
+        <if test="maxrows != null">
+            maxrows ${maxrows}
+        </if>
+    </update>
+
+    <update id="useDatabase" parameterType="java.lang.String">
+        use ${dbname}
+    </update>
+
+</mapper>
\ No newline at end of file

tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/jdbc/springbootdemo/dao/RainfallMapper.java

+package com.taosdata.jdbc.springbootdemo.dao;
+
+import java.util.Map;
+
+public interface RainfallMapper {
+
+
+    int save(Map<String, Object> map);
+}

tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/jdbc/springbootdemo/dao/RainfallMapper.xml

+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN" "http://mybatis.org/dtd/mybatis-3-mapper.dtd">
+
+<mapper namespace="com.taosdata.jdbc.springbootdemo.dao.RainfallMapper">
+
+    <insert id="save" parameterType="map">
+        INSERT INTO ${table} using ${dbname}.${stable} tags(#{values.station_code}, #{values.station_name}) (ts, name, code, rainfall) values (#{values.ts}, #{values.name}, #{values.code}, #{values.rainfall})
+    </insert>
+
+
+</mapper>
\ No newline at end of file

tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/jdbc/springbootdemo/dao/TableMapper.java

+package com.taosdata.jdbc.springbootdemo.dao;
+
+import com.taosdata.jdbc.springbootdemo.domain.TableMetadata;
+
+public interface TableMapper {
+
+    boolean createSTable(TableMetadata tableMetadata);
+}
\ No newline at end of file

tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/jdbc/springbootdemo/dao/TableMapper.xml

+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN" "http://mybatis.org/dtd/mybatis-3-mapper.dtd">
+
+<mapper namespace="com.taosdata.jdbc.springbootdemo.dao.TableMapper">
+
+    <update id="createSTable" parameterType="com.taosdata.jdbc.springbootdemo.domain.TableMetadata">
+        create table if not exists ${dbname}.${tablename}
+        <foreach collection="fields" item="field" index="index" open="(" close=")" separator=",">
+            ${field.name} ${field.type}
+        </foreach>
+        TAGS
+        <foreach collection="tags" item="tag" index="index" open="(" close=")" separator=",">
+            ${tag.name} ${tag.type}
+        </foreach>
+    </update>
+
+    <update id="dropTable" parameterType="java.lang.String">
+        drop ${tablename}
+    </update>
+
+</mapper>
\ No newline at end of file

tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/jdbc/springbootdemo/domain/FieldMetadata.java

+package com.taosdata.jdbc.springbootdemo.domain;
+
+public class FieldMetadata {
+
+    private String name;
+    private String type;
+
+    public FieldMetadata(String name, String type) {
+        this.name = name;
+        this.type = type;
+    }
+
+    public String getName() {
+        return name;
+    }
+
+    public void setName(String name) {
+        this.name = name;
+    }
+
+    public String getType() {
+        return type;
+    }
+
+    public void setType(String type) {
+        this.type = type;
+    }
+}

tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/jdbc/springbootdemo/domain/Rainfall.java

+package com.taosdata.jdbc.springbootdemo.domain;
+
+import com.fasterxml.jackson.annotation.JsonFormat;
+
+import java.sql.Timestamp;
+
+public class Rainfall {
+
+    @JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss.SSS",timezone = "GMT+8")
+    private Timestamp ts;
+    private String name;
+    private String code;
+    private float rainfall;
+    private String station_code;
+    private String station_name;
+
+    public Timestamp getTs() {
+        return ts;
+    }
+
+    public void setTs(Timestamp ts) {
+        this.ts = ts;
+    }
+
+    public String getName() {
+        return name;
+    }
+
+    public void setName(String name) {
+        this.name = name;
+    }
+
+    public String getCode() {
+        return code;
+    }
+
+    public void setCode(String code) {
+        this.code = code;
+    }
+
+    public float getRainfall() {
+        return rainfall;
+    }
+
+    public void setRainfall(float rainfall) {
+        this.rainfall = rainfall;
+    }
+
+    public String getStation_code() {
+        return station_code;
+    }
+
+    public void setStation_code(String station_code) {
+        this.station_code = station_code;
+    }
+
+    public String getStation_name() {
+        return station_name;
+    }
+
+    public void setStation_name(String station_name) {
+        this.station_name = station_name;
+    }
+}

tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/jdbc/springbootdemo/domain/TableMetadata.java

+package com.taosdata.jdbc.springbootdemo.domain;
+
+import java.util.List;
+
+public class TableMetadata {
+
+    private String dbname;
+    private String tablename;
+    private List<FieldMetadata> fields;
+    private List<TagMetadata> tags;
+
+    public String getDbname() {
+        return dbname;
+    }
+
+    public void setDbname(String dbname) {
+        this.dbname = dbname;
+    }
+
+    public String getTablename() {
+        return tablename;
+    }
+
+    public void setTablename(String tablename) {
+        this.tablename = tablename;
+    }
+
+    public List<FieldMetadata> getFields() {
+        return fields;
+    }
+
+    public void setFields(List<FieldMetadata> fields) {
+        this.fields = fields;
+    }
+
+    public List<TagMetadata> getTags() {
+        return tags;
+    }
+
+    public void setTags(List<TagMetadata> tags) {
+        this.tags = tags;
+    }
+}

tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/jdbc/springbootdemo/domain/TagMetadata.java

+package com.taosdata.jdbc.springbootdemo.domain;
+
+public class TagMetadata {
+    private String name;
+    private String type;
+
+    public TagMetadata(String name, String type) {
+        this.name = name;
+        this.type = type;
+    }
+
+    public String getName() {
+        return name;
+    }
+
+    public void setName(String name) {
+        this.name = name;
+    }
+
+    public String getType() {
+        return type;
+    }
+
+    public void setType(String type) {
+        this.type = type;
+    }
+}

tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/jdbc/springbootdemo/domain/Weather.java

 package com.taosdata.jdbc.springbootdemo.domain;
 
+import com.fasterxml.jackson.annotation.JsonFormat;
+
 import java.sql.Timestamp;
 
 public class Weather {
 
+    @JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss.SSS",timezone = "GMT+8")
     private Timestamp ts;
 
     private int temperature;

tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/jdbc/springbootdemo/service/RainStationService.java

+package com.taosdata.jdbc.springbootdemo.service;
+
+import com.taosdata.jdbc.springbootdemo.dao.DatabaseMapper;
+import com.taosdata.jdbc.springbootdemo.dao.RainfallMapper;
+import com.taosdata.jdbc.springbootdemo.dao.TableMapper;
+import com.taosdata.jdbc.springbootdemo.domain.FieldMetadata;
+import com.taosdata.jdbc.springbootdemo.domain.Rainfall;
+import com.taosdata.jdbc.springbootdemo.domain.TableMetadata;
+import com.taosdata.jdbc.springbootdemo.domain.TagMetadata;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Service;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+@Service
+public class RainStationService {
+
+    @Autowired
+    private DatabaseMapper databaseMapper;
+    @Autowired
+    private TableMapper tableMapper;
+    @Autowired
+    private RainfallMapper rainfallMapper;
+
+    public boolean init() {
+        databaseMapper.dropDatabase("rainstation");
+
+        Map<String, String> map = new HashMap<>();
+        map.put("dbname", "rainstation");
+        map.put("keep", "36500");
+        map.put("days", "30");
+        map.put("blocks", "4");
+        databaseMapper.creatDatabaseWithParameters(map);
+
+        databaseMapper.useDatabase("rainstation");
+        return true;
+    }
+
+    public boolean createTable() {
+        TableMetadata tableMetadata = new TableMetadata();
+        tableMetadata.setDbname("rainstation");
+        tableMetadata.setTablename("monitoring");
+
+        List<FieldMetadata> fields = new ArrayList<>();
+        fields.add(new FieldMetadata("ts", "timestamp"));
+        fields.add(new FieldMetadata("name", "NCHAR(10)"));
+        fields.add(new FieldMetadata("code", " BINARY(8)"));
+        fields.add(new FieldMetadata("rainfall", "float"));
+        tableMetadata.setFields(fields);
+
+        List<TagMetadata> tags = new ArrayList<>();
+        tags.add(new TagMetadata("station_code", "BINARY(8)"));
+        tags.add(new TagMetadata("station_name", "NCHAR(10)"));
+        tableMetadata.setTags(tags);
+
+        tableMapper.createSTable(tableMetadata);
+        return true;
+    }
+
+
+    public int insert(Rainfall rainfall) {
+        Map<String, Object> map = new HashMap<>();
+        map.put("dbname", "rainstation");
+        map.put("table", "S_53646");
+        map.put("stable", "monitoring");
+        map.put("values", rainfall);
+        return rainfallMapper.save(map);
+    }
+}
\ No newline at end of file

tests/examples/JDBC/springbootdemo/src/main/java/com/taosdata/jdbc/springbootdemo/service/WeatherService.java

@@ -14,10 +14,8 @@ public class WeatherService {
     private WeatherMapper weatherMapper;
 
     public boolean init() {
-
         weatherMapper.createDB();
         weatherMapper.createTable();
-
         return true;
     }
 

tests/examples/JDBC/springbootdemo/src/main/resources/application.properties

 # datasource config
 spring.datasource.driver-class-name=com.taosdata.jdbc.TSDBDriver
-spring.datasource.url=jdbc:TAOS://127.0.0.1:6030/log
+spring.datasource.url=jdbc:TAOS://localhost:6030/log
 spring.datasource.username=root
 spring.datasource.password=taosdata