/* * Copyright (c) 2019 TAOS Data, Inc. * * This program is free software: you can use, redistribute, and/or modify * it under the terms of the GNU Affero General Public License, version 3 * or later ("AGPL"), as published by the Free * Software Foundation. * * This program is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. * * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . */ #include "indexComm.h" #include "index.h" #include "indexInt.h" #include "tcoding.h" #include "tcompare.h" #include "tdataformat.h" #include "ttypes.h" #include "tvariant.h" #define INDEX_DATA_BOOL_NULL 0x02 #define INDEX_DATA_TINYINT_NULL 0x80 #define INDEX_DATA_SMALLINT_NULL 0x8000 #define INDEX_DATA_INT_NULL 0x80000000LL #define INDEX_DATA_BIGINT_NULL 0x8000000000000000LL #define INDEX_DATA_TIMESTAMP_NULL TSDB_DATA_BIGINT_NULL #define INDEX_DATA_FLOAT_NULL 0x7FF00000 // it is an NAN #define INDEX_DATA_DOUBLE_NULL 0x7FFFFF0000000000LL // an NAN #define INDEX_DATA_NCHAR_NULL 0xFFFFFFFF #define INDEX_DATA_BINARY_NULL 0xFF #define INDEX_DATA_JSON_NULL 0xFFFFFFFF #define INDEX_DATA_JSON_null 0xFFFFFFFE #define INDEX_DATA_JSON_NOT_NULL 0x01 #define INDEX_DATA_UTINYINT_NULL 0xFF #define INDEX_DATA_USMALLINT_NULL 0xFFFF #define INDEX_DATA_UINT_NULL 0xFFFFFFFF #define INDEX_DATA_UBIGINT_NULL 0xFFFFFFFFFFFFFFFFL #define INDEX_DATA_NULL_STR "NULL" #define INDEX_DATA_NULL_STR_L "null" char JSON_COLUMN[] = "JSON"; char JSON_VALUE_DELIM = '&'; char* idxInt2str(int64_t val, char* dst, int radix) { char buffer[65] = {0}; char* p; int64_t new_val; uint64_t uval = (uint64_t)val; if (radix < 0) { if (val < 0) { *dst++ = '-'; uval = (uint64_t)0 - uval; /* Avoid integer overflow in (-val) for LLONG_MIN (BUG#31799). */ } } p = &buffer[sizeof(buffer) - 1]; *p = '\0'; new_val = (int64_t)(uval / 10); *--p = '0' + (char)(uval - (uint64_t)new_val * 10); val = new_val; while (val != 0) { new_val = val / 10; *--p = '0' + (char)(val - new_val * 10); val = new_val; } while ((*dst++ = *p++) != 0) ; return dst - 1; } __compar_fn_t idxGetCompar(int8_t type) { if (type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_NCHAR) { return (__compar_fn_t)strcmp; } return getComparFunc(type, 0); } static FORCE_INLINE TExeCond tCompareLessThan(void* a, void* b, int8_t type) { __compar_fn_t func = idxGetCompar(type); return tCompare(func, QUERY_LESS_THAN, a, b, type); } static FORCE_INLINE TExeCond tCompareLessEqual(void* a, void* b, int8_t type) { __compar_fn_t func = idxGetCompar(type); return tCompare(func, QUERY_LESS_EQUAL, a, b, type); } static FORCE_INLINE TExeCond tCompareGreaterThan(void* a, void* b, int8_t type) { __compar_fn_t func = idxGetCompar(type); return tCompare(func, QUERY_GREATER_THAN, a, b, type); } static FORCE_INLINE TExeCond tCompareGreaterEqual(void* a, void* b, int8_t type) { __compar_fn_t func = idxGetCompar(type); return tCompare(func, QUERY_GREATER_EQUAL, a, b, type); } static FORCE_INLINE TExeCond tCompareContains(void* a, void* b, int8_t type) { __compar_fn_t func = idxGetCompar(type); return tCompare(func, QUERY_TERM, a, b, type); } static FORCE_INLINE TExeCond tCompareEqual(void* a, void* b, int8_t type) { __compar_fn_t func = idxGetCompar(type); return tCompare(func, QUERY_TERM, a, b, type); } TExeCond tCompare(__compar_fn_t func, int8_t cmptype, void* a, void* b, int8_t dtype) { if (dtype == TSDB_DATA_TYPE_BINARY || dtype == TSDB_DATA_TYPE_NCHAR || dtype == TSDB_DATA_TYPE_VARBINARY) { return tDoCompare(func, cmptype, a, b); } #if 1 if (dtype == TSDB_DATA_TYPE_TIMESTAMP) { int64_t va = taosStr2int64(a); int64_t vb = taosStr2int64(b); return tDoCompare(func, cmptype, &va, &vb); } else if (dtype == TSDB_DATA_TYPE_BOOL || dtype == TSDB_DATA_TYPE_UTINYINT) { uint8_t va = taosStr2int64(a); uint8_t vb = taosStr2int64(b); return tDoCompare(func, cmptype, &va, &vb); } else if (dtype == TSDB_DATA_TYPE_TINYINT) { int8_t va = taosStr2int64(a); int8_t vb = taosStr2int64(b); return tDoCompare(func, cmptype, &va, &vb); } else if (dtype == TSDB_DATA_TYPE_SMALLINT) { int16_t va = taosStr2int64(a); int16_t vb = taosStr2int64(b); return tDoCompare(func, cmptype, &va, &vb); } else if (dtype == TSDB_DATA_TYPE_USMALLINT) { uint16_t va = taosStr2int64(a); uint16_t vb = taosStr2int64(b); return tDoCompare(func, cmptype, &va, &vb); } else if (dtype == TSDB_DATA_TYPE_INT) { int32_t va = taosStr2int64(a); int32_t vb = taosStr2int64(b); return tDoCompare(func, cmptype, &va, &vb); } else if (dtype == TSDB_DATA_TYPE_UINT) { uint32_t va = taosStr2int64(a); uint32_t vb = taosStr2int64(b); return tDoCompare(func, cmptype, &va, &vb); } else if (dtype == TSDB_DATA_TYPE_BIGINT) { int64_t va = taosStr2int64(a); int64_t vb = taosStr2int64(b); return tDoCompare(func, cmptype, &va, &vb); } else if (dtype == TSDB_DATA_TYPE_UBIGINT) { uint64_t va, vb; if (0 != toUInteger(a, strlen(a), 10, &va) || 0 != toUInteger(b, strlen(b), 10, &vb)) { return CONTINUE; } return tDoCompare(func, cmptype, &va, &vb); } else if (dtype == TSDB_DATA_TYPE_FLOAT) { float va = taosStr2Float(a, NULL); if (errno == ERANGE && va == -1) { return CONTINUE; } float vb = taosStr2Float(b, NULL); if (errno == ERANGE && va == -1) { return CONTINUE; } return tDoCompare(func, cmptype, &va, &vb); } else if (dtype == TSDB_DATA_TYPE_DOUBLE) { double va = taosStr2Double(a, NULL); if (errno == ERANGE && va == -1) { return CONTINUE; } double vb = taosStr2Double(b, NULL); if (errno == ERANGE && va == -1) { return CONTINUE; } return tDoCompare(func, cmptype, &va, &vb); } assert(0); return BREAK; #endif } TExeCond tDoCompare(__compar_fn_t func, int8_t comparType, void* a, void* b) { // optime later int32_t ret = func(a, b); switch (comparType) { case QUERY_LESS_THAN: if (ret < 0) return MATCH; break; case QUERY_LESS_EQUAL: { if (ret <= 0) return MATCH; break; } case QUERY_GREATER_THAN: { if (ret > 0) return MATCH; break; } case QUERY_GREATER_EQUAL: { if (ret >= 0) return MATCH; break; } case QUERY_TERM: { if (ret == 0) return MATCH; break; } default: return BREAK; } return CONTINUE; } static TExeCond (*rangeCompare[])(void* a, void* b, int8_t type) = { tCompareLessThan, tCompareLessEqual, tCompareGreaterThan, tCompareGreaterEqual, tCompareContains, tCompareEqual}; _cache_range_compare idxGetCompare(RangeType ty) { return rangeCompare[ty]; } char* idxPackJsonData(SIndexTerm* itm) { /* * |<-----colname---->|<-----dataType---->|<--------colVal---------->| * |<-----string----->|<-----uint8_t----->|<----depend on dataType-->| */ uint8_t ty = IDX_TYPE_GET_TYPE(itm->colType); int32_t sz = itm->nColName + itm->nColVal + sizeof(uint8_t) + sizeof(JSON_VALUE_DELIM) * 2 + 1; char* buf = (char*)taosMemoryCalloc(1, sz); char* p = buf; memcpy(p, itm->colName, itm->nColName); p += itm->nColName; memcpy(p, &JSON_VALUE_DELIM, sizeof(JSON_VALUE_DELIM)); p += sizeof(JSON_VALUE_DELIM); memcpy(p, &ty, sizeof(ty)); p += sizeof(ty); memcpy(p, &JSON_VALUE_DELIM, sizeof(JSON_VALUE_DELIM)); p += sizeof(JSON_VALUE_DELIM); memcpy(p, itm->colVal, itm->nColVal); return buf; } char* idxPackJsonDataPrefix(SIndexTerm* itm, int32_t* skip) { /* * |<-----colname---->|<-----dataType---->|<--------colVal---------->| * |<-----string----->|<-----uint8_t----->|<----depend on dataType-->| */ uint8_t ty = IDX_TYPE_GET_TYPE(itm->colType); int32_t sz = itm->nColName + itm->nColVal + sizeof(uint8_t) + sizeof(JSON_VALUE_DELIM) * 2 + 1; char* buf = (char*)taosMemoryCalloc(1, sz); char* p = buf; memcpy(p, itm->colName, itm->nColName); p += itm->nColName; memcpy(p, &JSON_VALUE_DELIM, sizeof(JSON_VALUE_DELIM)); p += sizeof(JSON_VALUE_DELIM); memcpy(p, &ty, sizeof(ty)); p += sizeof(ty); memcpy(p, &JSON_VALUE_DELIM, sizeof(JSON_VALUE_DELIM)); p += sizeof(JSON_VALUE_DELIM); *skip = p - buf; return buf; } char* idxPackJsonDataPrefixNoType(SIndexTerm* itm, int32_t* skip) { /* * |<-----colname---->|<-----dataType---->|<--------colVal---------->| * |<-----string----->|<-----uint8_t----->|<----depend on dataType-->| */ uint8_t ty = IDX_TYPE_GET_TYPE(itm->colType); int32_t sz = itm->nColName + itm->nColVal + sizeof(uint8_t) + sizeof(JSON_VALUE_DELIM) * 2 + 1; char* buf = (char*)taosMemoryCalloc(1, sz); char* p = buf; memcpy(p, itm->colName, itm->nColName); p += itm->nColName; memcpy(p, &JSON_VALUE_DELIM, sizeof(JSON_VALUE_DELIM)); p += sizeof(JSON_VALUE_DELIM); *skip = p - buf; return buf; } int idxUidCompare(const void* a, const void* b) { uint64_t l = *(uint64_t*)a; uint64_t r = *(uint64_t*)b; return l - r; } int32_t idxConvertData(void* src, int8_t type, void** dst) { int tlen = -1; switch (type) { case TSDB_DATA_TYPE_TIMESTAMP: tlen = taosEncodeFixedI64(NULL, *(int64_t*)src); *dst = taosMemoryCalloc(1, tlen + 1); tlen = taosEncodeFixedI64(dst, *(int64_t*)src); break; case TSDB_DATA_TYPE_BOOL: case TSDB_DATA_TYPE_UTINYINT: tlen = taosEncodeFixedU8(NULL, *(uint8_t*)src); *dst = taosMemoryCalloc(1, tlen + 1); tlen = taosEncodeFixedU8(dst, *(uint8_t*)src); break; case TSDB_DATA_TYPE_TINYINT: tlen = taosEncodeFixedI8(NULL, *(uint8_t*)src); *dst = taosMemoryCalloc(1, tlen + 1); tlen = taosEncodeFixedI8(dst, *(uint8_t*)src); break; case TSDB_DATA_TYPE_SMALLINT: tlen = taosEncodeFixedI16(NULL, *(int16_t*)src); *dst = taosMemoryCalloc(1, tlen + 1); tlen = taosEncodeFixedI16(dst, *(int16_t*)src); break; case TSDB_DATA_TYPE_USMALLINT: tlen = taosEncodeFixedU16(NULL, *(uint16_t*)src); *dst = taosMemoryCalloc(1, tlen + 1); tlen = taosEncodeFixedU16(dst, *(uint16_t*)src); break; case TSDB_DATA_TYPE_INT: tlen = taosEncodeFixedI32(NULL, *(int32_t*)src); *dst = taosMemoryCalloc(1, tlen + 1); tlen = taosEncodeFixedI32(dst, *(int32_t*)src); break; case TSDB_DATA_TYPE_FLOAT: tlen = taosEncodeBinary(NULL, src, sizeof(float)); *dst = taosMemoryCalloc(1, tlen + 1); tlen = taosEncodeBinary(dst, src, sizeof(float)); break; case TSDB_DATA_TYPE_UINT: tlen = taosEncodeFixedU32(NULL, *(uint32_t*)src); *dst = taosMemoryCalloc(1, tlen + 1); tlen = taosEncodeFixedU32(dst, *(uint32_t*)src); break; case TSDB_DATA_TYPE_BIGINT: tlen = taosEncodeFixedI64(NULL, *(int64_t*)src); *dst = taosMemoryCalloc(1, tlen + 1); tlen = taosEncodeFixedI64(dst, *(int64_t*)src); break; case TSDB_DATA_TYPE_DOUBLE: tlen = taosEncodeBinary(NULL, src, sizeof(double)); *dst = taosMemoryCalloc(1, tlen + 1); tlen = taosEncodeBinary(dst, src, sizeof(double)); break; case TSDB_DATA_TYPE_UBIGINT: tlen = taosEncodeFixedU64(NULL, *(uint64_t*)src); *dst = taosMemoryCalloc(1, tlen + 1); tlen = taosEncodeFixedU64(dst, *(uint64_t*)src); break; case TSDB_DATA_TYPE_NCHAR: { tlen = taosEncodeBinary(NULL, varDataVal(src), varDataLen(src)); *dst = taosMemoryCalloc(1, tlen + 1); tlen = taosEncodeBinary(dst, varDataVal(src), varDataLen(src)); break; } case TSDB_DATA_TYPE_VARCHAR: { // TSDB_DATA_TYPE_BINARY tlen = taosEncodeBinary(NULL, src, strlen(src)); *dst = taosMemoryCalloc(1, tlen + 1); tlen = taosEncodeBinary(dst, src, strlen(src)); break; } case TSDB_DATA_TYPE_VARBINARY: tlen = taosEncodeBinary(NULL, src, strlen(src)); *dst = taosMemoryCalloc(1, tlen + 1); tlen = taosEncodeBinary(dst, src, strlen(src)); break; default: TASSERT(0); break; } *dst = (char*)*dst - tlen; // indexMayFillNumbericData(*dst, tlen); return tlen; } int32_t idxConvertDataToStr(void* src, int8_t type, void** dst) { if (src == NULL) { *dst = strndup(INDEX_DATA_NULL_STR, (int)strlen(INDEX_DATA_NULL_STR)); return (int32_t)strlen(INDEX_DATA_NULL_STR); } int tlen = tDataTypes[type].bytes; int32_t bufSize = 64; switch (type) { case TSDB_DATA_TYPE_TIMESTAMP: *dst = taosMemoryCalloc(1, bufSize + 1); idxInt2str(*(int64_t*)src, *dst, -1); tlen = strlen(*dst); break; case TSDB_DATA_TYPE_BOOL: case TSDB_DATA_TYPE_UTINYINT: *dst = taosMemoryCalloc(1, bufSize + 1); idxInt2str(*(uint8_t*)src, *dst, 1); tlen = strlen(*dst); break; case TSDB_DATA_TYPE_TINYINT: *dst = taosMemoryCalloc(1, bufSize + 1); idxInt2str(*(int8_t*)src, *dst, 1); tlen = strlen(*dst); break; case TSDB_DATA_TYPE_SMALLINT: *dst = taosMemoryCalloc(1, bufSize + 1); idxInt2str(*(int16_t*)src, *dst, -1); tlen = strlen(*dst); break; case TSDB_DATA_TYPE_USMALLINT: *dst = taosMemoryCalloc(1, bufSize + 1); idxInt2str(*(uint16_t*)src, *dst, -1); tlen = strlen(*dst); break; case TSDB_DATA_TYPE_INT: *dst = taosMemoryCalloc(1, bufSize + 1); idxInt2str(*(int32_t*)src, *dst, -1); tlen = strlen(*dst); break; case TSDB_DATA_TYPE_UINT: *dst = taosMemoryCalloc(1, bufSize + 1); idxInt2str(*(uint32_t*)src, *dst, 1); tlen = strlen(*dst); break; case TSDB_DATA_TYPE_BIGINT: *dst = taosMemoryCalloc(1, bufSize + 1); sprintf(*dst, "%" PRIu64, *(uint64_t*)src); tlen = strlen(*dst); break; case TSDB_DATA_TYPE_UBIGINT: *dst = taosMemoryCalloc(1, bufSize + 1); idxInt2str(*(uint64_t*)src, *dst, 1); tlen = strlen(*dst); case TSDB_DATA_TYPE_FLOAT: *dst = taosMemoryCalloc(1, bufSize + 1); sprintf(*dst, "%.9lf", *(float*)src); tlen = strlen(*dst); break; case TSDB_DATA_TYPE_DOUBLE: *dst = taosMemoryCalloc(1, bufSize + 1); sprintf(*dst, "%.9lf", *(double*)src); tlen = strlen(*dst); break; case TSDB_DATA_TYPE_NCHAR: { tlen = taosEncodeBinary(NULL, varDataVal(src), varDataLen(src)); *dst = taosMemoryCalloc(1, tlen + 1); tlen = taosEncodeBinary(dst, varDataVal(src), varDataLen(src)); *dst = (char*)*dst - tlen; break; } case TSDB_DATA_TYPE_VARCHAR: { // TSDB_DATA_TYPE_BINARY tlen = taosEncodeBinary(NULL, varDataVal(src), varDataLen(src)); *dst = taosMemoryCalloc(1, tlen + 1); tlen = taosEncodeBinary(dst, varDataVal(src), varDataLen(src)); *dst = (char*)*dst - tlen; break; } case TSDB_DATA_TYPE_VARBINARY: tlen = taosEncodeBinary(NULL, varDataVal(src), varDataLen(src)); *dst = taosMemoryCalloc(1, tlen + 1); tlen = taosEncodeBinary(dst, varDataVal(src), varDataLen(src)); *dst = (char*)*dst - tlen; break; default: TASSERT(0); break; } return tlen; }