ValidationUtil.cpp 24.7 KB
Newer Older
1
// Copyright (C) 2019-2020 Zilliz. All rights reserved.
J
jinhai 已提交
2
//
3 4
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
J
jinhai 已提交
5
//
6 7 8 9 10
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software distributed under the License
// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
// or implied. See the License for the specific language governing permissions and limitations under the License.
J
jinhai 已提交
11

S
starlord 已提交
12
#include "utils/ValidationUtil.h"
J
jinhai 已提交
13
#include "Log.h"
14
#include "db/Types.h"
15
#include "db/Utils.h"
S
starlord 已提交
16
#include "db/engine/ExecutionEngine.h"
C
Cai Yudong 已提交
17 18
#include "knowhere/index/vector_index/ConfAdapter.h"
#include "knowhere/index/vector_index/helpers/IndexParameter.h"
19
#include "utils/StringHelpFunctions.h"
J
jinhai 已提交
20

Z
zhiru 已提交
21
#include <arpa/inet.h>
G
groot 已提交
22

Y
youny626 已提交
23
#ifdef MILVUS_GPU_VERSION
24

S
starlord 已提交
25
#include <cuda_runtime.h>
26

Y
youny626 已提交
27
#endif
G
groot 已提交
28

S
shengjh 已提交
29
#include <fiu-local.h>
Z
zhiru 已提交
30
#include <algorithm>
J
JinHai-CN 已提交
31
#include <cmath>
32
#include <limits>
S
starlord 已提交
33 34
#include <regex>
#include <string>
Z
zhiru 已提交
35

J
jinhai 已提交
36 37 38
namespace milvus {
namespace server {

39
namespace {
G
groot 已提交
40 41
constexpr size_t COLLECTION_NAME_SIZE_LIMIT = 255;
constexpr int64_t COLLECTION_DIMENSION_LIMIT = 32768;
S
starlord 已提交
42
constexpr int32_t INDEX_FILE_SIZE_LIMIT = 4096;  // index trigger size max = 4096 MB
43 44
constexpr int64_t M_BYTE = 1024 * 1024;
constexpr int64_t MAX_INSERT_DATA_SIZE = 256 * M_BYTE;
J
jinhai 已提交
45

46 47 48 49 50
Status
CheckParameterRange(const milvus::json& json_params, const std::string& param_name, int64_t min, int64_t max,
                    bool min_close = true, bool max_closed = true) {
    if (json_params.find(param_name) == json_params.end()) {
        std::string msg = "Parameter list must contain: ";
51
        msg += param_name;
52
        LOG_SERVER_ERROR_ << msg;
53
        return Status(SERVER_INVALID_ARGUMENT, msg);
54 55 56 57 58 59 60 61 62 63
    }

    try {
        int64_t value = json_params[param_name];
        bool min_err = min_close ? value < min : value <= min;
        bool max_err = max_closed ? value > max : value >= max;
        if (min_err || max_err) {
            std::string msg = "Invalid " + param_name + " value: " + std::to_string(value) + ". Valid range is " +
                              (min_close ? "[" : "(") + std::to_string(min) + ", " + std::to_string(max) +
                              (max_closed ? "]" : ")");
64
            LOG_SERVER_ERROR_ << msg;
65 66 67 68
            return Status(SERVER_INVALID_ARGUMENT, msg);
        }
    } catch (std::exception& e) {
        std::string msg = "Invalid " + param_name + ": ";
69
        msg += e.what();
70
        LOG_SERVER_ERROR_ << msg;
71
        return Status(SERVER_INVALID_ARGUMENT, msg);
72 73 74 75 76 77 78 79 80
    }

    return Status::OK();
}

Status
CheckParameterExistence(const milvus::json& json_params, const std::string& param_name) {
    if (json_params.find(param_name) == json_params.end()) {
        std::string msg = "Parameter list must contain: ";
81
        msg += param_name;
82
        LOG_SERVER_ERROR_ << msg;
83
        return Status(SERVER_INVALID_ARGUMENT, msg);
84 85 86 87 88 89
    }

    try {
        int64_t value = json_params[param_name];
        if (value < 0) {
            std::string msg = "Invalid " + param_name + " value: " + std::to_string(value);
90
            LOG_SERVER_ERROR_ << msg;
91 92 93 94
            return Status(SERVER_INVALID_ARGUMENT, msg);
        }
    } catch (std::exception& e) {
        std::string msg = "Invalid " + param_name + ": ";
95
        msg += e.what();
96
        LOG_SERVER_ERROR_ << msg;
97
        return Status(SERVER_INVALID_ARGUMENT, msg);
98 99 100 101 102 103 104
    }

    return Status::OK();
}

}  // namespace

S
starlord 已提交
105
Status
J
Jin Hai 已提交
106 107 108 109
ValidationUtil::ValidateCollectionName(const std::string& collection_name) {
    // Collection name shouldn't be empty.
    if (collection_name.empty()) {
        std::string msg = "Collection name should not be empty.";
110
        LOG_SERVER_ERROR_ << msg;
G
groot 已提交
111
        return Status(SERVER_INVALID_COLLECTION_NAME, msg);
J
jinhai 已提交
112 113
    }

J
Jin Hai 已提交
114
    std::string invalid_msg = "Invalid collection name: " + collection_name + ". ";
G
groot 已提交
115
    // Collection name size shouldn't exceed 255.
G
groot 已提交
116
    if (collection_name.size() > COLLECTION_NAME_SIZE_LIMIT) {
J
Jin Hai 已提交
117
        std::string msg = invalid_msg + "The length of a collection name must be less than 255 characters.";
118
        LOG_SERVER_ERROR_ << msg;
G
groot 已提交
119
        return Status(SERVER_INVALID_COLLECTION_NAME, msg);
J
jinhai 已提交
120 121
    }

J
Jin Hai 已提交
122 123
    // Collection name first character should be underscore or character.
    char first_char = collection_name[0];
J
jinhai 已提交
124
    if (first_char != '_' && std::isalpha(first_char) == 0) {
J
Jin Hai 已提交
125
        std::string msg = invalid_msg + "The first character of a collection name must be an underscore or letter.";
126
        LOG_SERVER_ERROR_ << msg;
G
groot 已提交
127
        return Status(SERVER_INVALID_COLLECTION_NAME, msg);
J
jinhai 已提交
128 129
    }

J
Jin Hai 已提交
130
    int64_t table_name_size = collection_name.size();
J
jinhai 已提交
131
    for (int64_t i = 1; i < table_name_size; ++i) {
J
Jin Hai 已提交
132
        char name_char = collection_name[i];
J
jinhai 已提交
133
        if (name_char != '_' && std::isalnum(name_char) == 0) {
J
Jin Hai 已提交
134
            std::string msg = invalid_msg + "Collection name can only contain numbers, letters, and underscores.";
135
            LOG_SERVER_ERROR_ << msg;
G
groot 已提交
136
            return Status(SERVER_INVALID_COLLECTION_NAME, msg);
J
jinhai 已提交
137 138 139
        }
    }

S
starlord 已提交
140
    return Status::OK();
J
jinhai 已提交
141 142
}

S
starlord 已提交
143
Status
144
ValidationUtil::ValidateTableDimension(int64_t dimension, int64_t metric_type) {
G
groot 已提交
145
    if (dimension <= 0 || dimension > COLLECTION_DIMENSION_LIMIT) {
J
Jin Hai 已提交
146 147
        std::string msg = "Invalid collection dimension: " + std::to_string(dimension) + ". " +
                          "The collection dimension must be within the range of 1 ~ " +
G
groot 已提交
148
                          std::to_string(COLLECTION_DIMENSION_LIMIT) + ".";
149
        LOG_SERVER_ERROR_ << msg;
S
starlord 已提交
150
        return Status(SERVER_INVALID_VECTOR_DIMENSION, msg);
J
jinhai 已提交
151
    }
152 153 154

    if (milvus::engine::utils::IsBinaryMetricType(metric_type)) {
        if ((dimension % 8) != 0) {
J
Jin Hai 已提交
155 156
            std::string msg = "Invalid collection dimension: " + std::to_string(dimension) + ". " +
                              "The collection dimension must be a multiple of 8";
157
            LOG_SERVER_ERROR_ << msg;
158 159 160 161 162
            return Status(SERVER_INVALID_VECTOR_DIMENSION, msg);
        }
    }

    return Status::OK();
J
jinhai 已提交
163 164
}

S
starlord 已提交
165
Status
166
ValidationUtil::ValidateCollectionIndexType(int32_t index_type) {
S
starlord 已提交
167 168
    int engine_type = static_cast<int>(engine::EngineType(index_type));
    if (engine_type <= 0 || engine_type > static_cast<int>(engine::EngineType::MAX_VALUE)) {
S
starlord 已提交
169 170
        std::string msg = "Invalid index type: " + std::to_string(index_type) + ". " +
                          "Make sure the index type is in IndexType list.";
171
        LOG_SERVER_ERROR_ << msg;
S
starlord 已提交
172
        return Status(SERVER_INVALID_INDEX_TYPE, msg);
J
jinhai 已提交
173
    }
174

175
#ifndef MILVUS_GPU_VERSION
176 177 178
    // special case, hybird index only available in customize faiss library
    if (engine_type == static_cast<int>(engine::EngineType::FAISS_IVFSQ8H)) {
        std::string msg = "Unsupported index type: " + std::to_string(index_type);
179
        LOG_SERVER_ERROR_ << msg;
180 181 182 183
        return Status(SERVER_INVALID_INDEX_TYPE, msg);
    }
#endif

S
starlord 已提交
184
    return Status::OK();
J
jinhai 已提交
185 186
}

187
Status
J
Jin Hai 已提交
188
ValidationUtil::ValidateIndexParams(const milvus::json& index_params,
189
                                    const engine::meta::CollectionSchema& collection_schema, int32_t index_type) {
190 191 192 193 194 195 196 197 198
    switch (index_type) {
        case (int32_t)engine::EngineType::FAISS_IDMAP:
        case (int32_t)engine::EngineType::FAISS_BIN_IDMAP: {
            break;
        }
        case (int32_t)engine::EngineType::FAISS_IVFFLAT:
        case (int32_t)engine::EngineType::FAISS_IVFSQ8:
        case (int32_t)engine::EngineType::FAISS_IVFSQ8H:
        case (int32_t)engine::EngineType::FAISS_BIN_IVFFLAT: {
G
typo  
groot 已提交
199
            auto status = CheckParameterRange(index_params, knowhere::IndexParams::nlist, 1, 999999);
200 201 202 203 204 205
            if (!status.ok()) {
                return status;
            }
            break;
        }
        case (int32_t)engine::EngineType::FAISS_PQ: {
G
typo  
groot 已提交
206
            auto status = CheckParameterRange(index_params, knowhere::IndexParams::nlist, 1, 999999);
207 208 209 210 211 212 213 214 215
            if (!status.ok()) {
                return status;
            }

            status = CheckParameterExistence(index_params, knowhere::IndexParams::m);
            if (!status.ok()) {
                return status;
            }

216 217
            // special check for 'm' parameter
            std::vector<int64_t> resset;
218
            milvus::knowhere::IVFPQConfAdapter::GetValidMList(collection_schema.dimension_, resset);
C
Cai Yudong 已提交
219
            int64_t m_value = index_params[knowhere::IndexParams::m];
220
            if (resset.empty()) {
J
Jin Hai 已提交
221
                std::string msg = "Invalid collection dimension, unable to get reasonable values for 'm'";
222
                LOG_SERVER_ERROR_ << msg;
G
groot 已提交
223
                return Status(SERVER_INVALID_COLLECTION_DIMENSION, msg);
224 225 226 227 228 229 230 231 232 233 234 235 236
            }

            auto iter = std::find(std::begin(resset), std::end(resset), m_value);
            if (iter == std::end(resset)) {
                std::string msg =
                    "Invalid " + std::string(knowhere::IndexParams::m) + ", must be one of the following values: ";
                for (size_t i = 0; i < resset.size(); i++) {
                    if (i != 0) {
                        msg += ",";
                    }
                    msg += std::to_string(resset[i]);
                }

237
                LOG_SERVER_ERROR_ << msg;
238 239 240
                return Status(SERVER_INVALID_ARGUMENT, msg);
            }

241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272
            break;
        }
        case (int32_t)engine::EngineType::NSG_MIX: {
            auto status = CheckParameterRange(index_params, knowhere::IndexParams::search_length, 10, 300);
            if (!status.ok()) {
                return status;
            }
            status = CheckParameterRange(index_params, knowhere::IndexParams::out_degree, 5, 300);
            if (!status.ok()) {
                return status;
            }
            status = CheckParameterRange(index_params, knowhere::IndexParams::candidate, 50, 1000);
            if (!status.ok()) {
                return status;
            }
            status = CheckParameterRange(index_params, knowhere::IndexParams::knng, 5, 300);
            if (!status.ok()) {
                return status;
            }
            break;
        }
        case (int32_t)engine::EngineType::HNSW: {
            auto status = CheckParameterRange(index_params, knowhere::IndexParams::M, 5, 48);
            if (!status.ok()) {
                return status;
            }
            status = CheckParameterRange(index_params, knowhere::IndexParams::efConstruction, 100, 500);
            if (!status.ok()) {
                return status;
            }
            break;
        }
273 274 275 276 277 278 279
        case (int32_t)engine::EngineType::ANNOY: {
            auto status = CheckParameterRange(index_params, knowhere::IndexParams::n_trees, 1, 1024);
            if (!status.ok()) {
                return status;
            }
            break;
        }
280 281
    }
    return Status::OK();
G
groot 已提交
282 283
}

S
starlord 已提交
284
Status
J
Jin Hai 已提交
285
ValidationUtil::ValidateSearchParams(const milvus::json& search_params,
286 287
                                     const engine::meta::CollectionSchema& collection_schema, int64_t topk) {
    switch (collection_schema.engine_type_) {
288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310
        case (int32_t)engine::EngineType::FAISS_IDMAP:
        case (int32_t)engine::EngineType::FAISS_BIN_IDMAP: {
            break;
        }
        case (int32_t)engine::EngineType::FAISS_IVFFLAT:
        case (int32_t)engine::EngineType::FAISS_IVFSQ8:
        case (int32_t)engine::EngineType::FAISS_IVFSQ8H:
        case (int32_t)engine::EngineType::FAISS_BIN_IVFFLAT:
        case (int32_t)engine::EngineType::FAISS_PQ: {
            auto status = CheckParameterRange(search_params, knowhere::IndexParams::nprobe, 1, 999999);
            if (!status.ok()) {
                return status;
            }
            break;
        }
        case (int32_t)engine::EngineType::NSG_MIX: {
            auto status = CheckParameterRange(search_params, knowhere::IndexParams::search_length, 10, 300);
            if (!status.ok()) {
                return status;
            }
            break;
        }
        case (int32_t)engine::EngineType::HNSW: {
G
typo  
groot 已提交
311
            auto status = CheckParameterRange(search_params, knowhere::IndexParams::ef, topk, 4096);
312 313 314 315 316
            if (!status.ok()) {
                return status;
            }
            break;
        }
317
        case (int32_t)engine::EngineType::ANNOY: {
318 319
            auto status = CheckParameterRange(search_params, knowhere::IndexParams::search_k, topk,
                                              std::numeric_limits<int64_t>::max());
320 321 322 323 324
            if (!status.ok()) {
                return status;
            }
            break;
        }
S
starlord 已提交
325
    }
S
starlord 已提交
326
    return Status::OK();
S
starlord 已提交
327 328
}

329
Status
J
Jin Hai 已提交
330
ValidationUtil::ValidateVectorData(const engine::VectorsData& vectors,
331
                                   const engine::meta::CollectionSchema& collection_schema) {
332 333
    uint64_t vector_count = vectors.vector_count_;
    if ((vectors.float_data_.empty() && vectors.binary_data_.empty()) || vector_count == 0) {
334 335 336 337
        return Status(SERVER_INVALID_ROWRECORD_ARRAY,
                      "The vector array is empty. Make sure you have entered vector records.");
    }

338
    if (engine::utils::IsBinaryMetricType(collection_schema.metric_type_)) {
339 340
        // check prepared binary data
        if (vectors.binary_data_.size() % vector_count != 0) {
J
Jin Hai 已提交
341 342
            return Status(SERVER_INVALID_ROWRECORD_ARRAY,
                          "The vector dimension must be equal to the collection dimension.");
343 344
        }

345
        if (vectors.binary_data_.size() * 8 / vector_count != collection_schema.dimension_) {
346
            return Status(SERVER_INVALID_VECTOR_DIMENSION,
J
Jin Hai 已提交
347
                          "The vector dimension must be equal to the collection dimension.");
348 349 350 351 352
        }
    } else {
        // check prepared float data
        fiu_do_on("SearchRequest.OnExecute.invalod_rowrecord_array", vector_count = vectors.float_data_.size() + 1);
        if (vectors.float_data_.size() % vector_count != 0) {
J
Jin Hai 已提交
353 354
            return Status(SERVER_INVALID_ROWRECORD_ARRAY,
                          "The vector dimension must be equal to the collection dimension.");
355
        }
356
        if (vectors.float_data_.size() / vector_count != collection_schema.dimension_) {
357
            return Status(SERVER_INVALID_VECTOR_DIMENSION,
J
Jin Hai 已提交
358
                          "The vector dimension must be equal to the collection dimension.");
359 360 361 362 363 364
        }
    }

    return Status::OK();
}

365 366
Status
ValidationUtil::ValidateVectorDataSize(const engine::VectorsData& vectors,
367
                                       const engine::meta::CollectionSchema& collection_schema) {
368 369
    std::string msg =
        "The amount of data inserted each time cannot exceed " + std::to_string(MAX_INSERT_DATA_SIZE / M_BYTE) + " MB";
370
    if (engine::utils::IsBinaryMetricType(collection_schema.metric_type_)) {
371 372 373 374 375 376 377 378 379 380 381 382 383
        if (vectors.binary_data_.size() > MAX_INSERT_DATA_SIZE) {
            return Status(SERVER_INVALID_ROWRECORD_ARRAY, msg);
        }

    } else {
        if (vectors.float_data_.size() * sizeof(float) > MAX_INSERT_DATA_SIZE) {
            return Status(SERVER_INVALID_ROWRECORD_ARRAY, msg);
        }
    }

    return Status::OK();
}

S
starlord 已提交
384
Status
385
ValidationUtil::ValidateCollectionIndexFileSize(int64_t index_file_size) {
Z
zhiru 已提交
386
    if (index_file_size <= 0 || index_file_size > INDEX_FILE_SIZE_LIMIT) {
S
starlord 已提交
387 388 389
        std::string msg = "Invalid index file size: " + std::to_string(index_file_size) + ". " +
                          "The index file size must be within the range of 1 ~ " +
                          std::to_string(INDEX_FILE_SIZE_LIMIT) + ".";
390
        LOG_SERVER_ERROR_ << msg;
S
starlord 已提交
391
        return Status(SERVER_INVALID_INDEX_FILE_SIZE, msg);
S
starlord 已提交
392 393
    }

S
starlord 已提交
394
    return Status::OK();
S
starlord 已提交
395 396
}

S
starlord 已提交
397
Status
398
ValidationUtil::ValidateCollectionIndexMetricType(int32_t metric_type) {
G
groot 已提交
399
    if (metric_type <= 0 || metric_type > static_cast<int32_t>(engine::MetricType::MAX_VALUE)) {
S
starlord 已提交
400
        std::string msg = "Invalid index metric type: " + std::to_string(metric_type) + ". " +
G
groot 已提交
401
                          "Make sure the metric type is in MetricType list.";
402
        LOG_SERVER_ERROR_ << msg;
S
starlord 已提交
403
        return Status(SERVER_INVALID_INDEX_METRIC_TYPE, msg);
S
starlord 已提交
404
    }
S
starlord 已提交
405
    return Status::OK();
S
starlord 已提交
406 407
}

S
starlord 已提交
408
Status
409 410
ValidationUtil::ValidateSearchTopk(int64_t top_k) {
    if (top_k <= 0 || top_k > QUERY_MAX_TOPK) {
S
starlord 已提交
411 412
        std::string msg =
            "Invalid topk: " + std::to_string(top_k) + ". " + "The topk must be within the range of 1 ~ 2048.";
413
        LOG_SERVER_ERROR_ << msg;
S
starlord 已提交
414
        return Status(SERVER_INVALID_TOPK, msg);
415 416
    }

S
starlord 已提交
417
    return Status::OK();
418 419
}

420 421 422 423
Status
ValidationUtil::ValidatePartitionName(const std::string& partition_name) {
    if (partition_name.empty()) {
        std::string msg = "Partition name should not be empty.";
424
        LOG_SERVER_ERROR_ << msg;
G
groot 已提交
425
        return Status(SERVER_INVALID_COLLECTION_NAME, msg);
426 427
    }

428
    std::string invalid_msg = "Invalid partition name: " + partition_name + ". ";
G
groot 已提交
429
    // Collection name size shouldn't exceed 255.
G
groot 已提交
430
    if (partition_name.size() > COLLECTION_NAME_SIZE_LIMIT) {
431
        std::string msg = invalid_msg + "The length of a partition name must be less than 255 characters.";
432
        LOG_SERVER_ERROR_ << msg;
G
groot 已提交
433
        return Status(SERVER_INVALID_COLLECTION_NAME, msg);
434 435
    }

J
Jin Hai 已提交
436
    // Collection name first character should be underscore or character.
437 438 439
    char first_char = partition_name[0];
    if (first_char != '_' && std::isalpha(first_char) == 0) {
        std::string msg = invalid_msg + "The first character of a partition name must be an underscore or letter.";
440
        LOG_SERVER_ERROR_ << msg;
G
groot 已提交
441
        return Status(SERVER_INVALID_COLLECTION_NAME, msg);
442 443 444 445 446 447 448
    }

    int64_t table_name_size = partition_name.size();
    for (int64_t i = 1; i < table_name_size; ++i) {
        char name_char = partition_name[i];
        if (name_char != '_' && std::isalnum(name_char) == 0) {
            std::string msg = invalid_msg + "Partition name can only contain numbers, letters, and underscores.";
449
            LOG_SERVER_ERROR_ << msg;
G
groot 已提交
450
            return Status(SERVER_INVALID_COLLECTION_NAME, msg);
451 452 453 454
        }
    }

    return Status::OK();
455 456
}

G
groot 已提交
457 458
Status
ValidationUtil::ValidatePartitionTags(const std::vector<std::string>& partition_tags) {
459 460 461 462 463 464 465
    for (const std::string& tag : partition_tags) {
        // trim side-blank of tag, only compare valid characters
        // for example: " ab cd " is treated as "ab cd"
        std::string valid_tag = tag;
        StringHelpFunctions::TrimStringBlank(valid_tag);
        if (valid_tag.empty()) {
            std::string msg = "Invalid partition tag: " + valid_tag + ". " + "Partition tag should not be empty.";
466
            LOG_SERVER_ERROR_ << msg;
467
            return Status(SERVER_INVALID_PARTITION_TAG, msg);
G
groot 已提交
468
        }
469 470 471 472

        // max length of partition tag
        if (valid_tag.length() > 255) {
            std::string msg = "Invalid partition tag: " + valid_tag + ". " + "Partition tag exceed max length(255).";
473
            LOG_SERVER_ERROR_ << msg;
474 475
            return Status(SERVER_INVALID_PARTITION_TAG, msg);
        }
G
groot 已提交
476 477 478 479 480
    }

    return Status::OK();
}

S
starlord 已提交
481
Status
482
ValidationUtil::ValidateGpuIndex(int32_t gpu_index) {
Y
youny626 已提交
483
#ifdef MILVUS_GPU_VERSION
484 485
    int num_devices = 0;
    auto cuda_err = cudaGetDeviceCount(&num_devices);
S
shengjh 已提交
486 487
    fiu_do_on("ValidationUtil.ValidateGpuIndex.get_device_count_fail", cuda_err = cudaError::cudaErrorUnknown);

S
starlord 已提交
488
    if (cuda_err != cudaSuccess) {
S
starlord 已提交
489
        std::string msg = "Failed to get gpu card number, cuda error:" + std::to_string(cuda_err);
490
        LOG_SERVER_ERROR_ << msg;
S
starlord 已提交
491
        return Status(SERVER_UNEXPECTED_ERROR, msg);
492 493
    }

Z
zhiru 已提交
494
    if (gpu_index >= num_devices) {
S
starlord 已提交
495
        std::string msg = "Invalid gpu index: " + std::to_string(gpu_index);
496
        LOG_SERVER_ERROR_ << msg;
S
starlord 已提交
497
        return Status(SERVER_INVALID_ARGUMENT, msg);
498
    }
Y
youny626 已提交
499
#endif
500

S
starlord 已提交
501
    return Status::OK();
502 503
}

G
groot 已提交
504
#ifdef MILVUS_GPU_VERSION
G
groot 已提交
505

S
starlord 已提交
506
Status
C
Cai Yudong 已提交
507
ValidationUtil::GetGpuMemory(int32_t gpu_index, int64_t& memory) {
S
shengjh 已提交
508 509
    fiu_return_on("ValidationUtil.GetGpuMemory.return_error", Status(SERVER_UNEXPECTED_ERROR, ""));

510 511 512
    cudaDeviceProp deviceProp;
    auto cuda_err = cudaGetDeviceProperties(&deviceProp, gpu_index);
    if (cuda_err) {
Y
youny626 已提交
513 514
        std::string msg = "Failed to get gpu properties for gpu" + std::to_string(gpu_index) +
                          " , cuda error:" + std::to_string(cuda_err);
515
        LOG_SERVER_ERROR_ << msg;
S
starlord 已提交
516
        return Status(SERVER_UNEXPECTED_ERROR, msg);
517 518 519
    }

    memory = deviceProp.totalGlobalMem;
S
starlord 已提交
520
    return Status::OK();
521
}
G
groot 已提交
522

G
groot 已提交
523
#endif
524

S
starlord 已提交
525
Status
S
starlord 已提交
526
ValidationUtil::ValidateIpAddress(const std::string& ip_address) {
Z
zhiru 已提交
527 528 529
    struct in_addr address;

    int result = inet_pton(AF_INET, ip_address.c_str(), &address);
S
shengjh 已提交
530
    fiu_do_on("ValidationUtil.ValidateIpAddress.error_ip_result", result = 2);
Z
zhiru 已提交
531

Z
zhiru 已提交
532
    switch (result) {
S
starlord 已提交
533 534
        case 1:
            return Status::OK();
S
starlord 已提交
535 536
        case 0: {
            std::string msg = "Invalid IP address: " + ip_address;
537
            LOG_SERVER_ERROR_ << msg;
S
starlord 已提交
538 539 540 541
            return Status(SERVER_INVALID_ARGUMENT, msg);
        }
        default: {
            std::string msg = "IP address conversion error: " + ip_address;
542
            LOG_SERVER_ERROR_ << msg;
S
starlord 已提交
543 544
            return Status(SERVER_UNEXPECTED_ERROR, msg);
        }
Z
zhiru 已提交
545 546 547
    }
}

S
starlord 已提交
548
Status
S
starlord 已提交
549
ValidationUtil::ValidateStringIsNumber(const std::string& str) {
Y
yudong.cai 已提交
550 551
    if (str.empty() || !std::all_of(str.begin(), str.end(), ::isdigit)) {
        return Status(SERVER_INVALID_ARGUMENT, "Invalid number");
Z
zhiru 已提交
552
    }
Y
yudong.cai 已提交
553
    try {
B
BossZou 已提交
554
        int64_t value = std::stol(str);
S
shengjh 已提交
555
        fiu_do_on("ValidationUtil.ValidateStringIsNumber.throw_exception", throw std::exception());
556 557 558
        if (value < 0) {
            return Status(SERVER_INVALID_ARGUMENT, "Negative number");
        }
S
starlord 已提交
559
    } catch (...) {
Y
yudong.cai 已提交
560
        return Status(SERVER_INVALID_ARGUMENT, "Invalid number");
Z
zhiru 已提交
561
    }
Y
yudong.cai 已提交
562
    return Status::OK();
Z
zhiru 已提交
563 564
}

S
starlord 已提交
565
Status
S
starlord 已提交
566
ValidationUtil::ValidateStringIsBool(const std::string& str) {
S
shengjh 已提交
567
    fiu_return_on("ValidateStringNotBool", Status(SERVER_INVALID_ARGUMENT, "Invalid boolean: " + str));
Y
yudong.cai 已提交
568 569
    std::string s = str;
    std::transform(s.begin(), s.end(), s.begin(), ::tolower);
S
starlord 已提交
570
    if (s == "true" || s == "on" || s == "yes" || s == "1" || s == "false" || s == "off" || s == "no" || s == "0" ||
Y
yudong.cai 已提交
571
        s.empty()) {
S
starlord 已提交
572
        return Status::OK();
Z
zhiru 已提交
573
    }
S
starlord 已提交
574
    return Status(SERVER_INVALID_ARGUMENT, "Invalid boolean: " + str);
Z
zhiru 已提交
575 576
}

S
starlord 已提交
577
Status
S
starlord 已提交
578
ValidationUtil::ValidateStringIsFloat(const std::string& str) {
Y
yudong.cai 已提交
579 580
    try {
        float val = std::stof(str);
581 582 583
        if (val < 0.0) {
            return Status(SERVER_INVALID_ARGUMENT, "Negative float: " + str);
        }
S
starlord 已提交
584
    } catch (...) {
Y
yudong.cai 已提交
585
        return Status(SERVER_INVALID_ARGUMENT, "Invalid float: " + str);
Z
zhiru 已提交
586
    }
Y
yudong.cai 已提交
587
    return Status::OK();
Z
zhiru 已提交
588 589
}

S
starlord 已提交
590
Status
S
starlord 已提交
591
ValidationUtil::ValidateDbURI(const std::string& uri) {
Z
zhiru 已提交
592 593 594 595 596 597
    std::string dialectRegex = "(.*)";
    std::string usernameRegex = "(.*)";
    std::string passwordRegex = "(.*)";
    std::string hostRegex = "(.*)";
    std::string portRegex = "(.*)";
    std::string dbNameRegex = "(.*)";
S
starlord 已提交
598 599
    std::string uriRegexStr = dialectRegex + "\\:\\/\\/" + usernameRegex + "\\:" + passwordRegex + "\\@" + hostRegex +
                              "\\:" + portRegex + "\\/" + dbNameRegex;
Z
zhiru 已提交
600 601 602 603 604 605 606 607 608
    std::regex uriRegex(uriRegexStr);
    std::smatch pieces_match;

    bool okay = true;

    if (std::regex_match(uri, pieces_match, uriRegex)) {
        std::string dialect = pieces_match[1].str();
        std::transform(dialect.begin(), dialect.end(), dialect.begin(), ::tolower);
        if (dialect.find("mysql") == std::string::npos && dialect.find("sqlite") == std::string::npos) {
609
            LOG_SERVER_ERROR_ << "Invalid dialect in URI: dialect = " << dialect;
Z
zhiru 已提交
610 611 612
            okay = false;
        }

S
starlord 已提交
613 614 615 616 617 618
        /*
         *      Could be DNS, skip checking
         *
                std::string host = pieces_match[4].str();
                if (!host.empty() && host != "localhost") {
                    if (ValidateIpAddress(host) != SERVER_SUCCESS) {
619
                        LOG_SERVER_ERROR_ << "Invalid host ip address in uri = " << host;
S
starlord 已提交
620 621 622 623
                        okay = false;
                    }
                }
        */
Z
zhiru 已提交
624 625 626

        std::string port = pieces_match[5].str();
        if (!port.empty()) {
S
starlord 已提交
627 628
            auto status = ValidateStringIsNumber(port);
            if (!status.ok()) {
629
                LOG_SERVER_ERROR_ << "Invalid port in uri = " << port;
Z
zhiru 已提交
630 631 632
                okay = false;
            }
        }
S
starlord 已提交
633
    } else {
634
        LOG_SERVER_ERROR_ << "Wrong URI format: URI = " << uri;
Z
zhiru 已提交
635 636 637
        okay = false;
    }

S
starlord 已提交
638
    return (okay ? Status::OK() : Status(SERVER_INVALID_ARGUMENT, "Invalid db backend uri"));
Z
zhiru 已提交
639 640
}

641 642 643 644 645 646 647 648 649 650 651 652 653
Status
ValidationUtil::ValidateStoragePath(const std::string& path) {
    // Validate storage path if is valid, only correct absolute path will be validated pass
    // Invalid path only contain character[a-zA-Z], number[0-9], '-', and '_',
    // and path must start with '/'.
    // examples below are invalid
    // '/a//a', '/a--/a', '/-a/a', '/a@#/a', 'aaa/sfs'
    std::string path_pattern = "^\\/(\\w+-?\\/?)+$";
    std::regex regex(path_pattern);

    return std::regex_match(path, regex) ? Status::OK() : Status(SERVER_INVALID_ARGUMENT, "Invalid file path");
}

J
Jin Hai 已提交
654 655 656 657 658
bool
ValidationUtil::IsNumber(const std::string& s) {
    return !s.empty() && std::all_of(s.begin(), s.end(), ::isdigit);
}

S
starlord 已提交
659 660
}  // namespace server
}  // namespace milvus