refine code

Former-commit-id: d5ee6d0520568a27f7e75754ec05c98a05b7f246

cpp/README.md

@@ -13,7 +13,7 @@ Note: If you want to debug into third-parties, you can build debug with CXXFLAGS
     ./build.sh -t Release
     
 #### Step 3: build(output to cmake_build folder)
-cmake_build/src/vecwise_engine_server is the server
+cmake_build/src/vecwise_server is the server
 
 cmake_build/src/libvecwise_engine.a is the static library
 
@@ -33,8 +33,9 @@ Set config in cpp/conf/server_config.yaml
 
 Then luanch server with config:
     
-    cd [build output path]/src
-    vecwise_engine_server -c [sourcecode path]/cpp/conf/server_config.yaml
+    cd [build output path]
+    start_server.sh
+    stop_server.sh
 
 ### Luanch test_client(only for debug)
 If you want to test remote api, you can build test_client.

cpp/src/CMakeLists.txt

@@ -27,7 +27,7 @@ find_library(cuda_library cudart cublas HINTS /usr/local/cuda/lib64)
 
 add_library(vecwise_engine STATIC ${vecwise_engine_src})
 
-add_executable(vecwise_engine_server
+add_executable(vecwise_server
         ${config_files}
         ${server_files}
         ${utils_files}
@@ -48,4 +48,4 @@ set(dependency_libs
         vecwise_engine
         sqlite3
         )
-target_link_libraries(vecwise_engine_server ${dependency_libs} ${cuda_library})
+target_link_libraries(vecwise_server ${dependency_libs} ${cuda_library})

cpp/src/main.cpp

@@ -14,6 +14,7 @@
 
 #include "utils/SignalUtil.h"
 #include "utils/CommonUtil.h"
+#include "utils/LogUtil.h"
 
 INITIALIZE_EASYLOGGINGPP
 
@@ -27,12 +28,14 @@ main(int argc, char *argv[]) {
 
 //    zilliz::lib::gpu::InitMemoryAllocator();
 
+    signal(SIGINT, server::SignalUtil::HandleSignal);
     signal(SIGSEGV, server::SignalUtil::HandleSignal);
     signal(SIGUSR1, server::SignalUtil::HandleSignal);
     signal(SIGUSR2, server::SignalUtil::HandleSignal);
 
     std::string app_name = basename(argv[0]);
     static struct option long_options[] = {{"conf_file", required_argument, 0, 'c'},
+                                           {"log_conf_file", required_argument, 0, 'l'},
                                            {"help", no_argument, 0, 'h'},
                                            {"daemon", no_argument, 0, 'd'},
                                            {"pid_file", required_argument, 0, 'p'},
@@ -42,7 +45,7 @@ main(int argc, char *argv[]) {
     int64_t start_daemonized = 0;
 //    int pid_fd;
 
-    std::string config_filename;
+    std::string config_filename, log_config_file;
     std::string pid_filename;
 
     app_name = argv[0];
@@ -54,7 +57,7 @@ main(int argc, char *argv[]) {
     }
 
     int value;
-    while ((value = getopt_long(argc, argv, "c:p:dh", long_options, &option_index)) != -1) {
+    while ((value = getopt_long(argc, argv, "c:l:p:dh", long_options, &option_index)) != -1) {
         switch (value) {
             case 'c': {
                 char *config_filename_ptr = strdup(optarg);
@@ -63,6 +66,13 @@ main(int argc, char *argv[]) {
                 printf("Loading configuration from: %s\n", config_filename.c_str());
                 break;
             }
+            case 'l': {
+                char *log_filename_ptr = strdup(optarg);
+                log_config_file = log_filename_ptr;
+                free(log_filename_ptr);
+                printf("Initial log config from: %s\n", log_config_file.c_str());
+                break;
+            }
 
             case 'p': {
                 char *pid_filename_ptr = strdup(optarg);
@@ -87,6 +97,8 @@ main(int argc, char *argv[]) {
         }
     }
 
+    zilliz::vecwise::server::InitLog(log_config_file);
+
     server::Server* server_ptr = server::Server::Instance();
     server_ptr->Init(start_daemonized, pid_filename, config_filename);
     return server_ptr->Start();

cpp/src/server/Server.cpp

@@ -9,7 +9,6 @@
 #include "utils/Log.h"
 #include "utils/SignalUtil.h"
 #include "utils/TimeRecorder.h"
-#include "utils/LogUtil.h"
 
 
 #include <fcntl.h>
@@ -145,8 +144,6 @@ Server::Start() {
                 return 1;
             }
 
-            zilliz::vecwise::server::InitLog();
-
             //log path is defined by LoadConfig, so InitLog must be called after LoadConfig
             ServerConfig &config = ServerConfig::GetInstance();
             ConfigNode server_config = config.GetConfig(CONFIG_SERVER);

cpp/src/utils/LogUtil.cpp

@@ -13,7 +13,7 @@ namespace zilliz {
 namespace vecwise {
 namespace server {
 
-int32_t InitLog() {
+int32_t InitLog(const std::string& log_config_file) {
 #if 0
     ServerConfig &config = ServerConfig::GetInstance();
     ConfigNode log_config = config.GetConfig(CONFIG_LOG);
@@ -48,7 +48,7 @@ int32_t InitLog() {
     el::Configurations conf;
     conf.parseFromText(str_config);
 #else
-    el::Configurations conf("../../conf/vecwise_engine_log.conf");
+    el::Configurations conf(log_config_file);
 #endif
 
     el::Loggers::reconfigureAllLoggers(conf);

cpp/src/utils/LogUtil.h

@@ -12,7 +12,7 @@ namespace zilliz {
 namespace vecwise {
 namespace server {
 
-int32_t InitLog();
+int32_t InitLog(const std::string& log_config_file);
 
 inline std::string GetFileName(std::string filename) {
     int pos = filename.find_last_of('/');

cpp/start_server.sh

+#!/bin/bash
+
+./cmake_build/src/vecwise_server -c ./conf/server_config.yaml -l ./conf/vecwise_engine_log.conf &
+

cpp/stop_server.sh

+#!/bin/bash
+
+function kill_progress()
+{
+  kill -s SIGUSR2 $(pgrep $1)
+
+  sleep 2
+}
+
+STATUS=$(kill_progress "vecwise_server" )
+
+if [[ ${STATUS} == "false" ]];then
+  echo "vecwise_server closed abnormally!"
+else
+  echo "vecwise_server closed successfully!"
+fi

cpp/test_client/CMakeLists.txt

@@ -38,14 +38,18 @@ set(client_libs
     boost_system
     boost_filesystem
     thrift
+    faiss
     pthread)
 
-target_link_libraries(test_client ${client_libs})
+include_directories(/usr/local/cuda/include)
+find_library(cuda_library cudart cublas HINTS /usr/local/cuda/lib64)
 
-add_executable(skeleton_server
-        ../src/thrift/gen-cpp/VecService_server.skeleton.cpp
-        ../src/thrift/gen-cpp/VecService.cpp
-        ../src/thrift/gen-cpp/VectorService_constants.cpp
-        ../src/thrift/gen-cpp/VectorService_types.cpp)
+target_link_libraries(test_client ${client_libs} ${cuda_library})
 
-target_link_libraries(skeleton_server thrift)
+#add_executable(skeleton_server
+#        ../src/thrift/gen-cpp/VecService_server.skeleton.cpp
+#        ../src/thrift/gen-cpp/VecService.cpp
+#        ../src/thrift/gen-cpp/VectorService_constants.cpp
+#        ../src/thrift/gen-cpp/VectorService_types.cpp)
+#
+#target_link_libraries(skeleton_server thrift)

cpp/test_client/main.cpp

@@ -11,15 +11,20 @@
 #include <easylogging++.h>
 
 #include "src/ClientApp.h"
+#include "src/FaissTest.h"
 
 INITIALIZE_EASYLOGGINGPP
 
 void print_help(const std::string &app_name);
 
+
 int
 main(int argc, char *argv[]) {
     printf("Client start...\n");
 
+//    FaissTest::test();
+//    return 0;
+
     std::string app_name = basename(argv[0]);
     static struct option long_options[] = {{"conf_file", required_argument, 0, 'c'},
                                            {"help", no_argument, 0, 'h'},

cpp/test_client/src/ClientApp.cpp

@@ -75,12 +75,15 @@ void ClientApp::Run(const std::string &config_file) {
             group.index_type = 0;
             client.add_group(group);
 
-            VecTensor tensor;
-            for(int32_t i = 0; i < dim; i++) {
-                tensor.tensor.push_back((double)i);
+            for(int64_t k = 0; k < 10000; k++) {
+                VecTensor tensor;
+                for(int32_t i = 0; i < dim; i++) {
+                    tensor.tensor.push_back((double)(i + k));
+                }
+
+                VecTensorIdList result;
+                client.add_vector(result, group.id, tensor);
             }
-            VecTensorIdList result;
-            client.add_vector(result, group.id, tensor);
 
         } catch (apache::thrift::TException& ex) {
             printf("%s", ex.what());

cpp/test_client/src/FaissTest.cpp

+//
+// Created by yhmo on 19-4-17.
+//
+
+#include "FaissTest.h"
+
+#include "utils/TimeRecorder.h"
+
+#include <faiss/IndexFlat.h>
+#include <faiss/MetaIndexes.h>
+#include <faiss/index_io.h>
+#include <faiss/AutoTune.h>
+#include <faiss/gpu/GpuIndexFlat.h>
+#include <faiss/gpu/GpuIndexIVFFlat.h>
+#include <faiss/gpu/StandardGpuResources.h>
+
+#include <assert.h>
+
+namespace {
+    void test_flat() {
+        zilliz::vecwise::server::TimeRecorder recorder("test_flat");
+
+        int d = 64;                            // dimension
+        int nb = 100000;                       // database size
+        int nq = 10000;                        // nb of queries
+
+        float *xb = new float[d * nb];
+        float *xq = new float[d * nq];
+
+        for(int i = 0; i < nb; i++) {
+            for(int j = 0; j < d; j++)
+                xb[d * i + j] = drand48();
+            xb[d * i] += i / 1000.;
+        }
+
+        for(int i = 0; i < nq; i++) {
+            for(int j = 0; j < d; j++)
+                xq[d * i + j] = drand48();
+            xq[d * i] += i / 1000.;
+        }
+
+        recorder.Record("prepare data");
+
+        faiss::IndexFlatL2 index(d);           // call constructor
+
+        recorder.Record("declare index");
+
+        printf("is_trained = %s\n", index.is_trained ? "true" : "false");
+        index.add(nb, xb);                     // add vectors to the index
+        printf("ntotal = %ld\n", index.ntotal);
+
+        recorder.Record("add index");
+
+        int k = 4;
+
+        {       // sanity check: search 5 first vectors of xb
+            long *I = new long[k * 5];
+            float *D = new float[k * 5];
+
+            index.search(5, xb, k, D, I);
+
+            // print results
+            printf("I=\n");
+            for(int i = 0; i < 5; i++) {
+                for(int j = 0; j < k; j++)
+                    printf("%5ld ", I[i * k + j]);
+                printf("\n");
+            }
+
+            printf("D=\n");
+            for(int i = 0; i < 5; i++) {
+                for(int j = 0; j < k; j++)
+                    printf("%7g ", D[i * k + j]);
+                printf("\n");
+            }
+
+            delete [] I;
+            delete [] D;
+        }
+
+        recorder.Record("search top 4");
+
+        {       // search xq
+            long *I = new long[k * nq];
+            float *D = new float[k * nq];
+
+            index.search(nq, xq, k, D, I);
+
+            // print results
+            printf("I (5 first results)=\n");
+            for(int i = 0; i < 5; i++) {
+                for(int j = 0; j < k; j++)
+                    printf("%5ld ", I[i * k + j]);
+                printf("\n");
+            }
+
+            printf("I (5 last results)=\n");
+            for(int i = nq - 5; i < nq; i++) {
+                for(int j = 0; j < k; j++)
+                    printf("%5ld ", I[i * k + j]);
+                printf("\n");
+            }
+
+            delete [] I;
+            delete [] D;
+        }
+
+        recorder.Record("search xq");
+
+        delete [] xb;
+        delete [] xq;
+
+        recorder.Record("delete data");
+    }
+
+    void test_gpu() {
+        zilliz::vecwise::server::TimeRecorder recorder("test_gpu");
+
+        int d = 64;                            // dimension
+        int nb = 100000;                       // database size
+        int nq = 10000;                        // nb of queries
+
+        float *xb = new float[d * nb];
+        float *xq = new float[d * nq];
+
+        for(int i = 0; i < nb; i++) {
+            for(int j = 0; j < d; j++)
+                xb[d * i + j] = drand48();
+            xb[d * i] += i / 1000.;
+        }
+
+        for(int i = 0; i < nq; i++) {
+            for(int j = 0; j < d; j++)
+                xq[d * i + j] = drand48();
+            xq[d * i] += i / 1000.;
+        }
+
+        recorder.Record("prepare data");
+
+        faiss::gpu::StandardGpuResources res;
+
+        // Using a flat index
+
+        faiss::gpu::GpuIndexFlatL2 index_flat(&res, d);
+
+        recorder.Record("declare index");
+
+        printf("is_trained = %s\n", index_flat.is_trained ? "true" : "false");
+        index_flat.add(nb, xb);  // add vectors to the index
+        printf("ntotal = %ld\n", index_flat.ntotal);
+
+        recorder.Record("add index");
+
+        int k = 4;
+
+        {       // search xq
+            long *I = new long[k * nq];
+            float *D = new float[k * nq];
+
+            index_flat.search(nq, xq, k, D, I);
+
+            // print results
+            printf("I (5 first results)=\n");
+            for(int i = 0; i < 5; i++) {
+                for(int j = 0; j < k; j++)
+                    printf("%5ld ", I[i * k + j]);
+                printf("\n");
+            }
+
+            printf("I (5 last results)=\n");
+            for(int i = nq - 5; i < nq; i++) {
+                for(int j = 0; j < k; j++)
+                    printf("%5ld ", I[i * k + j]);
+                printf("\n");
+            }
+
+            delete [] I;
+            delete [] D;
+        }
+
+        recorder.Record("search top 4");
+
+        // Using an IVF index
+
+        int nlist = 100;
+        faiss::gpu::GpuIndexIVFFlat index_ivf(&res, d, nlist, faiss::METRIC_L2);
+        // here we specify METRIC_L2, by default it performs inner-product search
+
+        recorder.Record("declare index");
+
+        assert(!index_ivf.is_trained);
+        index_ivf.train(nb, xb);
+        assert(index_ivf.is_trained);
+
+        recorder.Record("train index");
+
+        index_ivf.add(nb, xb);  // add vectors to the index
+
+        recorder.Record("add index");
+
+        printf("is_trained = %s\n", index_ivf.is_trained ? "true" : "false");
+        printf("ntotal = %ld\n", index_ivf.ntotal);
+
+        {       // search xq
+            long *I = new long[k * nq];
+            float *D = new float[k * nq];
+
+            index_ivf.search(nq, xq, k, D, I);
+
+            // print results
+            printf("I (5 first results)=\n");
+            for(int i = 0; i < 5; i++) {
+                for(int j = 0; j < k; j++)
+                    printf("%5ld ", I[i * k + j]);
+                printf("\n");
+            }
+
+            printf("I (5 last results)=\n");
+            for(int i = nq - 5; i < nq; i++) {
+                for(int j = 0; j < k; j++)
+                    printf("%5ld ", I[i * k + j]);
+                printf("\n");
+            }
+
+            delete [] I;
+            delete [] D;
+        }
+
+        recorder.Record("search xq");
+
+        delete [] xb;
+        delete [] xq;
+
+        recorder.Record("delete data");
+    }
+}
+
+void FaissTest::test() {
+
+    int ngpus = faiss::gpu::getNumDevices();
+
+    printf("Number of GPUs: %d\n", ngpus);
+
+    test_flat();
+    test_gpu();
+}
\ No newline at end of file

cpp/test_client/src/FaissTest.h

+//
+// Created by yhmo on 19-4-17.
+//
+
+#ifndef VECWISE_ENGINE_FAISSTEST_H
+#define VECWISE_ENGINE_FAISSTEST_H
+
+
+class FaissTest {
+public:
+    static void test();
+};
+
+
+#endif //VECWISE_ENGINE_FAISSTEST_H