diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000000000000000000000000000000000000..9f11b755a17d8192c60f61cb17b8902dffbd9f23 --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +.idea/ diff --git a/README.md b/README.md index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..aebd13ec0566f5ded9693680b4a960dc8308c8a3 100644 --- a/README.md +++ b/README.md @@ -0,0 +1,10 @@ +# Vecwise Engine + +### Geting started + +- Install Miniconda first + - `conda create --name vec_engine python=3.6` + - `conda activate vec_engine` + - `conda install faiss-gpu cuda90 -c pytorch # For CUDA9.0` + - `conda install flask` + - `pip install flask-restful` diff --git a/engine/__init__.py b/engine/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/engine/app.py b/engine/app.py new file mode 100644 index 0000000000000000000000000000000000000000..cefa05e957c2b8c6e35459bdcfd07cb5d6cc3a7f --- /dev/null +++ b/engine/app.py @@ -0,0 +1,68 @@ +from flask import Flask +from flask_restful import Resource, Api + +app = Flask(__name__) +api = Api(app) + + +from flask_restful import reqparse +class Vector(Resource): + def __init__(self): + self.__parser = reqparse.RequestParser() + self.__parser.add_argument('groupid', type=str) + self.__parser.add_argument('vec', type=str) + + def post(self): + # args = self.__parser.parse_args() + # vec = args['vec'] + # groupid = args['groupid'] + return "vector post" + + +class VectorSearch(Resource): + def __init__(self): + self.__parser = reqparse.RequestParser() + self.__parser.add_argument('groupid', type=str) + + def post(self): + return "vectorSearch post" + + +class Index(Resource): + def __init__(self): + self.__parser = reqparse.RequestParser() + self.__parser.add_argument('groupid', type=str) + + def post(self): + return "index post" + + +class Group(Resource): + def __init__(self): + self.__parser = reqparse.RequestParser() + self.__parser.add_argument('groupid', type=str) + + def post(self, groupid): + return "group post" + + def get(self, groupid): + return "group get" + + def delete(self, groupid): + return "group delete" + + +class GroupList(Resource): + def get(self): + return "grouplist get" + + +api.add_resource(Vector, '/vector') +api.add_resource(Group, '/vector/group/') +api.add_resource(GroupList, '/vector/group') +api.add_resource(Index, '/vector/index') +api.add_resource(VectorSearch, '/vector/search') + + +if __name__ == '__main__': + app.run(debug=True) diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/tests/basic_test.py b/tests/basic_test.py new file mode 100644 index 0000000000000000000000000000000000000000..262910432e237e8131710fca795bb41cdd481312 --- /dev/null +++ b/tests/basic_test.py @@ -0,0 +1,52 @@ +import numpy as np + +d = 64 # dimension +nb = 100000 # database size +nq = 10000 # nb of queries +np.random.seed(1234) # make reproducible +xb = np.random.random((nb, d)).astype('float32') +xb[:, 0] += np.arange(nb) / 1000. +xq = np.random.random((nq, d)).astype('float32') +xq[:, 0] += np.arange(nq) / 1000. + +import faiss # make faiss available + +res = faiss.StandardGpuResources() # use a single GPU + +## Using a flat index + +index_flat = faiss.IndexFlatL2(d) # build a flat (CPU) index + +# make it a flat GPU index +gpu_index_flat = faiss.index_cpu_to_gpu(res, 0, index_flat) + +gpu_index_flat.add(xb) # add vectors to the index +print(gpu_index_flat.ntotal) + +k = 4 # we want to see 4 nearest neighbors +D, I = gpu_index_flat.search(xq, k) # actual search +print(I[:5]) # neighbors of the 5 first queries +print(I[-5:]) # neighbors of the 5 last queries + + +## Using an IVF index + +nlist = 100 +quantizer = faiss.IndexFlatL2(d) # the other index +index_ivf = faiss.IndexIVFFlat(quantizer, d, nlist, faiss.METRIC_L2) +# here we specify METRIC_L2, by default it performs inner-product search + +# make it an IVF GPU index +gpu_index_ivf = faiss.index_cpu_to_gpu(res, 0, index_ivf) + +assert not gpu_index_ivf.is_trained +gpu_index_ivf.train(xb) # add vectors to the index +assert gpu_index_ivf.is_trained + +gpu_index_ivf.add(xb) # add vectors to the index +print(gpu_index_ivf.ntotal) + +k = 4 # we want to see 4 nearest neighbors +D, I = gpu_index_ivf.search(xq, k) # actual search +print(I[:5]) # neighbors of the 5 first queries +print(I[-5:]) \ No newline at end of file