diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000000000000000000000000000000000000..8f550a1f4fb1c89f51d9c91ac27ed98c085523a4 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,3 @@ +.git/ +.gitignore +.idea/ diff --git a/Dockerfile b/Dockerfile index 90d990c68248fed0f25af51f6a8d62a3c4dd7c79..73c8a4228cb7c8ca26e4f25144945395339e2517 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,9 +1,15 @@ -From continuumio/miniconda -RUN conda update -y conda -RUN conda create --name vec_engine python=3.6 -RUN echo "source activate vec_engine" > ~/.bashrc -ENV PATH /opt/conda/envs/env/bin:$PATH -#RUN conda install -y faiss-gpu cuda90 -c pytorch -#RUN pip install flask flask-restful flask_sqlalchemy flask_script pymysql environs -WORKDIR /root/front-source -EXPOSE 5000 +FROM nvidia/cuda:9.0-devel-ubuntu16.04 + +ENV NVIDIA_DRIVER_CAPABILITIES compute,utility + +WORKDIR /app + +COPY environment.yaml install/miniconda.sh /app/ + +RUN ./miniconda.sh -p $HOME/miniconda -b -f \ + && echo ". /root/miniconda/etc/profile.d/conda.sh" >> /root/.bashrc \ + && /root/miniconda/bin/conda env create -f environment.yaml \ + && echo "conda activate vec_engine" >> /root/.bashrc \ + && rm /app/* + +COPY . /app diff --git a/README.md b/README.md index dfd1a8e82632783034e76e7f5c2ad156e04121fd..9fefb01b1f3f058fbf94de35afdaa7be03484441 100644 --- a/README.md +++ b/README.md @@ -1,50 +1,34 @@ -# Vecwise Engine +# Vecwise Engine Dev Guide -### Geting started +## Install via Conda +1. Install Miniconda first + - `bash vecwise_engine/install/miniconda.sh` -- Create .env base on .env.example +2. Create environment -- Install Miniconda first - - `conda create --name vec_engine python=3.6` - - `conda activate vec_engine` - - `conda install faiss-gpu cuda90 -c pytorch # For CUDA9.0` - - `conda install flask` - - `pip install flask-restful flask_sqlalchemy flask_script` - - `pip install pymysql environs flask-profiler` + - `conda env create -f vecwise_engine/environment.yaml` -### Create Database +3. Test your installation -- Install MySQL +## Install via Docker + +1. Install nvidia-docker + +2. `docker build -t cuda9.0/VecEngine .` + +3. `docker run -it cuda9.0/VecEngine bash` + + +## Create Database +1. Install MySQL - `sudo apt-get update` - `sudo apt-get install mariadb-server` -- Create user and database: +2. Create user and database: - `create user vecwise;` - `create database vecdata;` - `grant all privileges on vecdata.* to 'vecwise'@'%';` - `flush privileges;` -- Create table: - - `python` # enter python3 interaction environment - - `from engine import db` - - `db.create_all()` - -- table desc -group_table -+-------------+--------------+------+-----+---------+----------------+ -| Field | Type | Null | Key | Default | Extra | -+-------------+--------------+------+-----+---------+----------------+ -| id | int(11) | NO | PRI | NULL | auto_increment | -| group_name | varchar(100) | YES | | NULL | | -| file_number | int(11) | YES | | NULL | | -+-------------+--------------+------+-----+---------+----------------+ - -file_table -+------------+--------------+------+-----+---------+----------------+ -| Field | Type | Null | Key | Default | Extra | -+------------+--------------+------+-----+---------+----------------+ -| id | int(11) | NO | PRI | NULL | auto_increment | -| group_name | varchar(100) | YES | | NULL | | -| filename | varchar(100) | YES | | NULL | | -| row_number | int(11) | YES | | NULL | | -+------------+--------------+------+-----+---------+----------------+ +3. Create table: + - `cd vecwise_engine/pyengine && python manager.py create_all` \ No newline at end of file diff --git a/environment.yaml b/environment.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ae4ddba20578af07bff01b377f5c056db0164480 --- /dev/null +++ b/environment.yaml @@ -0,0 +1,71 @@ +name: vec_engine +channels: + - pytorch + - defaults +dependencies: + - blas=1.0=mkl + - ca-certificates=2019.1.23=0 + - certifi=2019.3.9=py36_0 + - click=7.0=py36_0 + - flask=1.0.2=py36_1 + - intel-openmp=2019.1=144 + - itsdangerous=1.1.0=py36_0 + - jinja2=2.10=py36_0 + - libedit=3.1.20181209=hc058e9b_0 + - libffi=3.2.1=hd88cf55_4 + - libgcc-ng=8.2.0=hdf63c60_1 + - libgfortran-ng=7.3.0=hdf63c60_0 + - libstdcxx-ng=8.2.0=hdf63c60_1 + - markupsafe=1.1.1=py36h7b6447c_0 + - mkl=2019.1=144 + - mkl_fft=1.0.10=py36ha843d7b_0 + - mkl_random=1.0.2=py36hd81dba3_0 + - ncurses=6.1=he6710b0_1 + - numpy=1.16.2=py36h7e9f1db_0 + - numpy-base=1.16.2=py36hde5b4d6_0 + - openssl=1.1.1b=h7b6447c_1 + - pip=19.0.3=py36_0 + - python=3.6.8=h0371630_0 + - readline=7.0=h7b6447c_5 + - setuptools=40.8.0=py36_0 + - sqlite=3.27.2=h7b6447c_0 + - tk=8.6.8=hbc83047_0 + - werkzeug=0.14.1=py36_0 + - wheel=0.33.1=py36_0 + - xz=5.2.4=h14c3975_4 + - zlib=1.2.11=h7b6447c_3 + - cuda90=1.0=h6433d27_0 + - faiss-gpu=1.5.0=py36_cuda9.0_1 + - pip: + - aniso8601==6.0.0 + - atomicwrites==1.3.0 + - attrs==19.1.0 + - chardet==3.0.4 + - environs==4.1.0 + - faiss==0.1 + - flask-httpauth==3.2.4 + - flask-profiler==1.8.1 + - flask-restful==0.3.7 + - flask-script==2.0.6 + - flask-sqlalchemy==2.3.2 + - get==2019.3.22 + - idna==2.8 + - marshmallow==2.19.1 + - more-itertools==6.0.0 + - pluggy==0.9.0 + - post==2019.3.22 + - public==2019.3.22 + - py==1.8.0 + - pymysql==0.9.3 + - pytest==4.3.1 + - python-dotenv==0.10.1 + - pytz==2018.9 + - query-string==2019.3.22 + - request==2019.3.22 + - requests==2.21.0 + - simplejson==3.16.0 + - six==1.12.0 + - sqlalchemy==1.3.1 + - urllib3==1.24.1 +prefix: /home/zilliz/opt/app/miniconda3/envs/vec_engine + diff --git a/install/miniconda.sh.REMOVED.git-id b/install/miniconda.sh.REMOVED.git-id new file mode 100644 index 0000000000000000000000000000000000000000..946315ee3b0177f1cbce116e612d3ed91db34d9e --- /dev/null +++ b/install/miniconda.sh.REMOVED.git-id @@ -0,0 +1 @@ +58cc8994e72d5852a06f40ca7222f5751f773473 \ No newline at end of file