diff --git a/CHANGELOG.md b/CHANGELOG.md index 00402ea15f9b9cf74d055b00f069150022187c4e..064ad9b43935f7b392cdc97032b2b57295fc3f4a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,8 +12,10 @@ Please mark all change in change log and use the ticket from JIRA. - \#80 - Print version information into log during server start - \#82 - Move easyloggingpp into "external" directory - \#92 - Speed up CMake build process +- \#96 - Remove .a file in milvus/lib for docker-version - \#118 - Using shared_ptr instead of weak_ptr to avoid performance loss +>>>>>>> main/0.5.1 ## Feature - \#115 - Using new structure for tasktable diff --git a/README.md b/README.md index 3d1979be4ae560e23569e191529f2f9503c16827..f18328f7480d5ef13bbcf24c066e0f77d5605d15 100644 --- a/README.md +++ b/README.md @@ -1,9 +1,12 @@ + ![Milvuslogo](https://github.com/milvus-io/docs/blob/master/assets/milvus_logo.png) ![LICENSE](https://img.shields.io/badge/license-Apache--2.0-brightgreen) ![Language](https://img.shields.io/badge/language-C%2B%2B-blue) [![codebeat badge](https://codebeat.co/badges/e030a4f6-b126-4475-a938-4723d54ec3a7?style=plastic)](https://codebeat.co/projects/github-com-jinhai-cn-milvus-master) +![Release](https://img.shields.io/badge/release-v0.5.0-orange) +![Release_date](https://img.shields.io/badge/release_date-October-yellowgreen) - [Slack Community](https://join.slack.com/t/milvusio/shared_invite/enQtNzY1OTQ0NDI3NjMzLWNmYmM1NmNjOTQ5MGI5NDhhYmRhMGU5M2NhNzhhMDMzY2MzNDdlYjM5ODQ5MmE3ODFlYzU3YjJkNmVlNDQ2ZTk) - [Twitter](https://twitter.com/milvusio) @@ -21,7 +24,7 @@ Milvus is an open source similarity search engine for massive feature vectors. D Milvus provides stable Python, Java and C++ APIs. -Keep up-to-date with newest releases and latest updates by reading Milvus [release notes](https://milvus.io/docs/en/Releases/v0.5.0/). +Keep up-to-date with newest releases and latest updates by reading Milvus [release notes](https://milvus.io/docs/en/release/v0.5.0/). - Heterogeneous computing @@ -89,7 +92,7 @@ Use Docker to install Milvus is a breeze. See the [Milvus install guide](https:/ ```shell $ cd [Milvus sourcecode path]/core -./ubuntu_build_deps.sh +$ ./ubuntu_build_deps.sh ``` ##### Step 2 Build @@ -167,7 +170,7 @@ Refer to [this link](https://github.com/milvus-io/milvus-sdk-java/tree/master/ex ## Contribution guidelines -Contributions are welcomed and greatly appreciated. If you want to contribute to Milvus, please read our [contribution guidelines](CONTRIBUTING.md). This project adheres to the [code of conduct](CODE_OF_CONDUCT.md) of Milvus. By participating, you are expected to uphold this code. +Contributions are welcomed and greatly appreciated. Please read our [contribution guidelines](CONTRIBUTING.md) for detailed contribution workflow. This project adheres to the [code of conduct](CODE_OF_CONDUCT.md) of Milvus. By participating, you are expected to uphold this code. We use [GitHub issues](https://github.com/milvus-io/milvus/issues/new/choose) to track issues and bugs. For general questions and public discussions, please join our community. @@ -196,5 +199,6 @@ Please read our [roadmap](https://milvus.io/docs/en/roadmap/) to learn about upc ## License -[Apache 2.0 license](LICENSE) +[Apache License 2.0](LICENSE) + diff --git a/ci/jenkins/Jenkinsfile b/ci/jenkins/Jenkinsfile index 0a16c7f2c3a367300627fa33fd45d5ac0419924f..67bff5ac1ed780cc20004b1f7eb9770b36bcbc64 100644 --- a/ci/jenkins/Jenkinsfile +++ b/ci/jenkins/Jenkinsfile @@ -1,10 +1,14 @@ -String cron_string = BRANCH_NAME == "master" ? "H 0 * * *" : "" -cron_string = BRANCH_NAME == "0.5.1" ? "H 1 * * *" : cron_string +String cron_timezone = "TZ=Asia/Shanghai" +String cron_string = BRANCH_NAME == "master" ? "H 0 * * * " : "" +cron_string = BRANCH_NAME == "0.5.1" ? "H 1 * * * " : cron_string pipeline { agent none - triggers { cron(cron_string) } + triggers { + cron """${cron_timezone} + ${cron_string}""" + } options { timestamps() diff --git a/ci/jenkins/jenkinsfile/deploySingle2Dev.groovy b/ci/jenkins/jenkinsfile/deploySingle2Dev.groovy index 738c714a0c7f35b9f3ec6fc17b3d812c8b0755bc..bc6c6f443812d8ec778dab361896b7ff0d29f764 100644 --- a/ci/jenkins/jenkinsfile/deploySingle2Dev.groovy +++ b/ci/jenkins/jenkinsfile/deploySingle2Dev.groovy @@ -3,7 +3,7 @@ sh 'helm repo update' dir ('milvus-helm') { checkout([$class: 'GitSCM', branches: [[name: "0.5.0"]], doGenerateSubmoduleConfigurations: false, extensions: [], submoduleCfg: [], userRemoteConfigs: [[credentialsId: "${params.GIT_CREDENTIALS_ID}", url: "https://github.com/milvus-io/milvus-helm.git", name: 'origin', refspec: "+refs/heads/0.5.0:refs/remotes/origin/0.5.0"]]]) dir ("milvus-gpu") { - sh "helm install --wait --timeout 300 --set engine.image.tag=${DOCKER_VERSION} --set expose.type=clusterIP --name ${env.PIPELINE_NAME}-${env.BUILD_NUMBER}-single-gpu -f ci/values.yaml -f ci/filebeat/values.yaml --namespace milvus ." + sh "helm install --wait --timeout 300 --set engine.image.tag=${DOCKER_VERSION} --set expose.type=clusterIP --name ${env.PIPELINE_NAME}-${env.BUILD_NUMBER}-single-gpu -f ci/db_backend/sqlite_values.yaml -f ci/filebeat/values.yaml --namespace milvus ." } } diff --git a/ci/jenkins/jenkinsfile/singleDevTest.groovy b/ci/jenkins/jenkinsfile/singleDevTest.groovy index adfadc927198feef25edf4489eb2956b9b23941d..16fe65a9b3b4dd400ccee917a0dad3adac109645 100644 --- a/ci/jenkins/jenkinsfile/singleDevTest.groovy +++ b/ci/jenkins/jenkinsfile/singleDevTest.groovy @@ -4,19 +4,21 @@ timeout(time: 60, unit: 'MINUTES') { sh "pytest . --alluredir=\"test_out/dev/single/sqlite\" --level=1 --ip ${env.PIPELINE_NAME}-${env.BUILD_NUMBER}-single-gpu-milvus-gpu-engine.milvus.svc.cluster.local" } // mysql database backend test - load "${env.WORKSPACE}/ci/jenkins/jenkinsfile/cleanupSingleDev.groovy" + // load "${env.WORKSPACE}/ci/jenkins/jenkinsfile/cleanupSingleDev.groovy" - if (!fileExists('milvus-helm')) { - dir ("milvus-helm") { - checkout([$class: 'GitSCM', branches: [[name: "0.5.0"]], doGenerateSubmoduleConfigurations: false, extensions: [], submoduleCfg: [], userRemoteConfigs: [[credentialsId: "${params.GIT_CREDENTIALS_ID}", url: "https://github.com/milvus-io/milvus-helm.git", name: 'origin', refspec: "+refs/heads/0.5.0:refs/remotes/origin/0.5.0"]]]) - } - } - dir ("milvus-helm") { - dir ("milvus-gpu") { - sh "helm install --wait --timeout 300 --set engine.image.tag=${DOCKER_VERSION} --set expose.type=clusterIP --name ${env.PIPELINE_NAME}-${env.BUILD_NUMBER}-single-gpu -f ci/db_backend/mysql_values.yaml -f ci/filebeat/values.yaml --namespace milvus ." - } - } - dir ("tests/milvus_python_test") { - sh "pytest . --alluredir=\"test_out/dev/single/mysql\" --level=1 --ip ${env.PIPELINE_NAME}-${env.BUILD_NUMBER}-single-gpu-milvus-gpu-engine.milvus.svc.cluster.local" - } + // Remove mysql-version tests: 10-28 + + // if (!fileExists('milvus-helm')) { + // dir ("milvus-helm") { + // checkout([$class: 'GitSCM', branches: [[name: "0.5.0"]], doGenerateSubmoduleConfigurations: false, extensions: [], submoduleCfg: [], userRemoteConfigs: [[credentialsId: "${params.GIT_CREDENTIALS_ID}", url: "https://github.com/milvus-io/milvus-helm.git", name: 'origin', refspec: "+refs/heads/0.5.0:refs/remotes/origin/0.5.0"]]]) + // } + // } + // dir ("milvus-helm") { + // dir ("milvus-gpu") { + // sh "helm install --wait --timeout 300 --set engine.image.tag=${DOCKER_VERSION} --set expose.type=clusterIP --name ${env.PIPELINE_NAME}-${env.BUILD_NUMBER}-single-gpu -f ci/db_backend/mysql_values.yaml -f ci/filebeat/values.yaml --namespace milvus ." + // } + // } + // dir ("tests/milvus_python_test") { + // sh "pytest . --alluredir=\"test_out/dev/single/mysql\" --level=1 --ip ${env.PIPELINE_NAME}-${env.BUILD_NUMBER}-single-gpu-milvus-gpu-engine.milvus.svc.cluster.local" + // } } diff --git a/core/src/index/knowhere/CMakeLists.txt b/core/src/index/knowhere/CMakeLists.txt index 2a499dc1a39c780ece60001dd0e4cdbfe2fd3eac..bece9058a9b4ff14b7627b1c61ac24b339a28769 100644 --- a/core/src/index/knowhere/CMakeLists.txt +++ b/core/src/index/knowhere/CMakeLists.txt @@ -81,27 +81,6 @@ target_link_libraries( ${depend_libs} ) -INSTALL(TARGETS - knowhere - SPTAGLibStatic - DESTINATION - lib) - -INSTALL(FILES - ${ARROW_STATIC_LIB} - ${ARROW_PREFIX}/lib/libjemalloc_pic.a - ${FAISS_STATIC_LIB} - ${LAPACK_STATIC_LIB} - ${BLAS_STATIC_LIB} - DESTINATION - lib - ) - -INSTALL(FILES ${OPENBLAS_REAL_STATIC_LIB} - RENAME "libopenblas.a" - DESTINATION lib - ) - set(INDEX_INCLUDE_DIRS ${INDEX_SOURCE_DIR}/knowhere ${INDEX_SOURCE_DIR}/thirdparty @@ -112,17 +91,4 @@ set(INDEX_INCLUDE_DIRS ${LAPACK_INCLUDE_DIR} ) -set(INDEX_INCLUDE_DIRS ${INDEX_INCLUDE_DIRS} PARENT_SCOPE) - -#INSTALL(DIRECTORY -# ${INDEX_SOURCE_DIR}/include/knowhere -# ${ARROW_INCLUDE_DIR}/arrow -# ${FAISS_PREFIX}/include/faiss -# ${OPENBLAS_INCLUDE_DIR}/ -# DESTINATION -# include) -# -#INSTALL(DIRECTORY -# ${SPTAG_SOURCE_DIR}/AnnService/inc/ -# DESTINATION -# include/SPTAG/AnnService/inc) +set(INDEX_INCLUDE_DIRS ${INDEX_INCLUDE_DIRS} PARENT_SCOPE) \ No newline at end of file diff --git a/tests/milvus_doc_test/markdown-link-extractor.py b/tests/milvus_doc_test/markdown-link-extractor.py new file mode 100644 index 0000000000000000000000000000000000000000..3744089e6937ae1df575e71530c9908949c35829 --- /dev/null +++ b/tests/milvus_doc_test/markdown-link-extractor.py @@ -0,0 +1,420 @@ +# -*- coding: utf-8 -*- +# Using Python 3.x + +import urllib.request +import urllib.error +from pathlib import Path +import requests +import json +from urllib.parse import urlparse +import markdown +import os +from os.path import join, getsize +from bs4 import BeautifulSoup +import re +from sys import platform +import argparse + + +class LinksFromMarkdown(object): + + def __init__(self, repository): + self.dictionary = repository + + def extract_links_from_markdown(self, repository): + + + if platform == "linux" or platform == "linux2": + # linux + link_file = "../link_reports/" + "extracted_links.json" + dirName = "../link_reports" + elif platform == "darwin": + # OS X + link_file = "../link_reports/" + "extracted_links.json" + dirName = "../link_reports" + elif platform == "win32": + # Windows... + link_file = "..\\link_reports\\" + "extracted_links.json" + dirName = "..\\link_reports" + + # repository = "D:\\GithubRepo\\docs-master\\docs-master" + + + try: + # Create target Directory + os.mkdir(dirName) + print("Directory ", dirName, " Created ") + except FileExistsError: + print("Directory ", dirName, " already exists") + + md_files = [] + + for root, dirs, files in os.walk(repository): + # print(root, "consumes", end=" ") + # print(sum(getsize(join(root, name)) for name in files), end=" ") + # print("bytes in", len(files), "non-directory files") + if len(files) != 0: + # print(files) + for file in files: + if file.endswith(".md") or file.endswith(".MD") or file.endswith(".mD") or file.endswith(".Md"): + md_files.append(join(root, file)) + # elif file.endswith(".png") or file.endswith(".PNG"): + # pics.append((join(root, file))) + + # print(md_files) + # print(pics) + + a_href_list = [] + + for md_file in md_files: + with open(md_file, "r", encoding="utf-8") as f: + html = markdown.markdown(f.read()) + # print(html) + soup = BeautifulSoup(html, "lxml") + a_hrefs = [(x.get('href')) for x in soup.find_all("a")] + + a_href_list.append(a_hrefs) + # print(a_hrefs) + # print(md_file) + + # Generates a dictionary that indicates each MD file and links extracted from the MD file + dictionary = dict(zip(md_files, a_href_list)) + + with open(link_file, "w+", encoding="utf-8") as f: + json.dump(dictionary, f) + + return link_file + + + # print(dictionary) + +class CheckExtractedLinksFromMarkdown(object): + + def __init__(self, link_file): + self.link_file = link_file + + def check_extracted_links(self, link_file): + + if platform == "linux" or platform == "linux2": + # linux + report_name = "../link_reports/" + "link_validation_report.html" + + elif platform == "darwin": + # OS X + report_name = "../link_reports/" + "link_validation_report.html" + + elif platform == "win32": + # Windows... + report_name = "..\\link_reports\\" + "link_validation_report.html" + + html_code = """
This markdown file does not contain any links.
""") + else: + + head_code = """Link | Status | Markdown File |
---|---|---|
Invalid Link Number """ + str(invalid_counter) +""" | """ + """This link is not string, which indicates that your MD file may not be well-formed.""" + """ | """ + key + """ |
""" + """""" + link + """""" + """ | """ + str(status_code) + """ | """ + key + """ |
""" + """""" + link + """""" + """ | """ + str( + status_code) + """ | """ + key + """ |
""" + """""" + link + """""" + """ | """ + str( + status_code) + """ The URL looks good but the anchor link does not work or is not using an anchor tag.""" + """ | """ + key + """ |
""" + """""" + link + """""" + """ | """ + str( + status_code) + """ """ + str(http_error) + """ The URL looks good but the page then returns an HTTP error. | """ + key + """ |
""" + """""" + link + """""" + """ | """ + str( + status_code) + """ """ + str(url_error) + """ The URL looks good but the page then returns a URL error. | """ + key + """ |
""" + """""" + link + """""" + """ | """ + str( + status_code) + """ | """ + key + """ |
""" + """""" + link + """""" + """ | """ + str( + timeout_error) + """ | """ + key + """ |
""" + """""" + link + """""" + """ | """ + str( + connection_error) + """ | """ + key + """ |
""" + """""" + link + """""" + """ | """ + str( + http_error) + """ | """ + key + """ |
""" + link + """ | The file link looks good. | """ + key + """ |
""" + link + """ | The file link is broken. | """ + key + """ |
""" + link + """ | The file link looks good. | """ + key + """ |
""" + link + """ | The file link is broken. | """ + key + """ |
""" + link + """ | The anchor link looks good. | """ + key + """ |
""" + link + """ | The anchor link is broken. | """ + key + """ |
The number of failed links: """ + str(fail_count) + """. The number of passed links: """ + str(pass_count) + """ Pass rate: """ + str(float(pass_count/(pass_count+fail_count))*100) + '%') + f.write("""
Click the button to sort the table by parent page:
+ """) + f.write("""""") + f.write( + """Failed Links | Status Code | Parent Page |
---|
""" + """Refer to this link for detailed report.""" + """
""") + +# Create the parser +my_parser = argparse.ArgumentParser(description='Check the links for all markdown files of a folder') + +# Add the arguments +my_parser.add_argument('Path', + metavar='path', + type=str, + help='The path to the repository that contains all markdown files.') + +# Execute the parse_args() method +args = my_parser.parse_args() + +repository = args.Path + +# Get link JSON file +LinksFromMarkdown_Milvus = LinksFromMarkdown(repository) +link_file = LinksFromMarkdown_Milvus.extract_links_from_markdown(repository) + +# Generate link validation report +CheckExtractedLinksFromMarkdown_Milvus = CheckExtractedLinksFromMarkdown(link_file) +report_name = CheckExtractedLinksFromMarkdown_Milvus.check_extracted_links(link_file) + +# Generate report summary +GenerateReportSummary_Milvus = GenerateReportSummary(report_name) +GenerateReportSummary_Milvus.generate_report_summary(report_name) \ No newline at end of file diff --git a/tests/milvus_doc_test/requirements.txt b/tests/milvus_doc_test/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..e099aa161a4d29d9686314edab58f03e385d11ea --- /dev/null +++ b/tests/milvus_doc_test/requirements.txt @@ -0,0 +1,3 @@ +requests>=2.22.0 +markdown>=3.1.1 +beautifulsoup4>=4.8.1 diff --git a/tests/milvus_python_test/test_connect.py b/tests/milvus_python_test/test_connect.py index 5ec95390116813122b16e54e7d800dede2e48f20..96ce1d3bdfc85b352b9162b1f7c4d65dbed79b39 100644 --- a/tests/milvus_python_test/test_connect.py +++ b/tests/milvus_python_test/test_connect.py @@ -5,7 +5,6 @@ import threading from multiprocessing import Process from utils import * -__version__ = '0.5.0' CONNECT_TIMEOUT = 12 diff --git a/tests/milvus_python_test/test_index.py b/tests/milvus_python_test/test_index.py index 65716d45aa048414265ec34bfd99094626683247..269e6137da0f968d5489a999484cf3440e780e51 100644 --- a/tests/milvus_python_test/test_index.py +++ b/tests/milvus_python_test/test_index.py @@ -301,7 +301,7 @@ class TestIndexBase: vectors = gen_vectors(nq, dim) table_list = [] for i in range(10): - table_name = gen_unique_str('test_create_index_multi_tables') + table_name = gen_unique_str() table_list.append(table_name) param = {'table_name': table_name, 'dimension': dim, @@ -769,7 +769,7 @@ class TestIndexIP: vectors = gen_vectors(nq, dim) table_list = [] for i in range(10): - table_name = gen_unique_str('test_create_index_multi_tables') + table_name = gen_unique_str() table_list.append(table_name) param = {'table_name': table_name, 'dimension': dim, diff --git a/tests/milvus_python_test/test_ping.py b/tests/milvus_python_test/test_ping.py index a55559bc63a242ff8527e22075836e2b5f3cfa5c..d63ab93f1150636a1b024e6046186f278cbb64b4 100644 --- a/tests/milvus_python_test/test_ping.py +++ b/tests/milvus_python_test/test_ping.py @@ -1,7 +1,7 @@ import logging import pytest -__version__ = '0.5.0' +__version__ = '0.5.1' class TestPing: diff --git a/tests/milvus_python_test/test_table.py b/tests/milvus_python_test/test_table.py index 88f7caca3c26c525cb957a2b949bc5fedcf32b62..6af38bac15f19bf480b88f3178334028b9c94e71 100644 --- a/tests/milvus_python_test/test_table.py +++ b/tests/milvus_python_test/test_table.py @@ -806,7 +806,7 @@ def create_index(connect, **params): return status func_map = { - 0:has, + # 0:has, 1:show, 10:create_table, 11:describe,