Merge pull request #263 from primihub/develop

1.6.0

.bazelrc

@@ -6,6 +6,16 @@ build:linux --copt=-DENABLE_SSE
 build:linux --linkopt=-lstdc++fs
 build:linux --define microsoft-apsi=false
 
+#build:linux --strip=never
+#build:linux --copt -fno-sanitize-recover=all
+#build:linux --copt -fsanitize=address
+#build:linux --copt -Og
+#build:linux --copt -g
+#build:linux --copt -fno-omit-frame-pointer
+#build:linux --linkopt -fsanitize=address
+#build:linux --linkopt -static-libasan
+
+
 # Enable remote execution so actions are performed on the remote systems.
 build:remote --remote_executor=grpc://118.190.39.100:27941
 

.github/workflows/main.yml

@@ -143,7 +143,7 @@ jobs:
     - name: bazel build
       run: |
         # cc_binary
-        bazel build --config=linux :node :cli :opt_paillier_c2py 
+        bazel build --config=linux :node :cli :opt_paillier_c2py :linkcontext 
       
   
   build-on-mac_x86_64:
@@ -162,7 +162,7 @@ jobs:
       run: |
         # cc_binary
         ./pre_build.sh
-        bazel build --config=darwin_x86_64 --config=macos :node :cli :opt_paillier_c2py
+        bazel build --config=darwin_x86_64 --config=macos :node :cli :opt_paillier_c2py :linkcontext
         cp ./bazel-bin/node ./bazel-bin/primihub-node
         cp ./bazel-bin/cli ./bazel-bin/primihub-cli
         chmod u+x ./bazel-bin/primihub-node

3rdparty/bazel-rules-leveldb/LICENSE

-                                 Apache License
-                           Version 2.0, January 2004
-                        http://www.apache.org/licenses/
-
-   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
-
-   1. Definitions.
-
-      "License" shall mean the terms and conditions for use, reproduction,
-      and distribution as defined by Sections 1 through 9 of this document.
-
-      "Licensor" shall mean the copyright owner or entity authorized by
-      the copyright owner that is granting the License.
-
-      "Legal Entity" shall mean the union of the acting entity and all
-      other entities that control, are controlled by, or are under common
-      control with that entity. For the purposes of this definition,
-      "control" means (i) the power, direct or indirect, to cause the
-      direction or management of such entity, whether by contract or
-      otherwise, or (ii) ownership of fifty percent (50%) or more of the
-      outstanding shares, or (iii) beneficial ownership of such entity.
-
-      "You" (or "Your") shall mean an individual or Legal Entity
-      exercising permissions granted by this License.
-
-      "Source" form shall mean the preferred form for making modifications,
-      including but not limited to software source code, documentation
-      source, and configuration files.
-
-      "Object" form shall mean any form resulting from mechanical
-      transformation or translation of a Source form, including but
-      not limited to compiled object code, generated documentation,
-      and conversions to other media types.
-
-      "Work" shall mean the work of authorship, whether in Source or
-      Object form, made available under the License, as indicated by a
-      copyright notice that is included in or attached to the work
-      (an example is provided in the Appendix below).
-
-      "Derivative Works" shall mean any work, whether in Source or Object
-      form, that is based on (or derived from) the Work and for which the
-      editorial revisions, annotations, elaborations, or other modifications
-      represent, as a whole, an original work of authorship. For the purposes
-      of this License, Derivative Works shall not include works that remain
-      separable from, or merely link (or bind by name) to the interfaces of,
-      the Work and Derivative Works thereof.
-
-      "Contribution" shall mean any work of authorship, including
-      the original version of the Work and any modifications or additions
-      to that Work or Derivative Works thereof, that is intentionally
-      submitted to Licensor for inclusion in the Work by the copyright owner
-      or by an individual or Legal Entity authorized to submit on behalf of
-      the copyright owner. For the purposes of this definition, "submitted"
-      means any form of electronic, verbal, or written communication sent
-      to the Licensor or its representatives, including but not limited to
-      communication on electronic mailing lists, source code control systems,
-      and issue tracking systems that are managed by, or on behalf of, the
-      Licensor for the purpose of discussing and improving the Work, but
-      excluding communication that is conspicuously marked or otherwise
-      designated in writing by the copyright owner as "Not a Contribution."
-
-      "Contributor" shall mean Licensor and any individual or Legal Entity
-      on behalf of whom a Contribution has been received by Licensor and
-      subsequently incorporated within the Work.
-
-   2. Grant of Copyright License. Subject to the terms and conditions of
-      this License, each Contributor hereby grants to You a perpetual,
-      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
-      copyright license to reproduce, prepare Derivative Works of,
-      publicly display, publicly perform, sublicense, and distribute the
-      Work and such Derivative Works in Source or Object form.
-
-   3. Grant of Patent License. Subject to the terms and conditions of
-      this License, each Contributor hereby grants to You a perpetual,
-      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
-      (except as stated in this section) patent license to make, have made,
-      use, offer to sell, sell, import, and otherwise transfer the Work,
-      where such license applies only to those patent claims licensable
-      by such Contributor that are necessarily infringed by their
-      Contribution(s) alone or by combination of their Contribution(s)
-      with the Work to which such Contribution(s) was submitted. If You
-      institute patent litigation against any entity (including a
-      cross-claim or counterclaim in a lawsuit) alleging that the Work
-      or a Contribution incorporated within the Work constitutes direct
-      or contributory patent infringement, then any patent licenses
-      granted to You under this License for that Work shall terminate
-      as of the date such litigation is filed.
-
-   4. Redistribution. You may reproduce and distribute copies of the
-      Work or Derivative Works thereof in any medium, with or without
-      modifications, and in Source or Object form, provided that You
-      meet the following conditions:
-
-      (a) You must give any other recipients of the Work or
-          Derivative Works a copy of this License; and
-
-      (b) You must cause any modified files to carry prominent notices
-          stating that You changed the files; and
-
-      (c) You must retain, in the Source form of any Derivative Works
-          that You distribute, all copyright, patent, trademark, and
-          attribution notices from the Source form of the Work,
-          excluding those notices that do not pertain to any part of
-          the Derivative Works; and
-
-      (d) If the Work includes a "NOTICE" text file as part of its
-          distribution, then any Derivative Works that You distribute must
-          include a readable copy of the attribution notices contained
-          within such NOTICE file, excluding those notices that do not
-          pertain to any part of the Derivative Works, in at least one
-          of the following places: within a NOTICE text file distributed
-          as part of the Derivative Works; within the Source form or
-          documentation, if provided along with the Derivative Works; or,
-          within a display generated by the Derivative Works, if and
-          wherever such third-party notices normally appear. The contents
-          of the NOTICE file are for informational purposes only and
-          do not modify the License. You may add Your own attribution
-          notices within Derivative Works that You distribute, alongside
-          or as an addendum to the NOTICE text from the Work, provided
-          that such additional attribution notices cannot be construed
-          as modifying the License.
-
-      You may add Your own copyright statement to Your modifications and
-      may provide additional or different license terms and conditions
-      for use, reproduction, or distribution of Your modifications, or
-      for any such Derivative Works as a whole, provided Your use,
-      reproduction, and distribution of the Work otherwise complies with
-      the conditions stated in this License.
-
-   5. Submission of Contributions. Unless You explicitly state otherwise,
-      any Contribution intentionally submitted for inclusion in the Work
-      by You to the Licensor shall be under the terms and conditions of
-      this License, without any additional terms or conditions.
-      Notwithstanding the above, nothing herein shall supersede or modify
-      the terms of any separate license agreement you may have executed
-      with Licensor regarding such Contributions.
-
-   6. Trademarks. This License does not grant permission to use the trade
-      names, trademarks, service marks, or product names of the Licensor,
-      except as required for reasonable and customary use in describing the
-      origin of the Work and reproducing the content of the NOTICE file.
-
-   7. Disclaimer of Warranty. Unless required by applicable law or
-      agreed to in writing, Licensor provides the Work (and each
-      Contributor provides its Contributions) on an "AS IS" BASIS,
-      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
-      implied, including, without limitation, any warranties or conditions
-      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
-      PARTICULAR PURPOSE. You are solely responsible for determining the
-      appropriateness of using or redistributing the Work and assume any
-      risks associated with Your exercise of permissions under this License.
-
-   8. Limitation of Liability. In no event and under no legal theory,
-      whether in tort (including negligence), contract, or otherwise,
-      unless required by applicable law (such as deliberate and grossly
-      negligent acts) or agreed to in writing, shall any Contributor be
-      liable to You for damages, including any direct, indirect, special,
-      incidental, or consequential damages of any character arising as a
-      result of this License or out of the use or inability to use the
-      Work (including but not limited to damages for loss of goodwill,
-      work stoppage, computer failure or malfunction, or any and all
-      other commercial damages or losses), even if such Contributor
-      has been advised of the possibility of such damages.
-
-   9. Accepting Warranty or Additional Liability. While redistributing
-      the Work or Derivative Works thereof, You may choose to offer,
-      and charge a fee for, acceptance of support, warranty, indemnity,
-      or other liability obligations and/or rights consistent with this
-      License. However, in accepting such obligations, You may act only
-      on Your own behalf and on Your sole responsibility, not on behalf
-      of any other Contributor, and only if You agree to indemnify,
-      defend, and hold each Contributor harmless for any liability
-      incurred by, or claims asserted against, such Contributor by reason
-      of your accepting any such warranty or additional liability.
-
-   END OF TERMS AND CONDITIONS
-
-   APPENDIX: How to apply the Apache License to your work.
-
-      To apply the Apache License to your work, attach the following
-      boilerplate notice, with the fields enclosed by brackets "[]"
-      replaced with your own identifying information. (Don't include
-      the brackets!)  The text should be enclosed in the appropriate
-      comment syntax for the file format. We also recommend that a
-      file or class name and description of purpose be included on the
-      same "printed page" as the copyright notice for easier
-      identification within third-party archives.
-
-   Copyright [yyyy] [name of copyright owner]
-
-   Licensed under the Apache License, Version 2.0 (the "License");
-   you may not use this file except in compliance with the License.
-   You may obtain a copy of the License at
-
-       http://www.apache.org/licenses/LICENSE-2.0
-
-   Unless required by applicable law or agreed to in writing, software
-   distributed under the License is distributed on an "AS IS" BASIS,
-   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-   See the License for the specific language governing permissions and
-   limitations under the License.

3rdparty/bazel-rules-leveldb/README.md

-# Bazel build rules for LevelDB
-
-Follows a "repos/deps" pattern (in order to help with recursive dependencies). To use:
-
-1. Copy `bazel/repos.bzl` into your repository at `3rdparty/bazel-rules-leveldb/repos.bzl` and add an empty `BUILD` (or `BUILD.bazel`) to `3rdparty/bazel-rules-leveldb` as well.
-
-2. Copy all of the directories from `3rdparty` that you ***don't*** already have in ***your*** repository's `3rdparty` directory.
-
-3. Either ... add the following to your `WORKSPACE` (or `WORKSPACE.bazel`):
-
-```bazel
-load("//3rdparty/bazel-rules-leveldb:repos.bzl", leveldb_repos="repos")
-leveldb_repos()
-
-load("@com_github_3rdparty_bazel_rules_leveldb//bazel:deps.bzl", leveldb_deps="deps")
-leveldb_deps()
-```
-
-Or ... to simplify others depending on ***your*** repository, add the following to your `repos.bzl`:
-
-```bazel
-load("//3rdparty/bazel-rules-leveldb:repos.bzl", leveldb="repos")
-
-def repos():
-    leveldb()
-```
-
-And the following to your `deps.bzl`:
-
-```bazel
-load("@com_github_3rdparty_bazel_rules_leveldb//bazel:deps.bzl", leveldb="deps")
-
-def deps():
-    leveldb()
-```
-
-4. You can then use `@com_github_google_leveldb//:leveldb` in your target's `deps`.
-
-5. Repeat the steps starting at (1) at the desired version of this repository that you want to use.
\ No newline at end of file

3rdparty/bazel-rules-leveldb/bazel/deps.bzl

-"""Dependency specific initialization."""
-
-load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
-load("@rules_foreign_cc//foreign_cc:repositories.bzl", "rules_foreign_cc_dependencies")
-
-def deps(repo_mapping = {}):
-    rules_foreign_cc_dependencies()
-
-    if "com_github_google_leveldb" not in native.existing_rules():
-        http_archive(
-            name = "com_github_google_leveldb",
-            url = "https://github.com/google/leveldb/archive/refs/tags/1.23.zip",
-            sha256 = "a6fa7eebd11de709c46bf1501600ed98bf95439d6967963606cc964931ce906f",
-            strip_prefix = "leveldb-1.23",
-            repo_mapping = repo_mapping,
-            build_file = "@com_github_3rdparty_bazel_rules_leveldb//:BUILD.bazel",
-        )

3rdparty/bazel-rules-leveldb/bazel/repos.bzl

-"""Adds repositories/archives."""
-
-########################################################################
-# DO NOT EDIT THIS FILE unless you are inside the
-# https://github.com/3rdparty/bazel-rules-leveldb repository. If you
-# encounter it anywhere else it is because it has been copied there in
-# order to simplify adding transitive dependencies. If you want a
-# different version of bazel-rules-leveldb follow the Bazel build
-# instructions at https://github.com/3rdparty/bazel-rules-leveldb.
-########################################################################
-
-load("@bazel_tools//tools/build_defs/repo:git.bzl", "git_repository")
-load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
-
-def repos(external = True, repo_mapping = {}):
-    if "rules_foreign_cc" not in native.existing_rules():
-        http_archive(
-            name = "rules_foreign_cc",
-            url = "https://github.com/bazelbuild/rules_foreign_cc/archive/0.2.0.tar.gz",
-            sha256 = "d54742ffbdc6924f222d2179f0e10e911c5c659c4ae74158e9fe827aad862ac6",
-            strip_prefix = "rules_foreign_cc-0.2.0",
-            repo_mapping = repo_mapping,
-        )
-
-    if external and "com_github_3rdparty_bazel_rules_leveldb" not in native.existing_rules():
-        git_repository(
-            name = "com_github_3rdparty_bazel_rules_leveldb",
-            remote = "https://github.com/3rdparty/bazel-rules-leveldb",
-            commit = "a735c49c87f239b18ccbe7cfcfcd620566106202",
-            shallow_since = "1620093785 -0700",
-            repo_mapping = repo_mapping,
-        )

BUILD.bazel

@@ -202,6 +202,38 @@ cc_library(
         ],
 )
 
+cc_library(
+    name = "endian_util",
+    hdrs = [
+        "src/primihub/util/endian_util.h",
+    ],
+    copts = C_OPT,
+    linkopts = LINK_OPTS,
+    linkstatic = False,
+)
+
+cc_library(
+    name = "communication_lib",
+    srcs = [
+        "src/primihub/util/network/link_context.cc",
+        "src/primihub/util/network/grpc_link_context.cc",
+    ],
+    hdrs = [
+        "src/primihub/util/network/link_factory.h",
+        "src/primihub/util/network/link_context.h",
+        "src/primihub/util/network/grpc_link_context.h",
+    ],
+    copts = C_OPT,
+    linkopts = LINK_OPTS,
+    linkstatic = False,
+    deps = [
+        "@com_github_glog_glog//:glog",
+        "@com_github_grpc_grpc//:grpc++",
+        ":common_lib",
+        ":worker_proto",
+    ],
+)
+
 cc_library(
   name = "network_lib",
   srcs = glob([
@@ -603,12 +635,13 @@ cc_library(
             "src/primihub/task/semantic/mpc_task.cc",
             "src/primihub/task/semantic/tee_task.cc",
             "src/primihub/task/semantic/task.cc",
-            "src/primihub/task/semantic/psi_client_task.cc",
             "src/primihub/task/semantic/private_server_base.cc",
             "src/primihub/task/semantic/fl_task.cc",
-            "src/primihub/task/semantic/psi_server_task.cc",
             "src/primihub/task/semantic/keyword_pir_client_task.cc",
             "src/primihub/task/semantic/keyword_pir_server_task.cc",
+            "src/primihub/task/semantic/psi_ecdh_task.cc",
+            "src/primihub/task/semantic/psi_task.cc",
+            "src/primihub/task/semantic/task_context.cc",
          ]),
         "//conditions:default": glob([
             "src/primihub/task/language/proto_parser.cc",
@@ -626,10 +659,11 @@ cc_library(
             "src/primihub/task/semantic/mpc_task.cc",
             "src/primihub/task/semantic/tee_task.cc",
             "src/primihub/task/semantic/task.cc",
-            "src/primihub/task/semantic/psi_client_task.cc",
             "src/primihub/task/semantic/private_server_base.cc",
             "src/primihub/task/semantic/fl_task.cc",
-            "src/primihub/task/semantic/psi_server_task.cc",
+            "src/primihub/task/semantic/psi_ecdh_task.cc",
+            "src/primihub/task/semantic/psi_task.cc",
+             "src/primihub/task/semantic/task_context.cc",
         ]),
     }),
     hdrs = select({
@@ -650,16 +684,17 @@ cc_library(
             "src/primihub/task/semantic/scheduler/psi_scheduler.h",
             "src/primihub/task/semantic/scheduler/aby3_scheduler.h",
             "src/primihub/task/semantic/scheduler/tee_scheduler.h",
-            "src/primihub/task/semantic/psi_server_task.h",
             "src/primihub/task/semantic/psi_kkrt_task.h",
             "src/primihub/task/semantic/keyword_pir_client_task.h",
             "src/primihub/task/semantic/keyword_pir_server_task.h",
-            "src/primihub/task/semantic/psi_client_task.h",
             "src/primihub/task/semantic/factory.h",
             "src/primihub/task/semantic/mpc_task.h",
             "src/primihub/task/semantic/fl_task.h",
             "src/primihub/task/semantic/tee_task.h",
             "src/primihub/util/util.h",
+            "src/primihub/task/semantic/psi_ecdh_task.h",
+            "src/primihub/task/semantic/psi_task.h",
+            "src/primihub/task/semantic/task_context.h",
         ]),
         "//conditions:default": glob([
             "src/primihub/task/language/proto_parser.h",
@@ -678,14 +713,15 @@ cc_library(
             "src/primihub/task/semantic/scheduler/psi_scheduler.h",
             "src/primihub/task/semantic/scheduler/aby3_scheduler.h",
             "src/primihub/task/semantic/scheduler/tee_scheduler.h",
-            "src/primihub/task/semantic/psi_server_task.h",
             "src/primihub/task/semantic/psi_kkrt_task.h",
-            "src/primihub/task/semantic/psi_client_task.h",
             "src/primihub/task/semantic/factory.h",
             "src/primihub/task/semantic/mpc_task.h",
             "src/primihub/task/semantic/fl_task.h",
             "src/primihub/task/semantic/tee_task.h",
             "src/primihub/util/util.h",
+            "src/primihub/task/semantic/psi_ecdh_task.h",
+            "src/primihub/task/semantic/psi_task.h",
+            "src/primihub/task/semantic/task_context.h",
         ]),
     }),
     copts = C_OPT + select({
@@ -702,7 +738,10 @@ cc_library(
     # don't remove it.
     linkopts = LINK_OPTS + PLACEHOLDER-PYTHON3.X-CONFIG,
     linkstatic = False,
-    deps = TASK_LIB_DEPS,
+    deps = TASK_LIB_DEPS + [
+        ":communication_lib",
+        ":endian_util",
+    ],
 )
 
 cc_library(
@@ -1077,6 +1116,22 @@ py_library(
     data = ["primihub_channel.so"],
 )
 
+pybind_extension(
+    name = "linkcontext",
+    srcs = [
+        "src/primihub/pybind_warpper/network/link_context_wrapper.cc",
+    ],
+    deps = [
+        ":util_lib",
+        ":communication_lib",
+    ],
+)
+
+py_library(
+    name = "linkcontext",
+    data = ["linkcontext.so"],
+)
+
 pybind_extension(
     name = "pybind_mpc",
     srcs = [
@@ -1208,10 +1263,10 @@ cc_binary(
         ":common_lib",
         ":util_lib",
         "@openssl",
+        "@com_github_stduuid//:stduuid_lib",
     ],
 )
 
-
 ## end of primihub
 
 ## unit tests

Dockerfile

@@ -20,7 +20,7 @@ ADD . /src
 
 # Bazel build primihub-node & primihub-cli & paillier shared library
 RUN bash pre_build.sh \
-  && bazel build --cxxopt=-D_AMD64_ --config=linux --define microsoft-apsi=true :node :cli :opt_paillier_c2py
+  && bazel build --config=linux --define cpu=amd64 --define microsoft-apsi=true :node :cli :opt_paillier_c2py :linkcontext
 
 FROM ubuntu:20.04 as runner
 
@@ -53,12 +53,19 @@ RUN mkdir -p src/primihub/protos data log
 COPY --from=builder /src/python ./python
 COPY --from=builder /src/src/primihub/protos/ ./src/primihub/protos/
 
+# Copy opt_paillier_c2py.so to /app/python, this enable setup.py find it.
+RUN cp $TARGET_PATH/opt_paillier_c2py.so /app/python/
+# Copy linkcontext.so to /app/python, this enable setup.py find it.
+RUN cp $TARGET_PATH/linkcontext.so /app/python/
+
+# The setup.py will copy opt_paillier_c2py.so to python library path.
 WORKDIR /app/python
 RUN python3 -m pip install --upgrade pip \
   && python3 -m pip install -r requirements.txt \
-  && python3 setup.py install \
-  && python3 setup.py solib --solib-path $TARGET_PATH 
+  && python3 setup.py install
 
+RUN rm -rf /app/python/opt_paillier_c2py.so
+RUN rm -rf /app/python/linkcontext.so
 WORKDIR /app
 
 # gRPC server port

Dockerfile.local

@@ -30,12 +30,13 @@ COPY ./config ./config
 # Copy primihub python sources to /app and setup to system python3
 COPY ./python ./python
 COPY ./src/primihub/protos/ ./src/primihub/protos/
+RUN cp $TARGET_PATH/opt_paillier_c2py.so /app/python/
+RUN cp $TARGET_PATH/linkcontext.so /app/python/
 
 WORKDIR /app/python
 RUN python3 -m pip install --upgrade pip \
-  && python3 -m pip install -r requirements.txt -i https://mirrors.aliyun.com/pypi/simple \
-  && python3 setup.py develop \
-  && python3 setup.py solib --solib-path $TARGET_PATH 
+  && python3 -m pip install -r requirements.txt -i https://pypi.douban.com/simple/ \
+  && python3 setup.py develop
 
 WORKDIR /app
 # ENV PYTHONPATH=/usr/lib/python3.9/site-packages/:$TARGET_PATH

Makefile

+
+linux_x86_64:
+	#bazel build --config=linux --define cpu=amd64  --define microsoft-apsi=true //:node //:cli
+	bazel build --config=linux --define cpu=amd64  --define microsoft-apsi=true //:node //:cli //:linkcontext //:opt_paillier_c2py
+
+macos_arm64:
+	bazel build --config=macos --define cpu=arm64 --define microsoft-apsi=true //:node //:cli //:opt_paillier_c2py //:linkcontext

WORKSPACE

-workspace(name = "primihub")
-
-load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
-load("@bazel_tools//tools/build_defs/repo:git.bzl", "git_repository", "new_git_repository")
-
-
-_ALL_CONTENT = """\
-filegroup(
-    name = "all_srcs",
-    srcs = glob(["**"]),
-    visibility = ["//visibility:public"],
-)
-"""
-
-
-http_archive(
-    name = "rules_foreign_cc",
-    sha256 = "6041f1374ff32ba711564374ad8e007aef77f71561a7ce784123b9b4b88614fc",
-    strip_prefix = "rules_foreign_cc-0.8.0",
-    url = "https://github.com/bazelbuild/rules_foreign_cc/archive/0.8.0.tar.gz",
-)
-
-
-load("@rules_foreign_cc//foreign_cc:repositories.bzl", "rules_foreign_cc_dependencies")
-
-rules_foreign_cc_dependencies()
-
-http_archive(
-    name = "openssl",
-    url = "https://github.com/openssl/openssl/archive/refs/tags/OpenSSL_1_1_1o.tar.gz",
-    #sha256 = "f56dd7d81ce8d3e395f83285bd700a1098ed5a4cb0a81ce9522e41e6db7e0389",
-    strip_prefix = "openssl-OpenSSL_1_1_1o",
-    build_file = "//bazel:openssl.BUILD",
-)
-
-git_repository(
-    name = "com_github_nelhage_rules_boost",
-    # commit = "1e3a69bf2d5cd10c34b74f066054cd335d033d71",
-    branch = "master",
-    remote = "https://github.com/primihub/rules_boost.git",
-    # shallow_since = "1591047380 -0700",
-)
-
-load("@com_github_nelhage_rules_boost//:boost/boost.bzl", "boost_deps")
-boost_deps()
-
-http_archive(
-  name="eigen",
-  build_file_content = _ALL_CONTENT,
-  strip_prefix = "eigen-3.4",
-  urls=["https://gitlab.com/libeigen/eigen/-/archive/3.4/eigen-3.4.tar.bz2"],
-  sha256 = "a6f7aaa7b19c289dfeb33281e1954f19bf2ba1c6cae2c182354d820f535abef8",
-)
-
-new_git_repository(
-    name = "lib_function2",
-    build_file = "//bazel:BUILD.function2",
-    remote = "https://github.com/Naios/function2.git",
-    commit = "b8cf935d096a87a645534e5c1015ee80960fe4de",
-    shallow_since = "1616573746 +0100",
-)
-
-
-new_git_repository(
-    name = "arrow",
-    build_file = "//bazel:BUILD.arrow",
-    patch_cmds = [
-        # TODO: Remove the fowllowing once arrow issue is resolved.
-        """sed -i.bak 's/type_traits/std::max<int16_t>(sizeof(int16_t), type_traits/g' cpp/src/parquet/column_reader.cc""",
-        """sed -i.bak 's/value_byte_size/value_byte_size)/g' cpp/src/parquet/column_reader.cc""",
-    ],
-    branch = "release-4.0.0",
-    remote = "https://github.com/primihub/arrow.git",
-)
-
-
-# grpc with openssl
-git_repository(
-    name = "com_github_grpc_grpc",
-    remote = "https://github.com/primihub/grpc.git",
-    commit = "8838117b07a4faac3a6baaf645a490987b2a12b1",
-)
-
-load("@com_github_grpc_grpc//bazel:grpc_deps.bzl", "grpc_deps")
-
-# Includes boringssl, and other dependencies.
-grpc_deps()
-
-load("@com_github_grpc_grpc//bazel:grpc_extra_deps.bzl", "grpc_extra_deps")
-
-# Loads transitive dependencies of GRPC.
-grpc_extra_deps()
-
-http_archive(
-    name = "com_github_glog_glog",
-    # sha256 = "cbba86b5a63063999e0fc86de620a3ad22d6fd2aa5948bff4995dcd851074a0b",
-    strip_prefix = "glog-c8f8135a5720aee7de8328b42e4c43f8aa2e60aa",
-    urls = ["https://github.com/google/glog/archive/c8f8135a5720aee7de8328b42e4c43f8aa2e60aa.zip"],
-)
-
-http_archive(
-    name = "com_github_google_flatbuffers",
-    url = "https://github.com/google/flatbuffers/archive/refs/tags/v2.0.0.tar.gz",
-    strip_prefix = "flatbuffers-2.0.0",
-    sha256 = "9ddb9031798f4f8754d00fca2f1a68ecf9d0f83dfac7239af1311e4fd9a565c4",
-)
-
-# gflags Needed for glog
-http_archive(
-    name = "com_github_gflags_gflags",
-    sha256 = "34af2f15cf7367513b352bdcd2493ab14ce43692d2dcd9dfc499492966c64dcf",
-    strip_prefix = "gflags-2.2.2",
-    urls = [
-        "https://github.com/gflags/gflags/archive/v2.2.2.tar.gz",
-    ],
-)
-
-# Abseil C++ libraries
-git_repository(
-    name = "com_google_absl",
-    remote = "https://github.com/abseil/abseil-cpp.git",
-    commit = "0f3bb466b868b523cf1dc9b2aaaed65c77b28862",
-    shallow_since = "1603283562 -0400",
-)
-
-# googletest
-http_archive(
-    name = "com_google_googletest",
-    urls = ["https://github.com/google/googletest/archive/refs/tags/release-1.10.0.zip"],
-    sha256 = "94c634d499558a76fa649edb13721dce6e98fb1e7018dfaeba3cd7a083945e91",
-    strip_prefix = "googletest-release-1.10.0",
-)
-
-http_archive(
-    name = "bazel_common",
-    url = "https://github.com/google/bazel-common/archive/refs/heads/master.zip",
-    strip_prefix = "bazel-common-master",
-    sha256 = "7034b3fb6b3051d70f33853fff48b0e931b57e35c6a32bba0280c6f2b6d6ee0c",
-)
-
-http_archive(
-    name = "bazel_skylib",
-    strip_prefix = None,
-    url = "https://github.com/bazelbuild/bazel-skylib/releases/download/1.0.2/bazel-skylib-1.0.2.tar.gz",
-    sha256 = "97e70364e9249702246c0e9444bccdc4b847bed1eb03c5a3ece4f83dfe6abc44",
-)
-
-load("@bazel_skylib//lib:versions.bzl", "versions")
-
-versions.check(minimum_bazel_version = "5.0.0")
-
-# json
-http_archive(
-    name = "nlohmann_json",
-    strip_prefix = "json-3.9.1",
-    urls = ["https://github.com/nlohmann/json/archive/v3.9.1.tar.gz"],
-    sha256 = "4cf0df69731494668bdd6460ed8cb269b68de9c19ad8c27abc24cd72605b2d5b",
-    build_file = "//bazel:BUILD.nlohmann_json",
-)
-
-
-
-# pybind11 , bazel ref:https://github.com/pybind/pybind11_bazel
-# _PYBIND11_COMMIT = "72cbbf1fbc830e487e3012862b7b720001b70672"
-http_archive(
-  name = "pybind11_bazel",
-  strip_prefix = "pybind11_bazel-72cbbf1fbc830e487e3012862b7b720001b70672",
-  urls = [
-        "https://github.com/pybind/pybind11_bazel/archive/72cbbf1fbc830e487e3012862b7b720001b70672.zip",
-        "https://primihub.oss-cn-beijing.aliyuncs.com/pybind11_bazel-72cbbf1fbc830e487e3012862b7b720001b70672.zip"
-        ],
-)
-
-# We still require the pybind library.
-http_archive(
-  name = "pybind11",
-  build_file = "@pybind11_bazel//:pybind11.BUILD",
-  strip_prefix = "pybind11-2.9.2",
-  urls = ["https://github.com/pybind/pybind11/archive/refs/tags/v2.9.2.tar.gz"],
-)
-load("@pybind11_bazel//:python_configure.bzl", "python_configure")
-python_configure(name = "local_config_python")
-
-
-# ======== arrow dependencies  start ========
-
-http_archive(
-    name = "brotli",
-    build_file = "//bazel:brotli.BUILD",
-    sha256 = "4c61bfb0faca87219ea587326c467b95acb25555b53d1a421ffa3c8a9296ee2c",
-    strip_prefix = "brotli-1.0.7",
-    urls = [
-        "https://storage.googleapis.com/mirror.tensorflow.org/github.com/google/brotli/archive/v1.0.7.tar.gz",
-        "https://github.com/google/brotli/archive/refs/tags/v1.0.7.tar.gz",
-        "https://primihub.oss-cn-beijing.aliyuncs.com/brotli-1.0.7.tar.gz"
-    ],
-)
-
-
-http_archive(
-    name = "bzip2",
-    build_file = "//bazel:bzip2.BUILD",
-    sha256 = "ab5a03176ee106d3f0fa90e381da478ddae405918153cca248e682cd0c4a2269",
-    strip_prefix = "bzip2-1.0.8",
-    urls = [
-        "https://sourceware.org/pub/bzip2/bzip2-1.0.8.tar.gz",
-        "https://fossies.org/linux/misc/bzip2-1.0.8.tar.gz",
-        "https://primihub.oss-cn-beijing.aliyuncs.com/bzip2-1.0.8.tar.gz"
-    ],
-)
-
-
-http_archive(
-    name = "double-conversion",
-    sha256 = "a63ecb93182134ba4293fd5f22d6e08ca417caafa244afaa751cbfddf6415b13",
-    strip_prefix = "double-conversion-3.1.5",
-    urls = [
-        "https://storage.googleapis.com/mirror.tensorflow.org/github.com/google/double-conversion/archive/v3.1.5.tar.gz",
-        "https://github.com/google/double-conversion/archive/v3.1.5.tar.gz",
-        "https://primihub.oss-cn-beijing.aliyuncs.com/double-conversion-3.1.5.tar.gz",
-
-    ],
-)
-
-http_archive(
-    name = "lz4",
-    build_file = "//bazel:lz4.BUILD",
-    patch_cmds = [
-        """sed -i.bak 's/__attribute__ ((__visibility__ ("default")))//g' lib/lz4frame.h """,
-    ],
-    sha256 = "658ba6191fa44c92280d4aa2c271b0f4fbc0e34d249578dd05e50e76d0e5efcc",
-    strip_prefix = "lz4-1.9.2",
-    urls = [
-        # "https://storage.googleapis.com/mirror.tensorflow.org/github.com/lz4/lz4/archive/v1.9.2.tar.gz",
-        "https://github.com/lz4/lz4/archive/v1.9.2.tar.gz",
-        "https://primihub.oss-cn-beijing.aliyuncs.com/lz4-1.9.2.tar.gz"
-    ],
-)
-
-http_archive(
-    name = "rapidjson",
-    build_file = "//bazel:rapidjson.BUILD",
-    sha256 = "30bd2c428216e50400d493b38ca33a25efb1dd65f79dfc614ab0c957a3ac2c28",
-    strip_prefix = "rapidjson-418331e99f859f00bdc8306f69eba67e8693c55e",
-    urls = [
-        "https://github.com/miloyip/rapidjson/archive/418331e99f859f00bdc8306f69eba67e8693c55e.tar.gz",
-        "https://primihub.oss-cn-beijing.aliyuncs.com/rapidjson-418331e99f859f00bdc8306f69eba67e8693c55e.tar.gz"
-    ],
-)
-
-# Note: snappy is placed earlier as tensorflow's snappy does not include snappy-c
-http_archive(
-    name = "snappy",
-    build_file = "//bazel:snappy.BUILD",
-    sha256 = "16b677f07832a612b0836178db7f374e414f94657c138e6993cbfc5dcc58651f",
-    strip_prefix = "snappy-1.1.8",
-    urls = [
-        "https://github.com/google/snappy/archive/1.1.8.tar.gz",
-        "https://primihub.oss-cn-beijing.aliyuncs.com/snappy-1.1.8.tar.gz"
-    ],
-)
-
-http_archive(
-    name = "thrift",
-    build_file = "//bazel:thrift.BUILD",
-    sha256 = "5da60088e60984f4f0801deeea628d193c33cec621e78c8a43a5d8c4055f7ad9",
-    strip_prefix = "thrift-0.13.0",
-    urls = [
-        "https://github.com/apache/thrift/archive/v0.13.0.tar.gz",
-        "https://primihub.oss-cn-beijing.aliyuncs.com/thrift-0.13.0.tar.gz"
-    ],
-)
-
-http_archive(
-    name = "xsimd",
-    build_file = "//bazel:xsimd.BUILD",
-    sha256 = "45337317c7f238fe0d64bb5d5418d264a427efc53400ddf8e6a964b6bcb31ce9",
-    strip_prefix = "xsimd-7.5.0",
-    urls = [
-        "https://github.com/xtensor-stack/xsimd/archive/refs/tags/7.5.0.tar.gz",
-    ],
-)
-
-http_archive(
-    name = "zlib",
-    build_file = "//bazel:zlib.BUILD",
-    sha256 = "c3e5e9fdd5004dcb542feda5ee4f0ff0744628baf8ed2dd5d66f8ca1197cb1a1",
-    strip_prefix = "zlib-1.2.11",
-    urls = [
-        "https://zlib.net/zlib-1.2.11.tar.gz",
-        "https://primihub.oss-cn-beijing.aliyuncs.com/zlib-1.2.11.tar.gz"
-    ],
-)
-
-http_archive(
-    name = "zstd",
-    build_file = "//bazel:zstd.BUILD",
-    sha256 = "a364f5162c7d1a455cc915e8e3cf5f4bd8b75d09bc0f53965b0c9ca1383c52c8",
-    strip_prefix = "zstd-1.4.4",
-    urls = [
-        "https://github.com/facebook/zstd/archive/v1.4.4.tar.gz",
-        "https://primihub.oss-cn-beijing.aliyuncs.com/zstd-1.4.4.tar.gz"
-    ],
-)
-
-http_archive(
-    name = "xz",
-    build_file = "//bazel:xz.BUILD",
-    sha256 = "0d2b89629f13dd1a0602810529327195eff5f62a0142ccd65b903bc16a4ac78a",
-    strip_prefix = "xz-5.2.5",
-    urls = [
-        "https://github.com/xz-mirror/xz/archive/v5.2.5.tar.gz",
-        "https://primihub.oss-cn-beijing.aliyuncs.com/xz-5.2.5.tar.gz"
-    ],
-)
-
-http_archive(
-    name = "io_opentelemetry_cpp",
-    # sha256 = "<sha256>",
-    strip_prefix = "opentelemetry-cpp-1.0.1",
-    urls = [
-        "https://github.com/open-telemetry/opentelemetry-cpp/archive/refs/tags/v1.0.1.tar.gz",
-        "https://primihub.oss-cn-beijing.aliyuncs.com/opentelemetry-cpp-1.0.1.tar.gz"
-    ],
-)
-
-# Load OpenTelemetry dependencies after load.
-load("@io_opentelemetry_cpp//bazel:repository.bzl", "opentelemetry_cpp_deps")
-
-opentelemetry_cpp_deps()
-
-
-http_archive(
-    name = "com_google_protobuf",
-    # sha256 = "<sha256>",
-    # strip_prefix = "opentelemetry-cpp-1.0.1",
-    urls = [
-        "https://github.com/protocolbuffers/protobuf/archive/refs/tags/v3.20.0.tar.gz",
-        "https://primihub.oss-cn-beijing.aliyuncs.com/protobuf-3.20.0.tar.gz"
-    ],
-)
-
-
-git_repository(
-    name = "cares-bazel",
-    branch = "master",
-    remote = "https://github.com/hobo0cn/cares-bazel.git",
-    patch_cmds = [
-        "git submodule update --init --recursive",
-    ],
-)
-
-# fmt bazle, ref: https://fossies.org/linux/fmt/support/bazel/README.md
-load("@bazel_tools//tools/build_defs/repo:git.bzl", "git_repository")
-
-new_git_repository(
-    name = "fmt",
-    build_file = "//bazel:fmt.BUILD",
-    remote = "https://github.com/fmtlib/fmt.git",
-    tag = "6.1.2",
-)
-
-
-new_git_repository(
-    name = "libp2p",
-    build_file = "//bazel:libp2p.BUILD",
-    remote = "https://github.com/primihub/cpp-libp2p.git",
-    branch="master",
-)
-
-# soralog , need by libp2p
-# TODO need change to glog
-new_git_repository(
-    name = "com_openmpc_soralog",
-    build_file = "//bazel:soralog.BUILD",
-    remote = "https://github.com/primihub/soralog.git",
-    branch="master",
-)
-
-# sqlite, need by libp2p
-http_archive(
-    name = "com_github_soramitsu_sqlite",
-    build_file = "//bazel:sqlite.BUILD",
-    strip_prefix = "libp2p-sqlite-modern-cpp-3.2",
-    urls = [
-        "https://github.com/soramitsu/libp2p-sqlite-modern-cpp/archive/refs/tags/v3.2.tar.gz",
-        "https://primihub.oss-cn-beijing.aliyuncs.com/libp2p-sqlite-modern-cpp-3.2.tar.gz"],
-)
-
-#yaml-cpp, need by libp2p
-git_repository(
-    name = "com_github_jbeder_yaml_cpp",
-    remote = "https://github.com/jbeder/yaml-cpp.git",
-    tag="yaml-cpp-0.7.0",
-)
-# hat_trie , need by libp2p
-new_git_repository(
-    name = "com_github_masterjedy_hat_hrie",
-    build_file = "//bazel:hat_trie.BUILD",
-    remote = "https://github.com/masterjedy/hat-trie.git",
-    branch="master",
-)
-
-# boost di, used by libp2p
-http_archive(
-    name = "com_github_masterjedy_di",
-    build_file = "//bazel:di.BUILD",
-    strip_prefix = "di-1.1.0",
-    urls = [
-        "https://github.com/boost-ext/di/archive/refs/tags/v1.1.0.tar.gz",
-        "https://primihub.oss-cn-beijing.aliyuncs.com/di-1.1.0.tar.gz"
-    ],
-)
-
-# seal primihub fork from microsoft, used by crypTFlow2 and APSI
-new_git_repository(
-  name = "com_github_primihub_seal_40",
-  build_file = "//bazel:BUILD.seal",
-  remote = "https://github.com/primihub/SEAL.git",
-  branch = "main",
-)
-
-http_archive(
-    name = "com_github_gmp",
-    build_file = "//bazel:gmp.BUILD",
-    #sha256 = "87b565e89a9a684fe4ebeeddb8399dce2599f9c9049854ca8c0dfbdea0e21912",
-    strip_prefix = "gmp-6.2.1",
-    urls = ["https://gmplib.org/download/gmp/gmp-6.2.1.tar.xz"],
-)
-
-# Google dense_hash_set
-http_archive(
-    name = "google_sparsehash",
-    build_file = "//bazel:BUILD.sparsehash",
-    strip_prefix = "sparsehash-master",
-    urls = ["https://github.com/google/sparsehash/archive/master.zip"],
-)
-
-# libPSI start
-new_git_repository(
-    name = "osu_libpsi",
-    build_file = "//bazel:BUILD.libpsi",
-    commit = "4c5d5a3e49533c8547dcd4869e6a9842b6ce5b90",
-    remote = "https://github.com/primihub/libPSI.git",
-)
-
-# Google dense_hash_set
-http_archive(
-    name = "google_sparsehash",
-    build_file = "//bazel:BUILD.sparsehash",
-    strip_prefix = "sparsehash-master",
-    urls = ["https://github.com/google/sparsehash/archive/master.zip"],
-)
-
-# libote
-new_git_repository(
-    name = "osu_libote",
-    build_file = "//bazel:libOTe.BUILD",
-    commit = "f455eb7bf83034ebca6cab42e3aea9d9b33f8102",
-    remote = "https://github.com/primihub/libOTe.git",
-)
-
-new_git_repository(
-    name = "ladnir_cryptoTools",
-    build_file = "//bazel:cryptoTools.BUILD",
-    commit = "d52e05e2e803006256ddb66f48a0d51080f4b285",
-    remote = "https://github.com/primihub/cryptoTools.git",
-    # shallow_since = "1591047380 -0700",
-)
-
-new_git_repository(
-    name = "github_ntl",
-    build_file = "//bazel:ntl.BUILD",
-    commit = "6b23216ce6b24dd04446122c2200b6f031621965",
-    remote = "https://github.com/primihub/ntl.git",
-    # shallow_since = "1591047380 -0700",
-)
-
-new_git_repository(
-    name = "toolkit_relic",
-    build_file = "//bazel:BUILD.relic",
-    remote = "https://github.com/relic-toolkit/relic.git",
-    commit = "3f616ad64c3e63039277b8c90915607b6a2c504c",
-    shallow_since = "1581106153 -0800",
-)
-
-http_archive(
-    name = "com_github_redis_hiredis",
-    build_file = "//bazel:BUILD.hiredis",
-    urls = ["https://github.com/redis/hiredis/archive/392de5d7f97353485df1237872cb682842e8d83f.tar.gz"],
-    sha256 = "2101650d39a8f13293f263e9da242d2c6dee0cda08d343b2939ffe3d95cf3b8b",
-    strip_prefix = "hiredis-392de5d7f97353485df1237872cb682842e8d83f"
-)
-
-# libote
-#http_archive(
-#    name = "osu_libote",
-#    build_file = "//external:libOTe.BUILD",
-#    #sha256 = "6f021f24136eb177af38af3bf5d53b3592a1fe1e71d1c098318488a85b0afc3a",
-#    strip_prefix = "libOTe-master",
-#    urls = ["https://github.com/osu-crypto/libOTe/archive/refs/heads/master.zip"],
-#)
-
-# cryptoTools
-#http_archive(
-#    name = "ladnir_cryptoTools",
-#    build_file = "//external:cryptoTools.BUILD",
-#    #sha256 = "6f021f24136eb177af38af3bf5d53b3592a1fe1e71d1c098318488a85b0afc3a",
-#    strip_prefix = "cryptoTools-master",
-#    urls = ["https://github.com/ladnir/cryptoTools/archive/refs/heads/master.zip"],
-#)
-
-#PSI
-load("@bazel_tools//tools/build_defs/repo:git.bzl", "git_repository")
-
-git_repository(
-   name = "org_openmined_psi",
-   remote = "https://github.com/primihub/PSI.git",
-   branch = "master",
-   init_submodules = True,
-)
-
-load("@org_openmined_psi//private_set_intersection:preload.bzl", "psi_preload")
-
-psi_preload()
-
-load("@org_openmined_psi//private_set_intersection:deps.bzl", "psi_deps")
-
-psi_deps()
-
-
-load("@bazel_tools//tools/build_defs/repo:git.bzl", "git_repository")
-
-
-git_repository(
-   name = "org_openmined_pir",
-   remote = "https://github.com/primihub/PIR.git",
-   branch = "master",
-   init_submodules = True,
-)
-
-load("@org_openmined_pir//pir:preload.bzl", "pir_preload")
-
-pir_preload()
-
-load("@org_openmined_pir//pir:deps.bzl", "pir_deps")
-
-pir_deps()
-
-
-# leveldb as local kv store
-load("//3rdparty/bazel-rules-leveldb/bazel:repos.bzl", leveldb_repos="repos")
-leveldb_repos()
-
-load("//3rdparty/bazel-rules-leveldb/bazel:deps.bzl", leveldb_deps="deps")
-leveldb_deps()
-
-# APSI
-new_git_repository(
-    name = "mircrosoft_apsi",
-    build_file = "//bazel:BUILD.APSI",
-    branch = "main",
-    remote = "https://github.com/primihub/APSI.git",
-)
-
-# needed by APSI
-new_git_repository(
-    name = "com_microsoft_kuku",
-    build_file = "//bazel:BUILD.kuku",
-    branch = "2.1.0",
-    remote = "https://github.com/microsoft/Kuku.git",
-)
-# needed by APSI
-git_repository(
-    name = "jsoncpp",
-    branch = "master",
-    remote = "https://github.com/open-source-parsers/jsoncpp.git",
-)
-
-#support sqlite
-http_archive(
-  name = "com_github_sqlite_wrapper",
-  build_file = "//bazel:BUILD.sqlite",
-  sha256 = "57f91ed44ef205fe97b8c6586002fe6031cd02771d1c5d8415d9c515ad1532d1",
-  strip_prefix = "SQLiteCpp-3.2.0",
-  urls = [
-    "https://github.com/SRombauts/SQLiteCpp/archive/refs/tags/3.2.0.tar.gz",
-  ]
-)

WORKSPACE

+WORKSPACE_CN
\ No newline at end of file

WORKSPACE_CN

+workspace(name = "primihub")
+
+load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
+load("@bazel_tools//tools/build_defs/repo:git.bzl", "git_repository", "new_git_repository")
+
+
+_ALL_CONTENT = """\
+filegroup(
+    name = "all_srcs",
+    srcs = glob(["**"]),
+    visibility = ["//visibility:public"],
+)
+"""
+
+
+http_archive(
+    name = "rules_foreign_cc",
+    sha256 = "6041f1374ff32ba711564374ad8e007aef77f71561a7ce784123b9b4b88614fc",
+    strip_prefix = "rules_foreign_cc-0.8.0",
+    url = "https://primihub.oss-cn-beijing.aliyuncs.com/tools/rules_foreign_cc-0.8.0.tar.gz",
+)
+
+
+load("@rules_foreign_cc//foreign_cc:repositories.bzl", "rules_foreign_cc_dependencies")
+
+rules_foreign_cc_dependencies()
+
+http_archive(
+    name = "openssl",
+    url = "https://primihub.oss-cn-beijing.aliyuncs.com/tools/openssl-OpenSSL_1_1_1o.tar.gz",
+    #sha256 = "f56dd7d81ce8d3e395f83285bd700a1098ed5a4cb0a81ce9522e41e6db7e0389",
+    strip_prefix = "openssl-OpenSSL_1_1_1o",
+    build_file = "//bazel:openssl.BUILD",
+)
+
+git_repository(
+    name = "com_github_nelhage_rules_boost",
+    # commit = "1e3a69bf2d5cd10c34b74f066054cd335d033d71",
+    branch = "master",
+    remote = "https://gitee.com/primihub/rules_boost.git",
+    # shallow_since = "1591047380 -0700",
+)
+load("@com_github_nelhage_rules_boost//:boost/boost.bzl", "boost_deps")
+boost_deps()
+
+http_archive(
+  name="eigen",
+  build_file_content = _ALL_CONTENT,
+  strip_prefix = "eigen-3.4",
+  urls=["https://primihub.oss-cn-beijing.aliyuncs.com/tools/eigen-3.4.tar.bz2"],
+  sha256 = "a6f7aaa7b19c289dfeb33281e1954f19bf2ba1c6cae2c182354d820f535abef8",
+)
+
+new_git_repository(
+    name = "lib_function2",
+    build_file = "//bazel:BUILD.function2",
+    remote = "https://gitee.com/mirrors_Naios/function2.git",
+    commit = "b8cf935d096a87a645534e5c1015ee80960fe4de",
+    shallow_since = "1616573746 +0100",
+)
+
+
+new_git_repository(
+    name = "arrow",
+    build_file = "//bazel:BUILD.arrow",
+    patch_cmds = [
+        # TODO: Remove the fowllowing once arrow issue is resolved.
+        """sed -i.bak 's/type_traits/std::max<int16_t>(sizeof(int16_t), type_traits/g' cpp/src/parquet/column_reader.cc""",
+        """sed -i.bak 's/value_byte_size/value_byte_size)/g' cpp/src/parquet/column_reader.cc""",
+    ],
+    branch = "release-4.0.0",
+    remote = "https://gitee.com/primihub/arrow.git",
+)
+
+
+# grpc with openssl
+git_repository(
+    name = "com_github_grpc_grpc",
+    remote = "https://gitee.com/primihub/grpc.git",
+    commit = "8838117b07a4faac3a6baaf645a490987b2a12b1",
+)
+
+load("@com_github_grpc_grpc//bazel:grpc_deps.bzl", "grpc_deps")
+
+# Includes boringssl, and other dependencies.
+grpc_deps()
+
+load("@com_github_grpc_grpc//bazel:grpc_extra_deps.bzl", "grpc_extra_deps")
+
+# Loads transitive dependencies of GRPC.
+grpc_extra_deps()
+
+http_archive(
+    name = "com_github_glog_glog",
+    # sha256 = "cbba86b5a63063999e0fc86de620a3ad22d6fd2aa5948bff4995dcd851074a0b",
+    strip_prefix = "glog-c8f8135a5720aee7de8328b42e4c43f8aa2e60aa",
+    urls = ["https://primihub.oss-cn-beijing.aliyuncs.com/tools/glog-c8f8135a5720aee7de8328b42e4c43f8aa2e60aa.zip"],
+)
+
+http_archive(
+    name = "com_github_google_flatbuffers",
+    url = "https://primihub.oss-cn-beijing.aliyuncs.com/tools/flatbuffers-2.0.0.tar.gz",
+    strip_prefix = "flatbuffers-2.0.0",
+    sha256 = "9ddb9031798f4f8754d00fca2f1a68ecf9d0f83dfac7239af1311e4fd9a565c4",
+)
+
+# gflags Needed for glog
+http_archive(
+    name = "com_github_gflags_gflags",
+    sha256 = "34af2f15cf7367513b352bdcd2493ab14ce43692d2dcd9dfc499492966c64dcf",
+    strip_prefix = "gflags-2.2.2",
+    urls = [
+        "https://primihub.oss-cn-beijing.aliyuncs.com/tools/gflags-2.2.2.tar.gz",
+    ],
+)
+
+# Abseil C++ libraries
+git_repository(
+    name = "com_google_absl",
+    remote = "https://gitee.com/primihub/abseil-cpp.git",
+    commit = "0f3bb466b868b523cf1dc9b2aaaed65c77b28862",
+    shallow_since = "1603283562 -0400",
+)
+
+# googletest
+http_archive(
+    name = "com_google_googletest",
+    urls = ["https://primihub.oss-cn-beijing.aliyuncs.com/tools/googletest-release-1.10.0.zip"],
+    sha256 = "94c634d499558a76fa649edb13721dce6e98fb1e7018dfaeba3cd7a083945e91",
+    strip_prefix = "googletest-release-1.10.0",
+)
+
+http_archive(
+    name = "bazel_common",
+    url = "https://primihub.oss-cn-beijing.aliyuncs.com/tools/bazel-common-master.zip",
+    strip_prefix = "bazel-common-master",
+    sha256 = "7034b3fb6b3051d70f33853fff48b0e931b57e35c6a32bba0280c6f2b6d6ee0c",
+)
+
+http_archive(
+    name = "bazel_skylib",
+    strip_prefix = None,
+    url = "https://primihub.oss-cn-beijing.aliyuncs.com/tools/bazel-skylib-1.0.2.tar.gz",
+    sha256 = "97e70364e9249702246c0e9444bccdc4b847bed1eb03c5a3ece4f83dfe6abc44",
+)
+
+load("@bazel_skylib//lib:versions.bzl", "versions")
+
+versions.check(minimum_bazel_version = "5.0.0")
+
+# json
+http_archive(
+    name = "nlohmann_json",
+    strip_prefix = "json-3.9.1",
+    urls = ["https://primihub.oss-cn-beijing.aliyuncs.com/tools/json-3.9.1.tar.gz"],
+    sha256 = "4cf0df69731494668bdd6460ed8cb269b68de9c19ad8c27abc24cd72605b2d5b",
+    build_file = "//bazel:BUILD.nlohmann_json",
+)
+
+
+
+# pybind11 , bazel ref:https://github.com/pybind/pybind11_bazel
+# _PYBIND11_COMMIT = "72cbbf1fbc830e487e3012862b7b720001b70672"
+http_archive(
+  name = "pybind11_bazel",
+  strip_prefix = "pybind11_bazel-72cbbf1fbc830e487e3012862b7b720001b70672",
+  urls = ["https://primihub.oss-cn-beijing.aliyuncs.com/pybind11_bazel-72cbbf1fbc830e487e3012862b7b720001b70672.zip"],
+)
+
+# We still require the pybind library.
+http_archive(
+  name = "pybind11",
+  build_file = "@pybind11_bazel//:pybind11.BUILD",
+  strip_prefix = "pybind11-2.9.2",
+  urls = ["https://primihub.oss-cn-beijing.aliyuncs.com/tools/pybind11-2.9.2.tar.gz"],
+)
+load("@pybind11_bazel//:python_configure.bzl", "python_configure")
+python_configure(name = "local_config_python")
+
+
+# ======== arrow dependencies  start ========
+
+http_archive(
+    name = "brotli",
+    build_file = "//bazel:brotli.BUILD",
+    sha256 = "4c61bfb0faca87219ea587326c467b95acb25555b53d1a421ffa3c8a9296ee2c",
+    strip_prefix = "brotli-1.0.7",
+    urls = [
+        "https://primihub.oss-cn-beijing.aliyuncs.com/brotli-1.0.7.tar.gz"
+    ],
+)
+
+
+http_archive(
+    name = "bzip2",
+    build_file = "//bazel:bzip2.BUILD",
+    sha256 = "ab5a03176ee106d3f0fa90e381da478ddae405918153cca248e682cd0c4a2269",
+    strip_prefix = "bzip2-1.0.8",
+    urls = [
+        "https://primihub.oss-cn-beijing.aliyuncs.com/bzip2-1.0.8.tar.gz"
+    ],
+)
+
+
+http_archive(
+    name = "double-conversion",
+    sha256 = "a63ecb93182134ba4293fd5f22d6e08ca417caafa244afaa751cbfddf6415b13",
+    strip_prefix = "double-conversion-3.1.5",
+    urls = [
+        "https://primihub.oss-cn-beijing.aliyuncs.com/double-conversion-3.1.5.tar.gz",
+    ],
+)
+
+http_archive(
+    name = "lz4",
+    build_file = "//bazel:lz4.BUILD",
+    patch_cmds = [
+        """sed -i.bak 's/__attribute__ ((__visibility__ ("default")))//g' lib/lz4frame.h """,
+    ],
+    sha256 = "658ba6191fa44c92280d4aa2c271b0f4fbc0e34d249578dd05e50e76d0e5efcc",
+    strip_prefix = "lz4-1.9.2",
+    urls = [
+        "https://primihub.oss-cn-beijing.aliyuncs.com/lz4-1.9.2.tar.gz"
+    ],
+)
+
+http_archive(
+    name = "rapidjson",
+    build_file = "//bazel:rapidjson.BUILD",
+    sha256 = "30bd2c428216e50400d493b38ca33a25efb1dd65f79dfc614ab0c957a3ac2c28",
+    strip_prefix = "rapidjson-418331e99f859f00bdc8306f69eba67e8693c55e",
+    urls = [
+        "https://primihub.oss-cn-beijing.aliyuncs.com/rapidjson-418331e99f859f00bdc8306f69eba67e8693c55e.tar.gz"
+    ],
+)
+
+# Note: snappy is placed earlier as tensorflow's snappy does not include snappy-c
+http_archive(
+    name = "snappy",
+    build_file = "//bazel:snappy.BUILD",
+    sha256 = "16b677f07832a612b0836178db7f374e414f94657c138e6993cbfc5dcc58651f",
+    strip_prefix = "snappy-1.1.8",
+    urls = [
+        "https://primihub.oss-cn-beijing.aliyuncs.com/snappy-1.1.8.tar.gz"
+    ],
+)
+
+http_archive(
+    name = "thrift",
+    build_file = "//bazel:thrift.BUILD",
+    sha256 = "5da60088e60984f4f0801deeea628d193c33cec621e78c8a43a5d8c4055f7ad9",
+    strip_prefix = "thrift-0.13.0",
+    urls = [
+        "https://primihub.oss-cn-beijing.aliyuncs.com/thrift-0.13.0.tar.gz"
+    ],
+)
+
+http_archive(
+    name = "xsimd",
+    build_file = "//bazel:xsimd.BUILD",
+    sha256 = "45337317c7f238fe0d64bb5d5418d264a427efc53400ddf8e6a964b6bcb31ce9",
+    strip_prefix = "xsimd-7.5.0",
+    urls = [
+        "https://primihub.oss-cn-beijing.aliyuncs.com/tools/xsimd-7.5.0.tar.gz",
+    ],
+)
+
+http_archive(
+    name = "zlib",
+    build_file = "//bazel:zlib.BUILD",
+    sha256 = "c3e5e9fdd5004dcb542feda5ee4f0ff0744628baf8ed2dd5d66f8ca1197cb1a1",
+    strip_prefix = "zlib-1.2.11",
+    urls = [
+        "https://primihub.oss-cn-beijing.aliyuncs.com/zlib-1.2.11.tar.gz"
+    ],
+)
+
+http_archive(
+    name = "zstd",
+    build_file = "//bazel:zstd.BUILD",
+    sha256 = "a364f5162c7d1a455cc915e8e3cf5f4bd8b75d09bc0f53965b0c9ca1383c52c8",
+    strip_prefix = "zstd-1.4.4",
+    urls = [
+        "https://primihub.oss-cn-beijing.aliyuncs.com/zstd-1.4.4.tar.gz"
+    ],
+)
+
+http_archive(
+    name = "xz",
+    build_file = "//bazel:xz.BUILD",
+    sha256 = "0d2b89629f13dd1a0602810529327195eff5f62a0142ccd65b903bc16a4ac78a",
+    strip_prefix = "xz-5.2.5",
+    urls = [
+        "https://primihub.oss-cn-beijing.aliyuncs.com/xz-5.2.5.tar.gz"
+    ],
+)
+
+http_archive(
+    name = "io_opentelemetry_cpp",
+    # sha256 = "<sha256>",
+    strip_prefix = "opentelemetry-cpp-1.0.1",
+    urls = [
+        "https://primihub.oss-cn-beijing.aliyuncs.com/opentelemetry-cpp-1.0.1.tar.gz"
+    ],
+)
+
+# Load OpenTelemetry dependencies after load.
+load("@io_opentelemetry_cpp//bazel:repository.bzl", "opentelemetry_cpp_deps")
+
+opentelemetry_cpp_deps()
+
+
+http_archive(
+    name = "com_google_protobuf",
+    # sha256 = "<sha256>",
+    # strip_prefix = "opentelemetry-cpp-1.0.1",
+    urls = [
+        "https://primihub.oss-cn-beijing.aliyuncs.com/protobuf-3.20.0.tar.gz"
+    ],
+)
+
+
+git_repository(
+    name = "cares-bazel",
+    branch = "master",
+    remote = "https://gitee.com/primihub/cares-bazel.git",
+    patch_cmds = [
+        "git submodule update --init --recursive",
+    ],
+)
+
+# fmt bazle, ref: https://fossies.org/linux/fmt/support/bazel/README.md
+load("@bazel_tools//tools/build_defs/repo:git.bzl", "git_repository")
+
+new_git_repository(
+    name = "fmt",
+    build_file = "//bazel:fmt.BUILD",
+    remote = "https://gitee.com/primihub/fmt.git",
+    tag = "6.1.2",
+)
+
+
+new_git_repository(
+    name = "libp2p",
+    build_file = "//bazel:libp2p.BUILD",
+    remote = "https://gitee.com/primihub/cpp-libp2p.git",
+    branch="master",
+)
+
+# soralog , need by libp2p
+# TODO need change to glog
+new_git_repository(
+    name = "com_openmpc_soralog",
+    build_file = "//bazel:soralog.BUILD",
+    remote = "https://gitee.com/primihub/soralog.git",
+    branch="master",
+)
+
+# sqlite, need by libp2p
+http_archive(
+    name = "com_github_soramitsu_sqlite",
+    build_file = "//bazel:sqlite.BUILD",
+    strip_prefix = "libp2p-sqlite-modern-cpp-3.2",
+    urls = ["https://primihub.oss-cn-beijing.aliyuncs.com/libp2p-sqlite-modern-cpp-3.2.tar.gz"],
+)
+
+#yaml-cpp, need by libp2p
+git_repository(
+    name = "com_github_jbeder_yaml_cpp",
+    remote = "https://gitee.com/primihub/yaml-cpp.git",
+    tag="yaml-cpp-0.7.0",
+)
+
+# hat_trie , need by libp2p
+new_git_repository(
+    name = "com_github_masterjedy_hat_hrie",
+    build_file = "//bazel:hat_trie.BUILD",
+    remote = "https://gitee.com/primihub/hat-trie.git",
+    branch="master",
+)
+
+# boost di, used by libp2p
+http_archive(
+    name = "com_github_masterjedy_di",
+    build_file = "//bazel:di.BUILD",
+    strip_prefix = "di-1.1.0",
+    urls = [
+        "https://primihub.oss-cn-beijing.aliyuncs.com/di-1.1.0.tar.gz"
+    ],
+)
+
+# seal primihub fork from microsoft, used by crypTFlow2 and APSI
+new_git_repository(
+  name = "com_github_primihub_seal_40",
+  build_file = "//bazel:BUILD.seal",
+  remote = "https://gitee.com/primihub/SEAL.git",
+  branch = "change_submodule_resource",
+)
+
+http_archive(
+    name = "com_github_gmp",
+    build_file = "//bazel:gmp.BUILD",
+    #sha256 = "87b565e89a9a684fe4ebeeddb8399dce2599f9c9049854ca8c0dfbdea0e21912",
+    strip_prefix = "gmp-6.2.1",
+    urls = ["https://primihub.oss-cn-beijing.aliyuncs.com/tools/gmp-6.2.1.tar.xz"],
+)
+
+# Google dense_hash_set
+http_archive(
+    name = "google_sparsehash",
+    build_file = "//bazel:BUILD.sparsehash",
+    strip_prefix = "sparsehash-master",
+    urls = ["https://primihub.oss-cn-beijing.aliyuncs.com/tools/sparsehash-master.zip"],
+)
+
+# libPSI start
+new_git_repository(
+    name = "osu_libpsi",
+    build_file = "//bazel:BUILD.libpsi",
+    commit = "4c5d5a3e49533c8547dcd4869e6a9842b6ce5b90",
+    remote = "https://gitee.com/primihub/libPSI.git",
+)
+
+# libote
+new_git_repository(
+    name = "osu_libote",
+    build_file = "//bazel:libOTe.BUILD",
+    commit = "f455eb7bf83034ebca6cab42e3aea9d9b33f8102",
+    remote = "https://gitee.com/primihub/libOTe.git",
+)
+
+new_git_repository(
+    name = "ladnir_cryptoTools",
+    build_file = "//bazel:cryptoTools.BUILD",
+    commit = "d52e05e2e803006256ddb66f48a0d51080f4b285",
+    remote = "https://gitee.com/primihub/cryptoTools.git",
+    # shallow_since = "1591047380 -0700",
+)
+
+new_git_repository(
+    name = "github_ntl",
+    build_file = "//bazel:ntl.BUILD",
+    commit = "6b23216ce6b24dd04446122c2200b6f031621965",
+    remote = "https://gitee.com/primihub/ntl.git",
+    # shallow_since = "1591047380 -0700",
+)
+
+new_git_repository(
+    name = "toolkit_relic",
+    build_file = "//bazel:BUILD.relic",
+    remote = "https://gitee.com/orzmzp/relic.git",
+    commit = "3f616ad64c3e63039277b8c90915607b6a2c504c",
+    shallow_since = "1581106153 -0800",
+)
+
+http_archive(
+    name = "com_github_redis_hiredis",
+    build_file = "//bazel:BUILD.hiredis",
+    urls = ["https://primihub.oss-cn-beijing.aliyuncs.com/tools/hiredis-392de5d7f97353485df1237872cb682842e8d83f.tar.gz"],
+    sha256 = "2101650d39a8f13293f263e9da242d2c6dee0cda08d343b2939ffe3d95cf3b8b",
+    strip_prefix = "hiredis-392de5d7f97353485df1237872cb682842e8d83f"
+)
+
+# libote
+#http_archive(
+#    name = "osu_libote",
+#    build_file = "//external:libOTe.BUILD",
+#    #sha256 = "6f021f24136eb177af38af3bf5d53b3592a1fe1e71d1c098318488a85b0afc3a",
+#    strip_prefix = "libOTe-master",
+#    urls = ["https://github.com/osu-crypto/libOTe/archive/refs/heads/master.zip"],
+#)
+
+# cryptoTools
+#http_archive(
+#    name = "ladnir_cryptoTools",
+#    build_file = "//external:cryptoTools.BUILD",
+#    #sha256 = "6f021f24136eb177af38af3bf5d53b3592a1fe1e71d1c098318488a85b0afc3a",
+#    strip_prefix = "cryptoTools-master",
+#    urls = ["https://github.com/ladnir/cryptoTools/archive/refs/heads/master.zip"],
+#)
+
+#PSI
+load("@bazel_tools//tools/build_defs/repo:git.bzl", "git_repository")
+
+git_repository(
+   name = "org_openmined_psi",
+   remote = "https://gitee.com/primihub/PSI.git",
+   branch = "master",
+   init_submodules = True,
+)
+
+load("@org_openmined_psi//private_set_intersection:preload.bzl", "psi_preload")
+
+psi_preload()
+
+load("@org_openmined_psi//private_set_intersection:deps.bzl", "psi_deps")
+
+psi_deps()
+
+
+load("@bazel_tools//tools/build_defs/repo:git.bzl", "git_repository")
+
+
+git_repository(
+   name = "org_openmined_pir",
+   remote = "https://gitee.com/primihub/PIR.git",
+   branch = "master",
+   init_submodules = True,
+)
+
+load("@org_openmined_pir//pir:preload.bzl", "pir_preload")
+
+pir_preload()
+
+load("@org_openmined_pir//pir:deps.bzl", "pir_deps")
+
+pir_deps()
+
+
+# leveldb as local kv store
+
+# load("//3rdparty/bazel-rules-leveldb/bazel:repos.bzl", leveldb_repos="repos")
+# leveldb_repos()
+
+# load("//3rdparty/bazel-rules-leveldb/bazel:deps.bzl", leveldb_deps="deps")
+# leveldb_deps()
+
+http_archive(
+    name = "com_github_google_leveldb",
+    build_file = "//bazel:BUILD.leveldb",
+    sha256 = "a6fa7eebd11de709c46bf1501600ed98bf95439d6967963606cc964931ce906f",
+    strip_prefix = "leveldb-1.23",
+    urls = [
+        "https://primihub.oss-cn-beijing.aliyuncs.com/tools/leveldb-1.23.zip"
+    ],
+)
+
+# APSI
+git_repository(
+   name = "mircrosoft_apsi",
+   branch = "bazel_version",
+   remote = "https://gitee.com/primihub/APSI.git",
+)
+
+load("@mircrosoft_apsi//bazel:repositories.bzl", "apsi_dependencies")
+apsi_dependencies("mircrosoft_apsi");
+
+# needed by APSI
+new_git_repository(
+    name = "com_microsoft_kuku",
+    build_file = "//bazel:BUILD.kuku",
+    branch = "2.1.0",
+    remote = "https://gitee.com/primihub/Kuku.git",
+)
+# needed by APSI
+git_repository(
+    name = "jsoncpp",
+    branch = "master",
+    remote = "https://gitee.com/primihub/jsoncpp.git",
+)
+
+#support sqlite
+http_archive(
+  name = "com_github_sqlite_wrapper",
+  build_file = "//bazel:BUILD.sqlite",
+  sha256 = "57f91ed44ef205fe97b8c6586002fe6031cd02771d1c5d8415d9c515ad1532d1",
+  strip_prefix = "SQLiteCpp-3.2.0",
+  urls = [
+    "https://primihub.oss-cn-beijing.aliyuncs.com/tools/SQLiteCpp-3.2.0.tar.gz",
+  ]
+)
+
+#generate uuid
+http_archive(
+  name = "com_github_stduuid",
+  build_file = "//bazel:BUILD.stduuid",
+  sha256 = "f554f6a9fe4d852fa217de6ab977afbf3f49e4a1dcb263afd61a94253c4c7a48",
+  strip_prefix = "stduuid-1.2.2",
+  urls = [
+    "https://primihub.oss-cn-beijing.aliyuncs.com/tools/stduuid-1.2.2.tar.gz",
+  ]
+)

WORKSPACE_GITHUB

+workspace(name = "primihub")
+
+load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
+load("@bazel_tools//tools/build_defs/repo:git.bzl", "git_repository", "new_git_repository")
+
+
+_ALL_CONTENT = """\
+filegroup(
+    name = "all_srcs",
+    srcs = glob(["**"]),
+    visibility = ["//visibility:public"],
+)
+"""
+
+
+http_archive(
+    name = "rules_foreign_cc",
+    sha256 = "6041f1374ff32ba711564374ad8e007aef77f71561a7ce784123b9b4b88614fc",
+    strip_prefix = "rules_foreign_cc-0.8.0",
+    url = "https://github.com/bazelbuild/rules_foreign_cc/archive/0.8.0.tar.gz",
+)
+
+
+load("@rules_foreign_cc//foreign_cc:repositories.bzl", "rules_foreign_cc_dependencies")
+
+rules_foreign_cc_dependencies()
+
+http_archive(
+    name = "openssl",
+    url = "https://github.com/openssl/openssl/archive/refs/tags/OpenSSL_1_1_1o.tar.gz",
+    #sha256 = "f56dd7d81ce8d3e395f83285bd700a1098ed5a4cb0a81ce9522e41e6db7e0389",
+    strip_prefix = "openssl-OpenSSL_1_1_1o",
+    build_file = "//bazel:openssl.BUILD",
+)
+
+git_repository(
+    name = "com_github_nelhage_rules_boost",
+    # commit = "1e3a69bf2d5cd10c34b74f066054cd335d033d71",
+    branch = "master",
+    remote = "https://github.com/primihub/rules_boost.git",
+    # shallow_since = "1591047380 -0700",
+)
+
+load("@com_github_nelhage_rules_boost//:boost/boost.bzl", "boost_deps")
+boost_deps()
+
+http_archive(
+  name="eigen",
+  build_file_content = _ALL_CONTENT,
+  strip_prefix = "eigen-3.4",
+  urls=["https://gitlab.com/libeigen/eigen/-/archive/3.4/eigen-3.4.tar.bz2"],
+  sha256 = "a6f7aaa7b19c289dfeb33281e1954f19bf2ba1c6cae2c182354d820f535abef8",
+)
+
+new_git_repository(
+    name = "lib_function2",
+    build_file = "//bazel:BUILD.function2",
+    remote = "https://github.com/Naios/function2.git",
+    commit = "b8cf935d096a87a645534e5c1015ee80960fe4de",
+    shallow_since = "1616573746 +0100",
+)
+
+
+new_git_repository(
+    name = "arrow",
+    build_file = "//bazel:BUILD.arrow",
+    patch_cmds = [
+        # TODO: Remove the fowllowing once arrow issue is resolved.
+        """sed -i.bak 's/type_traits/std::max<int16_t>(sizeof(int16_t), type_traits/g' cpp/src/parquet/column_reader.cc""",
+        """sed -i.bak 's/value_byte_size/value_byte_size)/g' cpp/src/parquet/column_reader.cc""",
+    ],
+    branch = "release-4.0.0",
+    remote = "https://github.com/primihub/arrow.git",
+)
+
+
+# grpc with openssl
+git_repository(
+    name = "com_github_grpc_grpc",
+    remote = "https://github.com/primihub/grpc.git",
+    commit = "8838117b07a4faac3a6baaf645a490987b2a12b1",
+)
+
+load("@com_github_grpc_grpc//bazel:grpc_deps.bzl", "grpc_deps")
+
+# Includes boringssl, and other dependencies.
+grpc_deps()
+
+load("@com_github_grpc_grpc//bazel:grpc_extra_deps.bzl", "grpc_extra_deps")
+
+# Loads transitive dependencies of GRPC.
+grpc_extra_deps()
+
+http_archive(
+    name = "com_github_glog_glog",
+    # sha256 = "cbba86b5a63063999e0fc86de620a3ad22d6fd2aa5948bff4995dcd851074a0b",
+    strip_prefix = "glog-c8f8135a5720aee7de8328b42e4c43f8aa2e60aa",
+    urls = ["https://github.com/google/glog/archive/c8f8135a5720aee7de8328b42e4c43f8aa2e60aa.zip"],
+)
+
+http_archive(
+    name = "com_github_google_flatbuffers",
+    url = "https://github.com/google/flatbuffers/archive/refs/tags/v2.0.0.tar.gz",
+    strip_prefix = "flatbuffers-2.0.0",
+    sha256 = "9ddb9031798f4f8754d00fca2f1a68ecf9d0f83dfac7239af1311e4fd9a565c4",
+)
+
+
+# gflags Needed for glog
+http_archive(
+    name = "com_github_gflags_gflags",
+    sha256 = "34af2f15cf7367513b352bdcd2493ab14ce43692d2dcd9dfc499492966c64dcf",
+    strip_prefix = "gflags-2.2.2",
+    urls = [
+        "https://github.com/gflags/gflags/archive/v2.2.2.tar.gz",
+    ],
+)
+
+# Abseil C++ libraries
+git_repository(
+    name = "com_google_absl",
+    remote = "https://github.com/abseil/abseil-cpp.git",
+    commit = "0f3bb466b868b523cf1dc9b2aaaed65c77b28862",
+    shallow_since = "1603283562 -0400",
+)
+
+# googletest
+http_archive(
+    name = "com_google_googletest",
+    urls = ["https://github.com/google/googletest/archive/refs/tags/release-1.10.0.zip"],
+    sha256 = "94c634d499558a76fa649edb13721dce6e98fb1e7018dfaeba3cd7a083945e91",
+    strip_prefix = "googletest-release-1.10.0",
+)
+
+http_archive(
+    name = "bazel_common",
+    url = "https://github.com/google/bazel-common/archive/refs/heads/master.zip",
+    strip_prefix = "bazel-common-master",
+    sha256 = "7034b3fb6b3051d70f33853fff48b0e931b57e35c6a32bba0280c6f2b6d6ee0c",
+)
+
+http_archive(
+    name = "bazel_skylib",
+    strip_prefix = None,
+    url = "https://github.com/bazelbuild/bazel-skylib/releases/download/1.0.2/bazel-skylib-1.0.2.tar.gz",
+    sha256 = "97e70364e9249702246c0e9444bccdc4b847bed1eb03c5a3ece4f83dfe6abc44",
+)
+
+load("@bazel_skylib//lib:versions.bzl", "versions")
+
+versions.check(minimum_bazel_version = "5.0.0")
+
+# json
+http_archive(
+    name = "nlohmann_json",
+    strip_prefix = "json-3.9.1",
+    urls = ["https://github.com/nlohmann/json/archive/v3.9.1.tar.gz"],
+    sha256 = "4cf0df69731494668bdd6460ed8cb269b68de9c19ad8c27abc24cd72605b2d5b",
+    build_file = "//bazel:BUILD.nlohmann_json",
+)
+
+
+
+# pybind11 , bazel ref:https://github.com/pybind/pybind11_bazel
+# _PYBIND11_COMMIT = "72cbbf1fbc830e487e3012862b7b720001b70672"
+http_archive(
+  name = "pybind11_bazel",
+  strip_prefix = "pybind11_bazel-72cbbf1fbc830e487e3012862b7b720001b70672",
+  urls = [
+        "https://github.com/pybind/pybind11_bazel/archive/72cbbf1fbc830e487e3012862b7b720001b70672.zip",
+        "https://primihub.oss-cn-beijing.aliyuncs.com/pybind11_bazel-72cbbf1fbc830e487e3012862b7b720001b70672.zip"
+        ],
+)
+
+# We still require the pybind library.
+http_archive(
+  name = "pybind11",
+  build_file = "@pybind11_bazel//:pybind11.BUILD",
+  strip_prefix = "pybind11-2.9.2",
+  urls = ["https://github.com/pybind/pybind11/archive/refs/tags/v2.9.2.tar.gz"],
+)
+load("@pybind11_bazel//:python_configure.bzl", "python_configure")
+python_configure(name = "local_config_python")
+
+
+# ======== arrow dependencies  start ========
+
+http_archive(
+    name = "brotli",
+    build_file = "//bazel:brotli.BUILD",
+    sha256 = "4c61bfb0faca87219ea587326c467b95acb25555b53d1a421ffa3c8a9296ee2c",
+    strip_prefix = "brotli-1.0.7",
+    urls = [
+        "https://storage.googleapis.com/mirror.tensorflow.org/github.com/google/brotli/archive/v1.0.7.tar.gz",
+        "https://github.com/google/brotli/archive/refs/tags/v1.0.7.tar.gz",
+        "https://primihub.oss-cn-beijing.aliyuncs.com/brotli-1.0.7.tar.gz"
+    ],
+)
+
+
+http_archive(
+    name = "bzip2",
+    build_file = "//bazel:bzip2.BUILD",
+    sha256 = "ab5a03176ee106d3f0fa90e381da478ddae405918153cca248e682cd0c4a2269",
+    strip_prefix = "bzip2-1.0.8",
+    urls = [
+        "https://sourceware.org/pub/bzip2/bzip2-1.0.8.tar.gz",
+        "https://fossies.org/linux/misc/bzip2-1.0.8.tar.gz",
+        "https://primihub.oss-cn-beijing.aliyuncs.com/bzip2-1.0.8.tar.gz"
+    ],
+)
+
+
+http_archive(
+    name = "double-conversion",
+    sha256 = "a63ecb93182134ba4293fd5f22d6e08ca417caafa244afaa751cbfddf6415b13",
+    strip_prefix = "double-conversion-3.1.5",
+    urls = [
+        "https://storage.googleapis.com/mirror.tensorflow.org/github.com/google/double-conversion/archive/v3.1.5.tar.gz",
+        "https://github.com/google/double-conversion/archive/v3.1.5.tar.gz",
+        "https://primihub.oss-cn-beijing.aliyuncs.com/double-conversion-3.1.5.tar.gz",
+
+    ],
+)
+
+http_archive(
+    name = "lz4",
+    build_file = "//bazel:lz4.BUILD",
+    patch_cmds = [
+        """sed -i.bak 's/__attribute__ ((__visibility__ ("default")))//g' lib/lz4frame.h """,
+    ],
+    sha256 = "658ba6191fa44c92280d4aa2c271b0f4fbc0e34d249578dd05e50e76d0e5efcc",
+    strip_prefix = "lz4-1.9.2",
+    urls = [
+        # "https://storage.googleapis.com/mirror.tensorflow.org/github.com/lz4/lz4/archive/v1.9.2.tar.gz",
+        "https://github.com/lz4/lz4/archive/v1.9.2.tar.gz",
+        "https://primihub.oss-cn-beijing.aliyuncs.com/lz4-1.9.2.tar.gz"
+    ],
+)
+
+http_archive(
+    name = "rapidjson",
+    build_file = "//bazel:rapidjson.BUILD",
+    sha256 = "30bd2c428216e50400d493b38ca33a25efb1dd65f79dfc614ab0c957a3ac2c28",
+    strip_prefix = "rapidjson-418331e99f859f00bdc8306f69eba67e8693c55e",
+    urls = [
+        "https://github.com/miloyip/rapidjson/archive/418331e99f859f00bdc8306f69eba67e8693c55e.tar.gz",
+        "https://primihub.oss-cn-beijing.aliyuncs.com/rapidjson-418331e99f859f00bdc8306f69eba67e8693c55e.tar.gz"
+    ],
+)
+
+# Note: snappy is placed earlier as tensorflow's snappy does not include snappy-c
+http_archive(
+    name = "snappy",
+    build_file = "//bazel:snappy.BUILD",
+    sha256 = "16b677f07832a612b0836178db7f374e414f94657c138e6993cbfc5dcc58651f",
+    strip_prefix = "snappy-1.1.8",
+    urls = [
+        "https://github.com/google/snappy/archive/1.1.8.tar.gz",
+        "https://primihub.oss-cn-beijing.aliyuncs.com/snappy-1.1.8.tar.gz"
+    ],
+)
+
+http_archive(
+    name = "thrift",
+    build_file = "//bazel:thrift.BUILD",
+    sha256 = "5da60088e60984f4f0801deeea628d193c33cec621e78c8a43a5d8c4055f7ad9",
+    strip_prefix = "thrift-0.13.0",
+    urls = [
+        "https://github.com/apache/thrift/archive/v0.13.0.tar.gz",
+        "https://primihub.oss-cn-beijing.aliyuncs.com/thrift-0.13.0.tar.gz"
+    ],
+)
+
+http_archive(
+    name = "xsimd",
+    build_file = "//bazel:xsimd.BUILD",
+    sha256 = "45337317c7f238fe0d64bb5d5418d264a427efc53400ddf8e6a964b6bcb31ce9",
+    strip_prefix = "xsimd-7.5.0",
+    urls = [
+        "https://github.com/xtensor-stack/xsimd/archive/refs/tags/7.5.0.tar.gz",
+    ],
+)
+
+http_archive(
+    name = "zlib",
+    build_file = "//bazel:zlib.BUILD",
+    sha256 = "c3e5e9fdd5004dcb542feda5ee4f0ff0744628baf8ed2dd5d66f8ca1197cb1a1",
+    strip_prefix = "zlib-1.2.11",
+    urls = [
+        "https://zlib.net/zlib-1.2.11.tar.gz",
+        "https://primihub.oss-cn-beijing.aliyuncs.com/zlib-1.2.11.tar.gz"
+    ],
+)
+
+http_archive(
+    name = "zstd",
+    build_file = "//bazel:zstd.BUILD",
+    sha256 = "a364f5162c7d1a455cc915e8e3cf5f4bd8b75d09bc0f53965b0c9ca1383c52c8",
+    strip_prefix = "zstd-1.4.4",
+    urls = [
+        "https://github.com/facebook/zstd/archive/v1.4.4.tar.gz",
+        "https://primihub.oss-cn-beijing.aliyuncs.com/zstd-1.4.4.tar.gz"
+    ],
+)
+
+http_archive(
+    name = "xz",
+    build_file = "//bazel:xz.BUILD",
+    sha256 = "0d2b89629f13dd1a0602810529327195eff5f62a0142ccd65b903bc16a4ac78a",
+    strip_prefix = "xz-5.2.5",
+    urls = [
+        "https://github.com/xz-mirror/xz/archive/v5.2.5.tar.gz",
+        "https://primihub.oss-cn-beijing.aliyuncs.com/xz-5.2.5.tar.gz"
+    ],
+)
+
+http_archive(
+    name = "io_opentelemetry_cpp",
+    # sha256 = "<sha256>",
+    strip_prefix = "opentelemetry-cpp-1.0.1",
+    urls = [
+        "https://github.com/open-telemetry/opentelemetry-cpp/archive/refs/tags/v1.0.1.tar.gz",
+        "https://primihub.oss-cn-beijing.aliyuncs.com/opentelemetry-cpp-1.0.1.tar.gz"
+    ],
+)
+
+# Load OpenTelemetry dependencies after load.
+load("@io_opentelemetry_cpp//bazel:repository.bzl", "opentelemetry_cpp_deps")
+
+opentelemetry_cpp_deps()
+
+
+http_archive(
+    name = "com_google_protobuf",
+    # sha256 = "<sha256>",
+    # strip_prefix = "opentelemetry-cpp-1.0.1",
+    urls = [
+        "https://github.com/protocolbuffers/protobuf/archive/refs/tags/v3.20.0.tar.gz",
+        "https://primihub.oss-cn-beijing.aliyuncs.com/protobuf-3.20.0.tar.gz"
+    ],
+)
+
+
+git_repository(
+    name = "cares-bazel",
+    branch = "master",
+    remote = "https://github.com/hobo0cn/cares-bazel.git",
+    patch_cmds = [
+        "git submodule update --init --recursive",
+    ],
+)
+
+# fmt bazle, ref: https://fossies.org/linux/fmt/support/bazel/README.md
+load("@bazel_tools//tools/build_defs/repo:git.bzl", "git_repository")
+
+new_git_repository(
+    name = "fmt",
+    build_file = "//bazel:fmt.BUILD",
+    remote = "https://github.com/fmtlib/fmt.git",
+    tag = "6.1.2",
+)
+
+
+new_git_repository(
+    name = "libp2p",
+    build_file = "//bazel:libp2p.BUILD",
+    remote = "https://github.com/primihub/cpp-libp2p.git",
+    branch="master",
+)
+
+# soralog , need by libp2p
+# TODO need change to glog
+new_git_repository(
+    name = "com_openmpc_soralog",
+    build_file = "//bazel:soralog.BUILD",
+    remote = "https://github.com/primihub/soralog.git",
+    branch="master",
+)
+
+# sqlite, need by libp2p
+http_archive(
+    name = "com_github_soramitsu_sqlite",
+    build_file = "//bazel:sqlite.BUILD",
+    strip_prefix = "libp2p-sqlite-modern-cpp-3.2",
+    urls = [
+        "https://github.com/soramitsu/libp2p-sqlite-modern-cpp/archive/refs/tags/v3.2.tar.gz",
+        "https://primihub.oss-cn-beijing.aliyuncs.com/libp2p-sqlite-modern-cpp-3.2.tar.gz"],
+)
+
+#yaml-cpp, need by libp2p
+git_repository(
+    name = "com_github_jbeder_yaml_cpp",
+    remote = "https://github.com/jbeder/yaml-cpp.git",
+    tag="yaml-cpp-0.7.0",
+)
+# hat_trie , need by libp2p
+new_git_repository(
+    name = "com_github_masterjedy_hat_hrie",
+    build_file = "//bazel:hat_trie.BUILD",
+    remote = "https://github.com/masterjedy/hat-trie.git",
+    branch="master",
+)
+
+# boost di, used by libp2p
+http_archive(
+    name = "com_github_masterjedy_di",
+    build_file = "//bazel:di.BUILD",
+    strip_prefix = "di-1.1.0",
+    urls = [
+        "https://github.com/boost-ext/di/archive/refs/tags/v1.1.0.tar.gz",
+        "https://primihub.oss-cn-beijing.aliyuncs.com/di-1.1.0.tar.gz"
+    ],
+)
+
+# seal primihub fork from microsoft, used by crypTFlow2 and APSI
+new_git_repository(
+  name = "com_github_primihub_seal_40",
+  build_file = "//bazel:BUILD.seal",
+  remote = "https://github.com/primihub/SEAL.git",
+  branch = "change_submodule_resource",
+  #branch = "main",
+)
+
+http_archive(
+    name = "com_github_gmp",
+    build_file = "//bazel:gmp.BUILD",
+    #sha256 = "87b565e89a9a684fe4ebeeddb8399dce2599f9c9049854ca8c0dfbdea0e21912",
+    strip_prefix = "gmp-6.2.1",
+    urls = ["https://gmplib.org/download/gmp/gmp-6.2.1.tar.xz"],
+)
+
+# Google dense_hash_set
+http_archive(
+    name = "google_sparsehash",
+    build_file = "//bazel:BUILD.sparsehash",
+    strip_prefix = "sparsehash-master",
+    urls = ["https://github.com/google/sparsehash/archive/master.zip"],
+)
+
+# libPSI start
+new_git_repository(
+    name = "osu_libpsi",
+    build_file = "//bazel:BUILD.libpsi",
+    commit = "4c5d5a3e49533c8547dcd4869e6a9842b6ce5b90",
+    remote = "https://github.com/primihub/libPSI.git",
+)
+
+# libote
+new_git_repository(
+    name = "osu_libote",
+    build_file = "//bazel:libOTe.BUILD",
+    commit = "f455eb7bf83034ebca6cab42e3aea9d9b33f8102",
+    remote = "https://github.com/primihub/libOTe.git",
+)
+
+new_git_repository(
+    name = "ladnir_cryptoTools",
+    build_file = "//bazel:cryptoTools.BUILD",
+    commit = "d52e05e2e803006256ddb66f48a0d51080f4b285",
+    remote = "https://github.com/primihub/cryptoTools.git",
+    # shallow_since = "1591047380 -0700",
+)
+
+new_git_repository(
+    name = "github_ntl",
+    build_file = "//bazel:ntl.BUILD",
+    commit = "6b23216ce6b24dd04446122c2200b6f031621965",
+    remote = "https://github.com/primihub/ntl.git",
+    # shallow_since = "1591047380 -0700",
+)
+
+new_git_repository(
+    name = "toolkit_relic",
+    build_file = "//bazel:BUILD.relic",
+    remote = "https://github.com/relic-toolkit/relic.git",
+    commit = "3f616ad64c3e63039277b8c90915607b6a2c504c",
+    shallow_since = "1581106153 -0800",
+)
+
+http_archive(
+    name = "com_github_redis_hiredis",
+    build_file = "//bazel:BUILD.hiredis",
+    urls = ["https://github.com/redis/hiredis/archive/392de5d7f97353485df1237872cb682842e8d83f.tar.gz"],
+    sha256 = "2101650d39a8f13293f263e9da242d2c6dee0cda08d343b2939ffe3d95cf3b8b",
+    strip_prefix = "hiredis-392de5d7f97353485df1237872cb682842e8d83f"
+)
+
+# libote
+#http_archive(
+#    name = "osu_libote",
+#    build_file = "//external:libOTe.BUILD",
+#    #sha256 = "6f021f24136eb177af38af3bf5d53b3592a1fe1e71d1c098318488a85b0afc3a",
+#    strip_prefix = "libOTe-master",
+#    urls = ["https://github.com/osu-crypto/libOTe/archive/refs/heads/master.zip"],
+#)
+
+# cryptoTools
+#http_archive(
+#    name = "ladnir_cryptoTools",
+#    build_file = "//external:cryptoTools.BUILD",
+#    #sha256 = "6f021f24136eb177af38af3bf5d53b3592a1fe1e71d1c098318488a85b0afc3a",
+#    strip_prefix = "cryptoTools-master",
+#    urls = ["https://github.com/ladnir/cryptoTools/archive/refs/heads/master.zip"],
+#)
+
+#PSI
+load("@bazel_tools//tools/build_defs/repo:git.bzl", "git_repository")
+
+git_repository(
+   name = "org_openmined_psi",
+   remote = "https://github.com/primihub/PSI.git",
+   branch = "master",
+   init_submodules = True,
+)
+
+load("@org_openmined_psi//private_set_intersection:preload.bzl", "psi_preload")
+
+psi_preload()
+
+load("@org_openmined_psi//private_set_intersection:deps.bzl", "psi_deps")
+
+psi_deps()
+
+
+load("@bazel_tools//tools/build_defs/repo:git.bzl", "git_repository")
+
+
+git_repository(
+   name = "org_openmined_pir",
+   remote = "https://github.com/primihub/PIR.git",
+   branch = "master",
+   init_submodules = True,
+)
+
+load("@org_openmined_pir//pir:preload.bzl", "pir_preload")
+
+pir_preload()
+
+load("@org_openmined_pir//pir:deps.bzl", "pir_deps")
+
+pir_deps()
+
+
+# leveldb as local kv store
+# load("//3rdparty/bazel-rules-leveldb/bazel:repos.bzl", leveldb_repos="repos")
+# leveldb_repos()
+
+# load("//3rdparty/bazel-rules-leveldb/bazel:deps.bzl", leveldb_deps="deps")
+# leveldb_deps()
+
+http_archive(
+    name = "com_github_google_leveldb",
+    build_file = "//bazel:BUILD.leveldb",
+    sha256 = "a6fa7eebd11de709c46bf1501600ed98bf95439d6967963606cc964931ce906f",
+    strip_prefix = "leveldb-1.23",
+    urls = [
+        "https://github.com/google/leveldb/archive/refs/tags/1.23.zip"
+    ],
+)
+
+# APSI
+git_repository(
+    name = "mircrosoft_apsi",
+    branch = "bazel_version",
+    remote = "https://github.com/primihub/APSI.git",
+)
+
+load("@mircrosoft_apsi//bazel:repositories.bzl", "apsi_dependencies")
+apsi_dependencies("mircrosoft_apsi");
+
+# needed by APSI
+new_git_repository(
+    name = "com_microsoft_kuku",
+    build_file = "//bazel:BUILD.kuku",
+    branch = "2.1.0",
+    remote = "https://github.com/microsoft/Kuku.git",
+)
+# needed by APSI
+git_repository(
+    name = "jsoncpp",
+    branch = "master",
+    remote = "https://github.com/open-source-parsers/jsoncpp.git",
+)
+
+#support sqlite
+http_archive(
+  name = "com_github_sqlite_wrapper",
+  build_file = "//bazel:BUILD.sqlite",
+  sha256 = "57f91ed44ef205fe97b8c6586002fe6031cd02771d1c5d8415d9c515ad1532d1",
+  strip_prefix = "SQLiteCpp-3.2.0",
+  urls = [
+    "https://github.com/SRombauts/SQLiteCpp/archive/refs/tags/3.2.0.tar.gz",
+  ]
+)
+
+#generate uuid
+http_archive(
+  name = "com_github_stduuid",
+  build_file = "//bazel:BUILD.stduuid",
+  sha256 = "f554f6a9fe4d852fa217de6ab977afbf3f49e4a1dcb263afd61a94253c4c7a48",
+  strip_prefix = "stduuid-1.2.2",
+  urls = [
+    "https://github.com/mariusbancila/stduuid/archive/refs/tags/v1.2.2.tar.gz",
+  ]
+)

bazel/BUILD.relic

@@ -17,9 +17,9 @@ genrule(
     cmd = """
         set -x
         RELIC_ROOT=$$(dirname $(location CMakeLists.txt))
-        pushd $$RELIC_ROOT 
+        pushd $$RELIC_ROOT
             cmake -DARCH=ARM -DWSIZE=32 .
-            make
+            make -j`nproc`
         popd
         if [ "$${OSTYPE}" == "linux-gnu" ]; then
             objcopy --localize-symbol=bn_init $$RELIC_ROOT/lib/librelic_s.a

bazel/BUILD.seal

@@ -16,10 +16,13 @@ cmake(
    #    "https_proxy": "http://127.0.0.1:1080",
    #    "http_proxy": "http://127.0.0.1:1080",
    #},
+    build_args = [
+        "-j`nproc`",
+    ],
     cache_entries = {
         "SEAL_THROW_ON_TRANSPARENT_CIPHERTEXT": "OFF",
     },
-   defines = [
+    defines = [
         "SEAL_USE_CXX17=ON",
         "SEAL_USE_INTRIN=ON",
         "SEAL_USE_MSGSL=ON",
@@ -27,7 +30,7 @@ cmake(
         "SEAL_BUILD_TESTS=OFF",
         "BUILD_SHARED_LIBS=OFF",
         "CMAKE_BUILD_TYPE=Release",
-   ],
+    ],
    install = True,
 #    make_commands = [
 #         "make -j",

bazel/BUILD.stduuid

+# Description:
+package(default_visibility = ["//visibility:public"])
+cc_library(
+  name = "stduuid_lib",
+  hdrs = glob([
+    "include/uuid.h",
+  ]),
+  copts = [
+    '--std=c++17',
+  ],
+  includes = ["include",],
+  deps = [
+    ":internal_gsl_lib", 
+  ],
+)
+
+cc_library(
+  name = "internal_gsl_lib",
+  textual_hdrs = [
+    "gsl/span",
+    "gsl/gsl_assert",
+    "gsl/gsl_util",
+    "gsl/gsl_byte",
+    "gsl/gsl_algorithm",
+    "gsl/gsl",
+    "gsl/pointers",
+    "gsl/multi_span",
+    "gsl/string_span",
+  ],
+  includes = ["gsl"],
+  copts = [
+      "--std=c++17",
+  ],
+)

bazel/gmp.BUILD

@@ -57,7 +57,7 @@ genrule(
             if ! ./configure --prefix=/tmp/gmp --enable-cxx --disable-assembly > $$CONFIGURE_LOG; then
                 cat $$CONFIGURE_LOG
             fi
-            if ! make > $$MAKE_LOG; then
+            if ! make -j`nproc` > $$MAKE_LOG; then
                 cat $$MAKE_LOG
             fi
             if ! make install > $$MAKE_LOG; then

bazel/ntl.BUILD

@@ -101,7 +101,7 @@ include_files = [
     "include/NTL/vector.h",
     "include/NTL/version.h",
     "include/NTL/xdouble.h",
-    
+
     "include/NTL/config.h",
     "include/NTL/mach_desc.h",
     "include/NTL/gmp_aux.h",
@@ -125,7 +125,7 @@ genrule(
         'cd $$TMP_DIR',
         'cd src',
         './configure PREFIX=$$INSTALL_DIR NTL_THREADS=on NTL_THREAD_BOOST=on NTL_EXCEPTIONS=on SHARED=on NTL_STD_CXX11=on NTL_SAFE_VECTORS=off TUNE=generic',
-        'make && make install',
+        'make -j`nproc` && make install',
 	'cp $$TMP_DIR/include/NTL/config.h $$INSTALL_DIR/NTL',
 	'cp $$TMP_DIR/include/NTL/mach_desc.h $$INSTALL_DIR/NTL',
 	'cp $$TMP_DIR/include/NTL/gmp_aux.h $$INSTALL_DIR/NTL',

build_local.sh

@@ -9,7 +9,7 @@ fi
 
 bash pre_build.sh
 
-bazel build --cxxopt=-D_AMD64_ --config=linux --define microsoft-apsi=true :node :cli :opt_paillier_c2py
+bazel build --config=linux --define cpu=amd64 --define microsoft-apsi=true :node :cli :opt_paillier_c2py :linkcontext
 
 if [ $? -ne 0 ]; then
     echo "Build failed!!!"
@@ -41,4 +41,4 @@ cp -r ./src $BASE_DIR/
 cd $BASE_DIR
 find ./ -name "_objs" > .dockerignore
 
-docker build -t $IMAGE_NAME:$tag . -f Dockerfile.local 
\ No newline at end of file
+docker build -t $IMAGE_NAME:$tag . -f Dockerfile.local 

config/pir_server_config.json

 {
     "table_params": {
-        "hash_func_count": 1,
+        "hash_func_count": 2,
         "table_size": 409,
         "max_items_per_bin": 20
     },

config/primihub_node1.yaml

@@ -56,6 +56,14 @@ datasets:
     model: "csv"
     source: "/tmp/FL/homo_lr_test.data"
 
+  - description: "train_hetero_xgb_host"
+    model: "csv"
+    source: "/tmp/FL/hetero_xgb/train/train_breast_cancer_host.csv"
+
+  - description: "test_hetero_xgb_host"
+    model: "csv"
+    source: "/tmp/FL/hetero_xgb/test/test_breast_cancer_host.csv"
+
 localkv:
   model: "leveldb"
   path: "/data/localdb1"

config/primihub_node2.yaml

@@ -40,6 +40,14 @@ datasets:
     model: "csv"
     source: "/tmp/client_e.csv"
 
+  - description: "train_hetero_xgb_guest"
+    model: "csv"
+    source: "/tmp/FL/hetero_xgb/train/train_breast_cancer_guest.csv"
+
+  - description: "test_hetero_xgb_guest"
+    model: "csv"
+    source: "/tmp/FL/hetero_xgb/test/test_breast_cancer_guest.csv"
+
 localkv: 
   model: "leveldb"
   path: "/data/localdb2"

data/FL/hetero_xgb/test/test_breast_cancer_guest.csv

+mean radius,mean texture
+13.4,20.52
+13.21,25.25
+14.02,15.66
+14.26,18.17
+13.03,18.42
+11.34,18.61
+12.05,22.72
+11.7,19.11
+7.729,25.49
+10.26,14.71
+14.69,13.98
+14.62,24.02
+9.397,21.68
+16.84,19.46
+14.64,15.24
+15.46,11.89
+9.042,18.9
+20.51,27.81
+19.55,23.21
+20.94,23.56
+11.84,18.7
+16.24,18.77
+13.47,14.06
+11.84,18.94
+21.37,15.1
+11.93,10.91
+10.9,12.96
+13.73,22.61
+13.4,16.95
+18.77,21.43
+12.81,13.06
+20.57,17.77
+13.14,20.74
+16.16,21.54
+9.738,11.97
+12.68,23.84
+13.27,17.02
+19.0,18.91
+14.86,16.94
+15.08,25.74
+15.13,29.81
+8.878,15.49
+16.02,23.24
+10.32,16.35
+13.24,20.13
+21.56,22.39
+8.671,14.45
+14.03,21.25
+11.81,17.39
+14.54,27.54
+19.17,24.8
+14.86,23.21
+18.45,21.91
+11.6,18.36
+12.46,19.89
+11.47,16.03
+11.32,27.08
+12.27,29.97
+12.18,20.52
+21.75,20.99
+13.77,22.29
+16.07,19.65
+11.14,14.07
+12.87,16.21
+17.6,23.33
+12.04,28.14
+22.27,19.67
+17.46,39.28
+20.58,22.14
+9.465,21.01
+14.42,16.54
+16.13,20.68
+11.52,14.93
+14.6,23.29
+18.65,17.6
+13.0,25.13
+8.571,13.1
+13.05,18.59
+11.66,17.07
+8.734,16.84
+16.6,28.08
+20.18,23.97
+15.78,22.91
+12.43,17.0
+16.03,15.51
+12.32,12.39
+13.5,12.71
+8.196,16.84
+17.19,22.07
+20.6,29.33
+9.504,12.44
+15.12,16.68
+14.99,22.11
+15.22,30.62
+9.405,21.7
+9.876,19.4
+15.5,21.08
+12.89,14.11
+11.57,19.04
+11.43,17.31
+8.888,14.64
+12.36,18.54
+14.97,19.76
+14.05,27.15
+18.46,18.52
+9.268,12.87
+17.91,21.02
+12.88,18.22
+15.61,19.38
+17.42,25.56
+12.75,16.7
+20.18,19.54
+18.31,20.58
+15.04,16.74

data/FL/hetero_xgb/train/train_breast_cancer_guest.csv

+mean radius,mean texture
+10.05,17.53
+10.8,21.98
+16.14,14.86
+12.18,17.84
+12.25,22.44
+13.28,13.72
+11.74,14.02
+12.67,17.3
+14.22,23.12
+11.27,12.96
+20.2,26.83
+11.62,18.18
+14.34,13.47
+11.29,13.04
+10.26,16.58
+10.86,21.48
+18.01,20.56
+10.08,15.11
+13.2,15.82
+13.11,22.54
+11.3,18.19
+11.54,10.72
+14.74,25.42
+15.05,19.07
+23.21,26.97
+10.51,20.19
+12.34,12.27
+13.53,10.94
+19.59,18.15
+11.85,17.46
+11.22,19.86
+20.13,28.25
+13.59,17.84
+16.69,20.2
+12.06,18.9
+13.87,16.21
+12.56,19.07
+14.26,19.65
+9.029,17.33
+13.46,18.75
+9.876,17.27
+14.78,23.94
+14.04,15.98
+13.61,24.69
+12.72,17.67
+15.49,19.97
+19.68,21.68
+12.3,19.02
+10.97,17.2
+17.27,25.42
+11.43,15.39
+17.05,19.08
+13.71,20.83
+17.99,20.66
+13.71,18.68
+13.3,21.57
+9.333,21.94
+11.49,14.59
+10.49,18.61
+12.25,17.94
+18.31,18.58
+6.981,13.43
+9.787,19.94
+13.08,15.71
+11.15,13.08
+15.1,22.02
+18.63,25.11
+11.67,20.02
+13.49,22.3
+13.61,24.98
+12.99,14.23
+10.51,23.09
+14.76,14.74
+12.54,16.32
+19.21,18.57
+10.75,14.97
+10.17,14.88
+18.25,19.98
+13.0,21.82
+10.88,15.62
+10.25,16.18
+19.73,19.82
+23.29,26.67
+13.2,17.43
+13.64,15.6
+14.99,25.2
+12.87,19.54
+10.26,12.22
+17.35,23.06
+23.51,24.27
+19.4,18.18
+11.33,14.16
+12.21,18.02
+9.676,13.14
+20.59,21.24
+13.37,16.39
+13.15,15.34
+12.18,14.08
+8.618,11.79
+11.25,14.78
+23.09,19.83
+10.96,17.62
+20.92,25.09
+14.53,13.98
+17.29,22.13
+13.66,15.15
+12.83,22.33
+11.52,18.75
+13.43,19.63
+12.62,17.15
+13.75,23.77
+12.06,12.74
+11.89,21.17
+16.13,17.88
+11.2,29.37
+19.81,22.15
+16.3,15.7
+12.86,13.32
+13.64,16.34
+19.53,32.47
+10.82,24.21
+11.94,18.24
+12.31,16.52
+14.58,21.53
+11.08,14.71
+13.27,14.76
+16.78,18.8
+20.29,14.34
+12.27,17.92
+21.09,26.57
+10.48,19.86
+11.08,18.83
+10.03,21.28
+16.25,19.51
+12.45,15.7
+16.11,18.05
+16.26,21.88
+12.23,19.56
+12.77,22.47
+12.58,18.4
+24.25,20.2
+15.73,11.28
+15.34,14.26
+13.65,13.16
+19.89,20.26
+11.41,10.82
+13.34,15.86
+19.44,18.82
+13.17,18.22
+12.88,28.92
+9.683,19.34
+11.75,20.18
+9.423,27.88
+14.22,27.85
+12.89,15.7
+12.49,16.85
+12.91,16.33
+12.77,21.41
+8.95,15.76
+13.9,16.62
+11.13,16.62
+23.27,22.04
+11.64,18.33
+12.94,16.17
+9.173,13.86
+13.11,15.56
+12.76,13.37
+10.29,27.61
+13.9,19.24
+11.71,15.45
+13.45,18.3
+14.06,17.18
+21.61,22.28
+11.04,14.93
+19.18,22.49
+11.9,14.65
+13.0,20.78
+13.48,20.82
+9.295,13.9
+15.75,19.22
+17.3,17.08
+9.668,18.1
+13.82,24.49
+19.69,21.25
+13.56,13.9
+13.68,16.33
+11.8,16.58
+11.37,18.89
+12.42,15.04
+11.06,14.96
+18.82,21.97
+8.726,15.83
+14.61,15.69
+11.8,17.26
+9.904,18.06
+17.85,13.23
+17.47,24.68
+20.44,21.78
+7.691,25.44
+12.22,20.04
+10.48,14.98
+11.71,16.67
+18.03,16.85
+19.45,19.33
+12.76,18.84
+12.21,14.09
+13.16,20.54
+12.0,28.23
+13.96,17.05
+11.22,33.81
+15.71,13.93
+19.27,26.47
+12.36,21.8
+14.41,19.73
+13.17,21.81
+8.598,20.98
+14.45,20.22
+16.17,16.07
+13.62,23.23
+11.75,17.56
+9.742,19.12
+19.8,21.56
+20.09,23.86
+11.76,18.14
+13.8,15.79
+13.77,13.27
+16.74,21.59
+12.46,12.83
+17.57,15.05
+17.08,27.15
+13.85,19.6
+11.99,24.89
+13.05,13.84
+14.47,24.99
+10.2,17.48
+18.61,20.25
+14.4,26.99
+10.57,20.22
+17.68,20.74
+12.77,29.43
+12.83,15.73
+14.95,17.57
+28.11,18.47
+10.49,19.29
+14.8,17.66
+10.57,18.32
+21.1,20.52
+13.86,16.93
+12.65,18.17
+12.3,15.9
+14.19,23.81
+9.847,15.68
+12.47,18.6
+9.567,15.91
+15.53,33.56
+11.31,19.04
+16.65,21.38
+11.89,18.35
+19.53,18.9
+15.75,20.25
+19.79,25.12
+20.47,20.67
+12.63,20.76
+10.8,9.71
+13.59,21.84
+11.06,17.12
+13.44,21.58
+17.93,24.48
+13.38,30.72
+12.86,18.0
+12.96,18.29
+12.34,14.95
+12.03,17.93
+15.85,23.95
+12.89,13.12
+14.11,12.88
+18.66,17.12
+11.5,18.45
+12.62,23.97
+18.49,17.52
+19.19,15.94
+19.1,26.29
+14.87,16.67
+13.69,16.07
+12.85,21.37
+17.14,16.4
+9.777,16.99
+13.74,17.91
+9.731,15.34
+14.68,20.13
+13.98,19.62
+12.98,19.35
+11.34,21.26
+9.667,18.49
+11.87,21.54
+11.28,13.39
+20.48,21.46
+20.31,27.06
+25.73,17.46
+11.63,29.29
+18.22,18.87
+20.34,21.51
+14.58,13.66
+10.65,25.22
+11.36,17.57
+11.61,16.02
+12.47,17.31
+13.54,14.36
+16.27,20.71
+10.91,12.35
+13.7,17.64
+10.71,20.39
+16.5,18.29
+10.44,15.46
+14.87,20.21
+11.42,20.38
+17.75,28.03
+13.85,17.21
+19.16,26.6
+11.94,20.76
+18.05,16.15
+15.28,22.41
+15.19,13.21
+12.45,16.41
+10.16,19.59
+11.89,17.36
+12.72,13.78
+20.55,20.86
+15.7,20.31
+17.02,23.98
+11.45,20.97
+9.0,14.4
+15.66,23.2
+15.78,17.89
+12.54,18.07
+14.2,20.53
+14.48,21.46
+10.6,18.95
+14.25,21.72
+12.34,26.86
+13.05,19.31
+10.95,21.35
+14.29,16.82
+11.04,16.83
+11.76,21.6
+15.27,12.91
+14.81,14.7
+11.54,14.44
+11.26,19.83
+24.63,21.6
+19.59,25.0
+14.5,10.89
+10.18,17.53
+11.16,21.41
+14.25,22.15
+7.76,24.54
+13.46,28.21
+12.19,13.29
+11.71,17.19
+21.16,23.04
+12.16,18.03
+15.46,23.95
+13.28,20.28
+16.35,23.29
+12.1,17.72
+17.99,10.38
+12.39,17.48
+17.54,19.32
+10.94,18.59
+21.71,17.25
+27.42,26.27
+12.07,13.44
+11.6,12.84
+13.88,16.16
+11.95,14.96
+11.46,18.16
+12.7,12.17
+12.95,16.02
+13.01,22.22
+22.01,21.9
+19.55,28.77
+15.06,19.83
+18.08,21.84
+20.26,23.03
+14.59,22.68
+12.34,22.22
+14.97,16.95
+12.78,16.49
+11.6,24.49
+12.05,14.63
+9.606,16.84
+20.64,17.35
+12.8,17.46
+17.01,20.26
+13.85,15.18
+13.94,13.17
+15.0,15.51
+14.44,15.18
+12.9,15.92
+27.22,21.87
+15.46,19.48
+13.17,18.66
+12.2,15.21
+13.81,23.75
+12.0,15.65
+11.06,14.83
+15.37,22.76
+25.22,24.91
+14.9,22.53
+17.95,20.01
+14.96,19.1
+14.71,21.59
+20.16,19.66
+15.32,17.27
+13.78,15.79
+19.07,24.81
+11.27,15.5
+17.06,21.0
+9.742,15.67
+11.68,16.17
+18.22,18.7
+14.27,22.55
+15.1,16.39
+15.3,25.27
+11.41,14.92
+9.755,28.2
+11.74,14.69
+8.219,20.7
+19.4,23.5
+13.21,28.06
+13.87,20.7
+20.73,31.12
+11.26,19.96
+13.66,19.13
+14.95,18.77
+16.46,20.11
+14.42,19.77
+14.53,19.34
+12.4,17.68
+11.93,21.53
+17.2,24.52
+11.69,24.44
+10.66,15.15
+19.02,24.59
+11.13,22.44
+14.64,16.85
+8.597,18.6
+13.51,18.89
+14.92,14.93
+18.94,21.31
+18.81,19.98
+12.46,24.04
+9.436,18.32
+9.72,18.22
+11.51,23.93

pre_build.sh

@@ -23,20 +23,7 @@ if ! command -v python$U_V1.$U_V2-config >/dev/null 2>&1; then
   echo "please install python$U_V1.$U_V2-dev"
   exit
 fi
-#for mq
-#check operating system
-os_name=$(uname -s)
-if [[ "$os_name" == "Darwin" ]]; then
-zmq_lib_path=$(find /opt/ -name libzmq.a |head -n 1)
-zmq_lib_path=$(echo ${zmq_lib_path}|awk -F'libzmq.a' '{print $1}')
-else
-zmq_lib_path=$(find /usr/local/ -name libzmq.a |head -n 1)
-zmq_lib_path=$(echo ${zmq_lib_path}|awk -F'libzmq.a' '{print $1}')
-fi
 CONFIG=`python$U_V1.$U_V2-config --ldflags` && NEWLINE="[\"${CONFIG}\"] + [\"-lpython$U_V1.$U_V2\"]"
-if [[ -n "${zmq_lib_path}" ]]; then
-  NEWLINE="${NEWLINE} + [\"-L${zmq_lib_path}\"] + [\"-lzmq\"]"
-fi
 # Compatible with MacOS
 sed -e "s|PLACEHOLDER-PYTHON3.X-CONFIG|${NEWLINE}|g" BUILD.bazel > BUILD.bazel.tmp && mv BUILD.bazel.tmp BUILD.bazel
 echo "done"
@@ -47,4 +34,4 @@ fi
 
 if [ ! -d "log" ]; then
     mkdir log
-fi
\ No newline at end of file
+fi

python/demo.py

@@ -13,67 +13,44 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
 """
-
+import sys
+import logging
 import primihub as ph
-from primihub.client import primihub_cli as cli
-
-# client init
-# cli.init(config={"node": "127.0.0.1:8050", "cert": ""})
-# cli.init(config={"node": "192.168.99.26:8050", "cert": ""})
-# cli.init(config={"node": "192.168.99.26:50050", "cert": ""})
-cli.init(config={"node": "192.168.99.26:50050", "cert": ""})
-
-# from primihub import context, dataset
-
-# print(ph.context.Context.__dict__)
-
-# ph.dataset.define("guest_dataset")
-# ph.dataset.define("label_dataset")
-
-
-# # ph.dataset.define("test_dataset")
-
-# # define a remote method
-# @ph.context.function(role='host', protocol='xgboost', datasets=["label_dataset"], next_peer="*:5555")
-# def func1(value=1):
-#     print("params: ", str(value))
-
-#     # do something
-#     # next_peer = ph.context.Context.nodes_context["host"].next_peer
-#     # print("host next peer: ", next_peer)
-#     # ip, port = next_peer.split(":")
-#     # ios = IOService()
-#     # server = Session(ios, ip, port, "server")
-#     # channel = server.addChannel()
-#     # channel.recv()
-#     # channel.send(value)
-#     # print(channel.recv())
-#     # return value
-
-
-# # define a remote method
-# @ph.context.function(role='guest', protocol='xgboost', datasets=["guest_dataset"], next_peer="localhost:5555")
-# def func2(value=2):
-#     print("params: ", str(value))
-#     # do something
-#     # next_peer = ph.context.Context.nodes_context["host"].next_peer
-#     # print("guest next peer: ", next_peer)
-#     # ip, port = next_peer.split(":")
-#     # ios = IOService()
-#     # client = Session(ios, ip, port, "client")
-#     # channel = client.addChannel()
-#     # channel.send(b'guest ready')
-#     # pub = channel.recv()
-#     # channel.send(b'recved pub')
-#     # return value
-
+import time
+LOG_FORMAT = "[%(asctime)s][%(filename)s:%(lineno)d][%(levelname)s] %(message)s"
+DATE_FORMAT = "%m/%d/%Y %H:%M:%S %p"
+logging.basicConfig(level=logging.DEBUG, format=LOG_FORMAT, datefmt=DATE_FORMAT)
+logger = logging.getLogger("proxy")
+
+# define a remote method
+@ph.context.function(role='host', protocol='xgboost', datasets=["psi_client_data"], port="*:5555")
+def func1(value=1):
+    print("params: ", str(value))
+    #print(node.to_string())
+    link_context = ph.context.Context.get_link_conext()
+    node = ph.context.Context.Node("127.0.0.1", 50051, False)
+    #print(node.to_string())
+    channel = link_context.getChannel(node)
+    send_buf = "1111111111111111111111111111111111111111"
+    channel.send("test_key1", send_buf)
+
+    local_node = ph.context.Context.Node("127.0.0.1", 50053, False)
+    local_channel = link_context.getChannel(local_node)
+    recv_data = local_channel.recv("test_key")
+
+
+# define a remote method
+@ph.context.function(role='guest', protocol='xgboost', datasets=["psi_server_data"], port="localhost:5555")
+def func2(value=2):
+    logger.debug("params: ", str(value))
+    logger.debug("XXXXXXXXXXXXXXXXXXXXXXXXXXXXX")
+    link_context = ph.context.Context.get_link_conext()
+    node = ph.context.Context.Node("127.0.0.1", 50053, False)
+    channel = link_context.getChannel(node)
+    send_buf = "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"
+    channel.send("test_key", send_buf)
+
+    local_node = ph.context.Context.Node("127.0.0.1", 50051, False)
+    channel_local = link_context.getChannel(local_node)
+    recv_buf = channel_local.recv("test_key1")
 
-# # context
-# value1 = 1
-# value2 = 2
-# # cli.async_remote_execute((func1, value1), (func2, value2)) # 1111
-# print(ph.context.Context)
-# 
-print("###########")
-cli.start()
-print("###########")

python/primihub/context.py

@@ -4,7 +4,6 @@ from typing import Callable
 from cloudpickle import dumps
 from primihub.utils.logger_util import logger
 
-
 class NodeContext:
 
     def __init__(self, role, protocol, dataset_port_map, func=None):
@@ -58,6 +57,8 @@ class TaskContext:
         self.role_nodeid_map["arbiter"] = []
         self.role_nodeid_map["guest"] = []
         self.params_map = {}
+        self.link_context = None
+
 
     def get_protocol(self):
         """Get current task support protocol.
@@ -184,6 +185,26 @@ class TaskContext:
             return None
         return node_context.dataset_service_shared_ptr
 
+    def task_id(self):
+        return self.params_map["taskid"]
+
+    def job_id(self):
+        return self.params_map["jobid"]
+
+    def init_link_context(self):
+        import linkcontext
+        self.link_context = linkcontext.LinkFactory.createLinkContext(linkcontext.LinkMode.GRPC)
+        self.link_context.setTaskInfo(self.job_id(), self.task_id())
+
+    def get_link_conext(self):
+        if not self.link_context:
+            self.init_link_context()
+        return self.link_context
+
+    def Node(self, ip, port, use_tls, nodename = "default"):
+        import linkcontext
+        return linkcontext.Node(ip, port, use_tls, nodename)
+
 
 Context = TaskContext()
 
@@ -262,4 +283,5 @@ def function(protocol, role, datasets, port, task_type="default"):
 
         return wapper
 
-    return function_decorator
\ No newline at end of file
+    return function_decorator
+

python/primihub/dataset/dataset.py

@@ -28,6 +28,7 @@ from primihub.client.ph_grpc.src.primihub.protos import service_pb2
 from primihub.client.ph_grpc.src.primihub.protos import service_pb2_grpc
 
 def register_dataset(service_addr, driver, path, name):
+    logger.info("Dataset service is {}.".format(service_addr))
     channel = grpc.insecure_channel(service_addr)
     stub = service_pb2_grpc.DataServiceStub(channel)
     request = service_pb2.NewDatasetRequest()

python/primihub/examples/disxgb.py

-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-"""
- Copyright 2022 Primihub
-
- Licensed under the Apache License, Version 2.0 (the "License");
- you may not use this file except in compliance with the License.
- You may obtain a copy of the License at
-
-      https://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
- """
-
-import primihub as ph
-from primihub.channel.zmq_channel import IOService, Session
-from primihub.FL.model.xgboost.xgb_guest import XGB_GUEST
-from primihub.FL.model.xgboost.xgb_host import XGB_HOST
-import pandas as pd
-import numpy as np
-
-# define dataset
-ph.dataset.define("guest_dataset")
-ph.dataset.define("label_dataset")
-
-
-@ph.function(role='host', protocol='xgboost', datasets=["label_dataset"], next_peer="*:5555")
-def xgb_host_logic():
-    print("start xgb host logic...")
-    columnNames = (
-        'Sample code number',
-        'Clump Thickness',
-        'Uniformity of Cell Size',
-        'Uniformity of Cell Shape',
-        'Marginal Adhesion',
-        'Single Epithelial Cell Size',
-        'Bare Nuclei',
-        'Bland Chromatin',
-        'Normal Nucleoli',
-        'Mitoses',
-        'Class'
-    )
-    next_peer = ph.context.Context.nodes_context["host"].next_peer
-    ip, port = next_peer.split(":")
-    ios = IOService()
-    server = Session(ios,  ip, port, "server")
-    channel = server.addChannel()
-
-    data = ph.dataset.read(dataset_key="label_dataset", names=columnNames).df_data
-    labels = ['Sample code number', 'Clump Thickness', 'Uniformity of Cell Size', 'Class']  # noqa
-    X_host = data[
-        [x for x in data.columns if x not in labels]
-    ]
-    Y = data['Class'].values
-    xgb_host = XGB_HOST(n_estimators=1, max_depth=2, reg_lambda=1,
-                        min_child_weight=1, objective='linear', channel=channel)
-    y_hat = np.array([0.5] * Y.shape[0])
-    for t in range(xgb_host.n_estimators):
-        f_t = pd.Series([0] * Y.shape[0])
-        gh = xgb_host.get_gh(y_hat, Y)
-
-        print("recv guest: ", xgb_host.channel.recv())
-        xgb_host.channel.send(gh)
-        GH_guest = xgb_host.channel.recv()
-        xgb_host.tree_structure[t + 1] = xgb_host.xgb_tree(X_host, GH_guest, gh, f_t, 0)  # noqa
-        y_hat = y_hat + xgb_host.learning_rate * f_t
-
-    print(xgb_host.tree_structure[1])
-    print("start predict")
-    # print(xgb_host.predict_prob(data).head())
-    output_path = ph.context.Context.get_output()
-    return xgb_host.predict_prob(data).to_csv(output_path)
-
-
-@ph.function(role='guest', protocol='xgboost', datasets=["guest_dataset"], next_peer="localhost:5555")
-def xgb_guest_logic():
-    print("start xgb guest logic...")
-    columnNames = (
-        'Sample code number',
-        'Clump Thickness',
-        'Uniformity of Cell Size',
-        'Uniformity of Cell Shape',
-        'Marginal Adhesion',
-        'Single Epithelial Cell Size',
-        'Bare Nuclei',
-        'Bland Chromatin',
-        'Normal Nucleoli',
-        'Mitoses',
-        'Class'
-    )
-    ios = IOService()
-    next_peer = ph.context.Context.nodes_context["guest"].next_peer
-    ip, port = next_peer.split(":")
-    client = Session(ios,  ip, port, "client")
-    channel = client.addChannel()
-    channel.send(b'guest ready')
-    data = ph.dataset.read(dataset_key="guest_dataset", names=columnNames).df_data
-    X_guest = data[['Clump Thickness', 'Uniformity of Cell Size']]
-    xgb_guest = XGB_GUEST(n_estimators=1, max_depth=2, reg_lambda=1, min_child_weight=1, objective='linear', channel=channel)  # noqa
-
-    for t in range(xgb_guest.n_estimators):
-        gh_host = xgb_guest.channel.recv()
-        X_guest_gh = pd.concat([X_guest, gh_host], axis=1)
-        gh_sum = xgb_guest.get_GH(X_guest_gh)
-        xgb_guest.channel.send(gh_sum)
-        xgb_guest.cart_tree(X_guest_gh, 0)

python/primihub/examples/disxgb_en.py

-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-"""
- Copyright 2022 Primihub
-
- Licensed under the Apache License, Version 2.0 (the "License");
- you may not use this file except in compliance with the License.
- You may obtain a copy of the License at
-
-      https://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
- """
-import primihub as ph
-from primihub import dataset, context
-from primihub.primitive.opt_paillier_c2py_warpper import *
-from primihub.channel.zmq_channel import IOService, Session
-from primihub.FL.model.xgboost.xgb_guest_en import XGB_GUEST_EN
-from primihub.FL.model.xgboost.xgb_host_en import XGB_HOST_EN
-from primihub.FL.model.xgboost.xgb_guest import XGB_GUEST
-from primihub.FL.model.xgboost.xgb_host import XGB_HOST
-from primihub.FL.model.evaluation.evaluation import Regression_eva
-from primihub.FL.model.evaluation.evaluation import Classification_eva
-import pandas as pd
-import numpy as np
-import logging
-import pickle
-
-
-def get_logger(name):
-    LOG_FORMAT = "[%(asctime)s][%(filename)s:%(lineno)d][%(levelname)s] %(message)s"
-    DATE_FORMAT = "%m/%d/%Y %H:%M:%S %p"
-    logging.basicConfig(level=logging.DEBUG,
-                        format=LOG_FORMAT, datefmt=DATE_FORMAT)
-    logger = logging.getLogger(name)
-    return logger
-
-
-logger = get_logger("hetero_xgb")
-
-dataset.define("guest_dataset")
-dataset.define("label_dataset")
-
-ph.context.Context.func_params_map = {
-    "xgb_host_logic": ("paillier",),
-    "xgb_guest_logic": ("paillier",)
-}
-
-# Number of tree to fit.
-num_tree = 1
-# Max depth of each tree.
-max_depth = 1
-
-
-@ph.context.function(role='host', protocol='xgboost', datasets=['label_dataset'], port='8000', task_type="regression")
-def xgb_host_logic(cry_pri="paillier"):
-    logger.info("start xgb host logic...")
-    logger.info(ph.context.Context.dataset_map)
-    logger.info(ph.context.Context.node_addr_map)
-    logger.info(ph.context.Context.role_nodeid_map)
-    logger.info(ph.context.Context.params_map)
-
-    eva_type = ph.context.Context.params_map.get("taskType", None)
-    if eva_type is None:
-        logger.warn(
-            "taskType is not specified, set to default value 'regression'.")
-        eva_type = "regression"
-
-    eva_type = eva_type.lower()
-    if eva_type != "classification" and eva_type != "regression":
-        logger.error(
-            "Invalid value of taskType, possible value is 'regression', 'classification'.")
-        return
-
-    logger.info("Current task type is {}.".format(eva_type))
-
-    data = ph.dataset.read(dataset_key="label_dataset").df_data
-    columns_label_data = data.columns.tolist()
-    for index, row in data.iterrows():
-        for name in columns_label_data:
-            temp = row[name]
-            try:
-                float(temp)
-            except ValueError:
-                logger.error(
-                    "Find illegal string '{}', it's not a digit string.".format(temp))
-                return
-
-    # Get host's ip address.
-    role_node_map = ph.context.Context.get_role_node_map()
-    node_addr_map = ph.context.Context.get_node_addr_map()
-
-    if len(role_node_map["host"]) != 1:
-        logger.error("Current node of host party: {}".format(
-            role_node_map["host"]))
-        logger.error("In hetero XGB, only dataset of host party has label, "
-                     "so host party must have one, make sure it.")
-        return
-
-    host_node = role_node_map["host"][0]
-    next_peer = node_addr_map[host_node]
-    ip, port = next_peer.split(":")
-    ios = IOService()
-    server = Session(ios, ip, port, "server")
-    channel = server.addChannel()
-
-    dim = data.shape[0]
-    dim_train = dim / 10 * 8
-    data_train = data.loc[:dim_train, :].reset_index(drop=True)
-    data_test = data.loc[dim_train:dim, :].reset_index(drop=True)
-    label_true = ['Class']
-    y_true = data_test['Class'].values
-    data_test = data_test[
-        [x for x in data_test.columns if x not in label_true]
-    ]
-    logger.info(data_test.head())
-
-    labels = ['Class']  # noqa
-    X_host = data_train[
-        [x for x in data.columns if x not in labels]
-    ]
-    Y = data_train['Class'].values
-
-    if cry_pri == "paillier":
-        xgb_host = XGB_HOST_EN(n_estimators=num_tree, max_depth=max_depth, reg_lambda=1,
-                               sid=0, min_child_weight=1, objective='linear', channel=channel)
-        channel.recv()
-        xgb_host.channel.send(xgb_host.pub)
-        print(xgb_host.channel.recv())
-        y_hat = np.array([0.5] * Y.shape[0])
-
-        for t in range(xgb_host.n_estimators):
-            logger.info("Begin to trian tree {}.".format(t + 1))
-
-            xgb_host.record = 0
-            xgb_host.lookup_table = pd.DataFrame(
-                columns=['record_id', 'feature_id', 'threshold_value'])
-            f_t = pd.Series([0] * Y.shape[0])
-            gh = xgb_host.get_gh(y_hat, Y)
-            gh_en = pd.DataFrame(columns=['g', 'h'])
-            for item in gh.columns:
-                for index in gh.index:
-                    gh_en.loc[index, item] = opt_paillier_encrypt_crt(xgb_host.pub, xgb_host.prv,
-                                                                      int(gh.loc[index, item]))
-            logger.info("Encrypt finish.")
-
-            xgb_host.channel.send(gh_en)
-            GH_guest_en = xgb_host.channel.recv()
-            GH_guest = pd.DataFrame(
-                columns=['G_left', 'G_right', 'H_left', 'H_right', 'var', 'cut'])
-            for item in [x for x in GH_guest_en.columns if x not in ['cut', 'var']]:
-                for index in GH_guest_en.index:
-                    if GH_guest_en.loc[index, item] == 0:
-                        GH_guest.loc[index, item] = 0
-                    else:
-                        GH_guest.loc[index, item] = opt_paillier_decrypt_crt(xgb_host.pub, xgb_host.prv,
-                                                                             GH_guest_en.loc[index, item])
-
-            logger.info("Decrypt finish.")
-
-            for item in [x for x in GH_guest_en.columns if x not in ['G_left', 'G_right', 'H_left', 'H_right']]:
-                for index in GH_guest_en.index:
-                    GH_guest.loc[index, item] = GH_guest_en.loc[index, item]
-
-            xgb_host.tree_structure[t + 1], f_t = xgb_host.xgb_tree(X_host, GH_guest, gh, f_t, 0)  # noqa
-            xgb_host.lookup_table_sum[t + 1] = xgb_host.lookup_table
-            y_hat = y_hat + xgb_host.learning_rate * f_t
-
-            logger.info("Finish to trian tree {}.".format(t + 1))
-
-        predict_file_path = ph.context.Context.get_predict_file_path()
-        indicator_file_path = ph.context.Context.get_indicator_file_path()
-        model_file_path = ph.context.Context.get_model_file_path()
-        lookup_file_path = ph.context.Context.get_host_lookup_file_path()
-
-        with open(model_file_path, 'wb') as fm:
-            pickle.dump(xgb_host.tree_structure, fm)
-        with open(lookup_file_path, 'wb') as fl:
-            pickle.dump(xgb_host.lookup_table_sum, fl)
-        y_pre = xgb_host.predict_prob(data_test)
-        y_train_pre = xgb_host.predict_prob(X_host)
-        y_train_pre.to_csv(predict_file_path)
-        y_train_true = Y
-        Y_true = {"train": y_train_true, "test": y_true}
-        Y_pre = {"train": y_train_pre, "test": y_pre}
-        if eva_type == 'regression':
-            Regression_eva.get_result(Y_true, Y_pre, indicator_file_path)
-        elif eva_type == 'classification':
-            Classification_eva.get_result(Y_true, Y_pre, indicator_file_path)
-
-    elif cry_pri == "plaintext":
-        xgb_host = XGB_HOST(n_estimators=num_tree, max_depth=max_depth, reg_lambda=1,
-                            sid=0, min_child_weight=1, objective='linear', channel=channel)
-        channel.recv()
-        y_hat = np.array([0.5] * Y.shape[0])
-        for t in range(xgb_host.n_estimators):
-            logger.info("Begin to trian tree {}.".format(t))
-
-            xgb_host.record = 0
-            xgb_host.lookup_table = pd.DataFrame(
-                columns=['record_id', 'feature_id', 'threshold_value'])
-            f_t = pd.Series([0] * Y.shape[0])
-            gh = xgb_host.get_gh(y_hat, Y)
-            xgb_host.channel.send(gh)
-            GH_guest = xgb_host.channel.recv()
-            xgb_host.tree_structure[t + 1], f_t = xgb_host.xgb_tree(X_host, GH_guest, gh, f_t, 0)  # noqa
-            xgb_host.lookup_table_sum[t + 1] = xgb_host.lookup_table
-            y_hat = y_hat + xgb_host.learning_rate * f_t
-
-            logger.info("Finish to trian tree {}.".format(t))
-
-        predict_file_path = ph.context.Context.get_predict_file_path()
-        indicator_file_path = ph.context.Context.get_indicator_file_path()
-        model_file_path = ph.context.Context.get_model_file_path()
-        lookup_file_path = ph.context.Context.get_host_lookup_file_path()
-
-        with open(model_file_path, 'wb') as fm:
-            pickle.dump(xgb_host.tree_structure, fm)
-        with open(lookup_file_path, 'wb') as fl:
-            pickle.dump(xgb_host.lookup_table_sum, fl)
-        y_pre = xgb_host.predict_prob(data_test)
-        if eva_type == 'regression':
-            Regression_eva.get_result(y_true, y_pre, indicator_file_path)
-        elif eva_type == 'classification':
-            Classification_eva.get_result(y_true, y_pre, indicator_file_path)
-
-        xgb_host.predict_prob(data_test).to_csv(predict_file_path)
-
-
-@ph.context.function(role='guest', protocol='xgboost', datasets=['guest_dataset'], port='9000', task_type="regression")
-def xgb_guest_logic(cry_pri="paillier"):
-    print("start xgb guest logic...")
-    ios = IOService()
-
-    logger.info(ph.context.Context.dataset_map)
-    logger.info(ph.context.Context.node_addr_map)
-    logger.info(ph.context.Context.role_nodeid_map)
-    logger.info(ph.context.Context.params_map)
-
-    eva_type = ph.context.Context.params_map.get("taskType", None)
-    if eva_type is None:
-        logger.warn(
-            "taskType is not specified, set to default value 'regression'.")
-        eva_type = "regression"
-
-    eva_type = eva_type.lower()
-    if eva_type != "classification" and eva_type != "regression":
-        logger.error(
-            "Invalid value of taskType, possible value is 'regression', 'classification'.")
-        return
-
-    logger.info("Current task type is {}.".format(eva_type))
-
-    # Check dataset.
-    data = ph.dataset.read(dataset_key="guest_dataset").df_data
-    columns_label_data = data.columns.tolist()
-    for index, row in data.iterrows():
-        for name in columns_label_data:
-            temp = row[name]
-            try:
-                float(temp)
-            except ValueError:
-                logger.error(
-                    "Find illegal string '{}', it's not a digit string.".format(temp))
-                return
-
-    # Get host's ip address.
-    role_node_map = ph.context.Context.get_role_node_map()
-    node_addr_map = ph.context.Context.get_node_addr_map()
-
-    if len(role_node_map["host"]) != 1:
-        logger.error("Current node of host party: {}".format(
-            role_node_map["host"]))
-        logger.error("In hetero XGB, only dataset of host party has label,"
-                     "so host party must have one, make sure it.")
-        return
-
-    host_node = role_node_map["host"][0]
-    next_peer = node_addr_map[host_node]
-
-    ip, port = next_peer.split(":")
-    client = Session(ios, ip, port, "client")
-    channel = client.addChannel()
-
-    dim = data.shape[0]
-    dim_train = dim / 10 * 8
-    X_guest = data.loc[:dim_train, :].reset_index(drop=True)
-    data_test = data.loc[dim_train:dim, :].reset_index(drop=True)
-
-    if cry_pri == "paillier":
-        xgb_guest = XGB_GUEST_EN(n_estimators=num_tree, max_depth=max_depth, reg_lambda=1, min_child_weight=1,
-                                 objective='linear',
-                                 sid=1, channel=channel)  # noqa
-        channel.send(b'guest ready')
-        pub = xgb_guest.channel.recv()
-        xgb_guest.channel.send(b'recved pub')
-
-        for t in range(xgb_guest.n_estimators):
-            xgb_guest.record = 0
-            xgb_guest.lookup_table = pd.DataFrame(
-                columns=['record_id', 'feature_id', 'threshold_value'])
-            gh_host = xgb_guest.channel.recv()
-            X_guest_gh = pd.concat([X_guest, gh_host], axis=1)
-            print(X_guest_gh)
-            gh_sum = xgb_guest.get_GH(X_guest_gh, pub)
-            xgb_guest.channel.send(gh_sum)
-            xgb_guest.cart_tree(X_guest_gh, 0, pub)
-            xgb_guest.lookup_table_sum[t + 1] = xgb_guest.lookup_table
-
-        lookup_file_path = ph.context.Context.get_guest_lookup_file_path()
-
-        with open(lookup_file_path, 'wb') as fl:
-            pickle.dump(xgb_guest.lookup_table_sum, fl)
-        xgb_guest.predict(data_test)
-        xgb_guest.predict(X_guest)
-    elif cry_pri == "plaintext":
-        xgb_guest = XGB_GUEST(n_estimators=num_tree, max_depth=max_depth, reg_lambda=1, min_child_weight=1,
-                              objective='linear',
-                              sid=1, channel=channel)  # noqa
-        channel.send(b'guest ready')
-        for t in range(xgb_guest.n_estimators):
-            xgb_guest.record = 0
-            xgb_guest.lookup_table = pd.DataFrame(
-                columns=['record_id', 'feature_id', 'threshold_value'])
-            gh_host = xgb_guest.channel.recv()
-            X_guest_gh = pd.concat([X_guest, gh_host], axis=1)
-            print(X_guest_gh)
-            gh_sum = xgb_guest.get_GH(X_guest_gh)
-            xgb_guest.channel.send(gh_sum)
-            xgb_guest.cart_tree(X_guest_gh, 0)
-            xgb_guest.lookup_table_sum[t + 1] = xgb_guest.lookup_table
-
-        lookup_file_path = ph.context.Context.get_guest_lookup_file_path()
-
-        with open(lookup_file_path, 'wb') as fl:
-            pickle.dump(xgb_guest.lookup_table_sum, fl)
-        xgb_guest.predict(data_test)
-        xgb_guest.predict(X_guest)

python/primihub/executor.py

@@ -67,10 +67,16 @@ class Executor:
                 while process.exitcode is None:
                     process.join(timeout=5)
                     logger.debug("Wait for FL task to finish, pid is {}".format(process.pid))
-                logger.debug("end execute {}".format(process.exitcode))
+                logger.info("end execute with exit code: {}".format(process.exitcode))
+                # process.exitcode 0 success, -exit_code failed
+                if (process.exitcode != 0):
+                    err_msg = "Task executes failed with exit code: {}".format(process.exitcode)
+                    logger.error(err_msg)
+                    raise Exception(err_msg)
             except Exception as e:
                 logger.error("Exception: ", str(e))
                 traceback.print_exc()
+                raise e
             finally:
                 Context.clean_content()
         else:
@@ -83,10 +89,16 @@ class Executor:
                 while process.exitcode is None:
                     process.join(timeout=5)
                     logger.debug("Wait for FL task to finish, pid is {}".format(process.pid))
-                logger.debug("end execute {}".format(process.exitcode))
+                logger.info("end execute with exit code: {}".format(process.exitcode))
+                # process.exitcode 0 success, -exit_code failed
+                if (process.exitcode != 0):
+                    err_msg = "Task executes failed with exit code: {}".format(process.exitcode)
+                    logger.error(err_msg)
+                    raise Exception(err_msg)
             except Exception as e:
                 logger.error("Exception: ", str(e))
                 traceback.print_exc()
+                raise e
             finally:
                 Context.clean_content()
 

python/requirements.txt

-pyarrow==4.0.0
+pyarrow==6.0.1
 pandas
 pytest
 numpy
@@ -7,8 +7,8 @@ pyzmq
 bitarray
 mmh3
 
-grpcio==1.47.0
-grpcio-tools==1.47.0
+grpcio==1.43.0
+grpcio-tools==1.43.0
 protobuf==3.20.0
 
 pydantic
@@ -19,4 +19,9 @@ phe==1.5.0
 mysql-connector-python
 loguru
 gevent
-
+redis
+ray==2.2.0
+transformers
+requests
+line_profiler
+matplotlib

python/setup.py

 import os
+import sys
 import shlex
 import subprocess
+import shutil
 import sys
 import distutils
+import tempfile
 from distutils.sysconfig import get_python_lib
 from os import path
 import site
@@ -38,32 +41,61 @@ with open('requirements.txt', encoding='utf-8') as reqs:
 
 
 def solib2sitepackage(solib_path=None):
-    # add ../bazel-bin to PYTHONPATH on LINUX
-    print("ADD ../bazel-bin to PYTHONPATH on LINUX")
-    if not solib_path:
-        solib_path = path.abspath(path.join(path.dirname(__file__), "../bazel-bin"))  # noqa
-    SOLIB_PATH = solib_path
-    print("SOLIB_PATH: ", SOLIB_PATH)
-    print("* " * 30)
-    site_packages_path = get_python_lib()
-    print(site_packages_path)
-    pth_file = site_packages_path + "/primihub_so.pth"
-    print(pth_file)
-    try:
-        with open(pth_file, "w") as pth:
-            print("--", SOLIB_PATH)
-            pth.write(SOLIB_PATH)
-        site.addsitedir(SOLIB_PATH)
-        print("sys path: ", sys.path)
-    except Exception as e:
-        print(e)
+    if '--user' in sys.argv:
+        paths = site.getusersitepackages()
+    else:
+        paths = get_python_lib()
+
+    module_list = ["opt_paillier_c2py.so", "linkcontext.so"]
+    for module_name in module_list:
+        module_installed = False
+        if os.path.isfile("../bazel-bin/{}".format(module_name)):
+            shutil.copyfile("../bazel-bin/{}".format(module_name),
+                    paths + "/{}".format(module_name))
+            print("Install {} finish, file found in '../bazel-bin'.".format(module_name))
+            module_installed = True
+        else:
+            print("Can't not find file ../bazel-bin/{}, try to find ./{}.".format(module_name, module_name))
+        if module_installed:
+            continue
+        if os.path.isfile("./{}".format(module_name)):
+            shutil.copyfile("./{}".format(module_name),
+                    paths + "/{}".format(module_name))
+            print("Install {} finish, file found in './'.".format(module_name))
+        else:
+            print("Can't not find file ./{}.".format(module_name))
+            print("Ignore {} due to file not found.".format(module_name))
+
+
+def clean_proto():
+    os.chdir("../")
+    python_out_dir = "python/primihub/client/ph_grpc"
+    proto_dir = python_out_dir + "/src/primihub/protos/"
+
+    dir_files = os.listdir(proto_dir)
+    file_to_remove = []
+    for f_name in dir_files:
+        if f_name.find("pb2.py") != -1:
+            file_to_remove.append(f_name)
+        if f_name.find("pb2_grpc.py") != -1:
+            file_to_remove.append(f_name)
+
+    for fname in file_to_remove:
+        os.remove(proto_dir + fname)
+        print("Remove {}.".format(fname))
+
+    os.chdir("python")
+    print(os.getcwd())
 
 
 def compile_proto():
     print("compile proto...")
+    os.chdir("../")
     print(os.getcwd())
-    os.chdir("..")
-    print(os.getcwd())
+
+    python_out_dir = "python/primihub/client/ph_grpc"
+    grpc_python_out = "python/primihub/client/ph_grpc"
+
     cmd_str = """{pyexe} -m grpc_tools.protoc \
                         --python_out={python_out} \
                         --grpc_python_out={grpc_python_out} -I. \
@@ -71,8 +103,8 @@ def compile_proto():
                         src/primihub/protos/service.proto \
                         src/primihub/protos/worker.proto \
                         src/primihub/protos/pir.proto \
-                        src/primihub/protos/psi.proto""".format(python_out="python/primihub/client/ph_grpc",
-                                                                grpc_python_out="python/primihub/client/ph_grpc",
+                        src/primihub/protos/psi.proto""".format(python_out=python_out_dir,
+                                                                grpc_python_out=grpc_python_out,
                                                                 pyexe=sys.executable)
 
     print("cmd: %s" % cmd_str)
@@ -81,6 +113,7 @@ def compile_proto():
     out, err = p.communicate()
     if p.returncode != 0:
         raise RuntimeError("Error compiling proto file: {0}".format(err))
+
     os.chdir("python")
     print(os.getcwd())
 
@@ -90,9 +123,9 @@ class PostDevelopCommand(develop):
 
     def run(self):
         # PUT YOUR POST-INSTALL SCRIPT HERE or CALL A FUNCTION
+        compile_proto()
         develop.run(self)
         solib2sitepackage()
-        compile_proto()
 
 
 class PostInstallCommand(install):
@@ -100,9 +133,10 @@ class PostInstallCommand(install):
 
     def run(self):
         # PUT YOUR POST-INSTALL SCRIPT HERE or CALL A FUNCTION
+        compile_proto()
         install.run(self)
         solib2sitepackage()
-        compile_proto()
+        clean_proto()
 
 
 class ProtoCommand(distutils.cmd.Command):
@@ -124,7 +158,7 @@ class ProtoCommand(distutils.cmd.Command):
 
 
 class AddSoLibCommand(distutils.cmd.Command):
-    """Run command: 
+    """Run command:
         add so lib path to Python path.
     """
 

src/primihub/algorithm/arithmetic.cc

@@ -33,7 +33,7 @@ ArithmeticExecutor<Dbit>::ArithmeticExecutor(
     : AlgorithmBase(dataset_service) {
   this->algorithm_name_ = "arithmetic";
 
-  std::map<std::string, Node> &node_map = config.node_map;
+  auto& node_map = config.node_map;
   // LOG(INFO) << node_map.size();
   std::map<uint16_t, rpc::Node> party_id_node_map;
   for (auto iter = node_map.begin(); iter != node_map.end(); iter++) {
@@ -356,7 +356,7 @@ int ArithmeticExecutor<Dbit>::_LoadDatasetFromCSV(std::string &filename) {
         table->column(num_col - 1)->chunk(k));
     array_len += array->length();
   }
-  
+
   LOG(INFO) << "Label column '" << col_names[num_col - 1] << "' has "
             << array_len << " values.";
   // Force the same value count in every column.

src/primihub/algorithm/cryptflow2_maxpool.cc

@@ -89,7 +89,7 @@ MaxPoolExecutor::MaxPoolExecutor(
   this->algorithm_name_ = "maxpool";
   // set the party id to be 1,2.
   uint16_t party_index = 0;
-  std::map<std::string, Node> &node_map = config.node_map;
+  auto& node_map = config.node_map;
   for (auto iter = node_map.begin(); iter != node_map.end(); iter++) {
     rpc::Node &node = iter->second;
     auto *vm = node.mutable_vm();

src/primihub/algorithm/falcon_lenet.cc

@@ -19,7 +19,7 @@ FalconLenetExecutor::FalconLenetExecutor(
   this->algorithm_name_ = "Falcon_Lenet_Train";
 
   node_id_ = config.node_id;
-  std::map<std::string, Node> node_map = config.node_map;
+  auto& node_map = config.node_map;
   auto iter = node_map.find(node_id_);
   if (iter == node_map.end()) {
     std::stringstream ss;

src/primihub/algorithm/logistic.cc

@@ -104,7 +104,7 @@ LogisticRegressionExecutor::LogisticRegressionExecutor(
 {
   this->algorithm_name_ = "logistic_regression";
 
-  std::map<std::string, Node> &node_map = config.node_map;
+  auto& node_map = config.node_map;
   LOG(INFO) << node_map.size();
   std::map<uint16_t, rpc::Node> party_id_node_map;
   for (auto iter = node_map.begin(); iter != node_map.end(); iter++) {

src/primihub/cli/cli.h

@@ -40,8 +40,10 @@
 #include "absl/strings/str_cat.h"
 
 #include "src/primihub/protos/worker.grpc.pb.h"
+#include "src/primihub/protos/common.pb.h"
 #include "src/primihub/common/config/config.h"
 #include "src/primihub/util/util.h"
+#include "src/primihub/protos/service.grpc.pb.h"
 
 using primihub::rpc::VMNode;
 using primihub::rpc::PushTaskRequest;

src/primihub/data_store/csv/csv_driver.cc

@@ -125,7 +125,7 @@ std::shared_ptr<Cursor> &CSVDriver::initCursor(const std::string &filePath) {
 
 // FIXME to be deleted write file in driver directly.
 int CSVDriver::write(std::shared_ptr<arrow::Table> table,
-                     std::string &filePath) {
+                     const std::string& filePath) {
   filePath_ = filePath;
   auto result = arrow::io::FileOutputStream::Open(filePath);
   if (!result.ok()) {

src/primihub/data_store/csv/csv_driver.h

@@ -49,7 +49,7 @@ public:
   std::string getDataURL() const override;
   // FIXME to be deleted
   int write(std::shared_ptr<arrow::Table> table,
-                     std::string &filePath);
+                    const std::string &filePath);
 
 private:
   std::string filePath_;

src/primihub/task/semantic/psi_server_task.cc

@@ -15,7 +15,6 @@
  */
 
 #include "private_set_intersection/cpp/psi_server.h"
-
 #include "src/primihub/task/semantic/psi_server_task.h"
 #include "src/primihub/util/util.h"