concrete-sys

.github/workflows/concrete_compiler_test_cpu.yml

@@ -50,17 +50,23 @@ jobs:
           sudo apt install moreutils
           cd compilers/concrete-compiler/compiler
           ./scripts/format_cpp.sh
-      - name: Upload format diff
+      - name: Upload format_cpp_diff
         if: failure()
         uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3
         with:
-          name: format_diff
-          path: compilers/concrete-compiler/compiler/format_diff.patch
+          name: format_cpp_diff
+          path: compilers/concrete-compiler/compiler/format_cpp_diff.patch
       - name: Format with cmake-format (Cmake)
         run: |
           pip3 install cmakelang
           cd compilers/concrete-compiler/compiler
           ./scripts/format_cmake.sh
+      - name: Upload format_cmake_diff
+        if: failure()
+        uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3
+        with:
+          name: format_cmake_diff
+          path: compilers/concrete-compiler/compiler/format_cmake_diff.patch
       - name: Format with black (Python)
         run: |
           cd compilers/concrete-compiler/compiler
@@ -110,6 +116,7 @@ jobs:
             set -e
             cd /concrete/compilers/concrete-compiler/compiler
             rm -rf /build/*
+            cargo install cxxbridge-cmd
             make DATAFLOW_EXECUTION_ENABLED=ON Python3_EXECUTABLE=$PYTHON_EXEC BUILD_DIR=/build all
             echo "Debug: ccache statistics (after the build):"
             ccache -s

.github/workflows/concrete_compiler_test_cpu_distributed.yml

@@ -66,6 +66,7 @@ jobs:
         run: |
           cd compilers/concrete-compiler/compiler
           rm -rf /shared/build
+          cargo install cxxbridge-cmd
           make HPX_DIR=/shared/hpx install-hpx-from-source
           make HPX_DIR=/shared/hpx BUILD_DIR=/shared/build CCACHE=ON DATAFLOW_EXECUTION_ENABLED=ON BINDINGS_PYTHON_ENABLED=OFF build-end-to-end-tests
 

.github/workflows/concrete_compiler_test_gpu.yml

@@ -71,6 +71,7 @@ jobs:
             set -e
             cd /concrete/compilers/concrete-compiler/compiler
             rm -rf /build/*
+            cargo install cxxbridge-cmd
             mkdir -p /tmp/concrete_compiler/gpu_tests/
             make BINDINGS_PYTHON_ENABLED=OFF Python3_EXECUTABLE=$PYTHON_EXEC CUDA_SUPPORT=ON CUDA_PATH=${{ env.CUDA_PATH }} run-end-to-end-tests-gpu
             echo "Debug: ccache statistics (after the build):"

.github/workflows/concrete_compiler_test_macos_cpu.yml

@@ -60,6 +60,7 @@ jobs:
           cd compilers/concrete-compiler/compiler
           echo "Debug: ccache statistics (prior to the build):"
           ccache -s
+          cargo install cxxbridge-cmd
           make Python3_EXECUTABLE="${PYTHON_EXEC}" all
           echo "Debug: ccache statistics (after the build):"
           ccache -s

compilers/concrete-compiler/compiler/CMakeLists.txt

@@ -85,35 +85,7 @@ install(EXPORT concrete-protocol DESTINATION "./")
 # -------------------------------------------------------------------------------
 
 set(CONCRETE_OPTIMIZER_DIR "${PROJECT_SOURCE_DIR}/../../concrete-optimizer")
-if(CMAKE_BUILD_TYPE STREQUAL "Debug")
-  set(CONCRETE_OPTIMIZER_BUILD_DIR "${CONCRETE_OPTIMIZER_DIR}/target/debug")
-  set(CONCRETE_OPTIMIZER_PROFILE "dev")
-else()
-  set(CONCRETE_OPTIMIZER_BUILD_DIR "${CONCRETE_OPTIMIZER_DIR}/target/release")
-  set(CONCRETE_OPTIMIZER_PROFILE "release")
-endif()
 set(CONCRETE_OPTIMIZER_INCLUDE_DIR "${CONCRETE_OPTIMIZER_DIR}/concrete-optimizer-cpp/src/cpp")
-set(CONCRETE_OPTIMIZER_STATIC_LIB "${CONCRETE_OPTIMIZER_BUILD_DIR}/libconcrete_optimizer_cpp.a")
-
-ExternalProject_Add(
-  concrete_optimizer_rust
-  DOWNLOAD_COMMAND ""
-  CONFIGURE_COMMAND "" OUTPUT "${CONCRETE_OPTIMIZER_STATIC_LIB}"
-  BUILD_ALWAYS true
-  BUILD_COMMAND cargo build -p concrete-optimizer-cpp --profile ${CONCRETE_OPTIMIZER_PROFILE}
-  BINARY_DIR "${CONCRETE_OPTIMIZER_DIR}"
-  INSTALL_COMMAND cp ${CONCRETE_OPTIMIZER_STATIC_LIB} ${CMAKE_LIBRARY_OUTPUT_DIRECTORY}
-  LOG_BUILD ON
-  LOG_OUTPUT_ON_FAILURE ON)
-
-add_library(concrete_optimizer STATIC ${CONCRETE_OPTIMIZER_DIR}/concrete-optimizer-cpp/src/cpp/concrete-optimizer.cpp)
-
-target_link_libraries(concrete_optimizer PRIVATE pthread m dl "${CONCRETE_OPTIMIZER_STATIC_LIB}")
-install(TARGETS concrete_optimizer EXPORT concrete_optimizer)
-install(EXPORT concrete_optimizer DESTINATION "./")
-
-add_dependencies(concrete_optimizer concrete_optimizer_rust)
-# TODO - Remove the global include directory
 include_directories(${CONCRETE_OPTIMIZER_INCLUDE_DIR})
 
 # -------------------------------------------------------------------------------
@@ -127,48 +99,41 @@ set(CONCRETE_BACKENDS_DIR "${PROJECT_SOURCE_DIR}/../../../backends")
 # -------------------------------------------------------------------------------
 
 set(CONCRETE_CPU_DIR "${CONCRETE_BACKENDS_DIR}/concrete-cpu/implementation")
-set(CONCRETE_CPU_RELEASE_DIR "${CONCRETE_CPU_DIR}/target/release")
 set(CONCRETE_CPU_INCLUDE_DIR "${CONCRETE_CPU_DIR}/include")
-set(CONCRETE_CPU_STATIC_LIB "${CONCRETE_CPU_RELEASE_DIR}/libconcrete_cpu.a")
-
-ExternalProject_Add(
-  concrete_cpu_rust
-  DOWNLOAD_COMMAND ""
-  CONFIGURE_COMMAND "" OUTPUT "${CONCRETE_CPU_STATIC_LIB}"
-  BUILD_ALWAYS true
-  BUILD_COMMAND cargo +nightly-2024-09-30 build --release --features=nightly
-  BINARY_DIR "${CONCRETE_CPU_DIR}"
-  INSTALL_COMMAND ""
-  LOG_BUILD ON
-  LOG_OUTPUT_ON_FAILURE ON)
-
-add_library(concrete_cpu STATIC IMPORTED)
-set_target_properties(concrete_cpu PROPERTIES IMPORTED_LOCATION "${CONCRETE_CPU_STATIC_LIB}")
-add_dependencies(concrete_cpu concrete_cpu_rust)
 
 # -------------------------------------------------------------------------------
 # Concrete CPU Noise Model Configuration
 # -------------------------------------------------------------------------------
 
 set(CONCRETE_CPU_NOISE_MODEL_DIR "${CONCRETE_BACKENDS_DIR}/concrete-cpu/noise-model")
-set(CONCRETE_CPU_NOISE_MODEL_RELEASE_DIR "${CONCRETE_CPU_NOISE_MODEL_DIR}/target/release")
 set(CONCRETE_CPU_NOISE_MODEL_INCLUDE_DIR "${CONCRETE_CPU_NOISE_MODEL_DIR}/include")
-set(CONCRETE_CPU_NOISE_MODEL_STATIC_LIB "${CONCRETE_CPU_NOISE_MODEL_RELEASE_DIR}/libconcrete_cpu_noise_model.a")
+
+# -------------------------------------------------------------------------------
+# RUST DEPS BUNDLE
+# -------------------------------------------------------------------------------
+
+set(RUST_DEPS_BUNDLE_DIR "${PROJECT_SOURCE_DIR}/../../../tools/rust_deps_bundle")
+if(CMAKE_BUILD_TYPE STREQUAL "Debug")
+  set(RUST_DEPS_BUNDLE_BUILD_DIR "${RUST_DEPS_BUNDLE_DIR}/target/debug")
+else()
+  set(RUST_DEPS_BUNDLE_BUILD_DIR "${RUST_DEPS_BUNDLE_DIR}/target/release")
+endif()
+set(RUST_DEPS_BUNDLE_LIB "${RUST_DEPS_BUNDLE_BUILD_DIR}/librust_deps_bundle.a")
 
 ExternalProject_Add(
-  concrete_cpu_noise_model_rust
+  rust_deps_bundle_build
   DOWNLOAD_COMMAND ""
-  CONFIGURE_COMMAND "" OUTPUT "${CONCRETE_CPU_NOISE_MODEL_STATIC_LIB}"
+  CONFIGURE_COMMAND "" OUTPUT "${RUST_DEPS_BUNDLE_LIB}"
   BUILD_ALWAYS true
-  BUILD_COMMAND cargo build --release
-  BINARY_DIR "${CONCRETE_CPU_NOISE_MODEL_DIR}"
+  BUILD_COMMAND cargo +nightly-2024-09-30 build --release
+  BINARY_DIR "${RUST_DEPS_BUNDLE_DIR}"
   INSTALL_COMMAND ""
   LOG_BUILD ON
   LOG_OUTPUT_ON_FAILURE ON)
 
-add_library(concrete_cpu_noise_model STATIC IMPORTED)
-set_target_properties(concrete_cpu_noise_model PROPERTIES IMPORTED_LOCATION "${CONCRETE_CPU_NOISE_MODEL_STATIC_LIB}")
-add_dependencies(concrete_cpu_noise_model concrete_cpu_noise_model_rust)
+add_library(rust_deps_bundle STATIC IMPORTED)
+set_target_properties(rust_deps_bundle PROPERTIES IMPORTED_LOCATION "${RUST_DEPS_BUNDLE_LIB}")
+add_dependencies(rust_deps_bundle rust_deps_bundle_build)
 
 # --------------------------------------------------------------------------------
 # Concrete Cuda Configuration

compilers/concrete-compiler/compiler/Makefile

@@ -120,7 +120,7 @@ ifeq ($(BUILD_TYPE),Debug)
 	LIBOMP_LINK_TO_LIBSTDCXX_OPT=-DLIBOMP_USE_STDCPPLIB=ON
 endif
 
-all: concretecompiler python-bindings build-tests build-benchmarks doc
+all: concretecompiler python-bindings build-tests build-benchmarks doc concrete-sys
 
 # HPX #####################################################
 
@@ -154,7 +154,7 @@ $(BUILD_DIR)/configured.stamp:
 	-DCMAKE_BUILD_TYPE=$(BUILD_TYPE) \
 	-DLLVM_ENABLE_ASSERTIONS=ON \
 	-DLLVM_ENABLE_RTTI=ON \
-	-DLLVM_LINK_LLVM_DYLIB=on \
+	-DLLVM_LINK_LLVM_DYLIB=OFF \
 	-DMLIR_ENABLE_BINDINGS_PYTHON=$(BINDINGS_PYTHON_ENABLED) \
 	-DCONCRETELANG_BINDINGS_PYTHON_ENABLED=$(BINDINGS_PYTHON_ENABLED) \
 	-DCONCRETELANG_DATAFLOW_EXECUTION_ENABLED=$(DATAFLOW_EXECUTION_ENABLED) \
@@ -195,7 +195,8 @@ clientlib: build-initialized
 serverlib: build-initialized
 	cmake --build $(BUILD_DIR) --target ConcretelangServerLib
 
-
+concrete-sys: build-initialized
+	cmake --build $(BUILD_DIR) --target ConcreteSys
 
 GITHUB_URL=https://api.github.com/repos/zama-ai/concrete-compiler-internal
 GITHUB_URL_LIST_ARTIFACTS="${GITHUB_URL}/actions/artifacts?name=${KEYSETCACHENAME}&per_page=1"
@@ -233,7 +234,7 @@ endif
 
 build-tests: build-unit-tests build-end-to-end-tests
 
-run-tests: run-check-tests run-unit-tests run-end-to-end-tests run-random-end-to-end-tests-for-each-options run-python-tests
+run-tests: run-check-tests run-unit-tests run-end-to-end-tests run-random-end-to-end-tests-for-each-options run-python-tests run-rust-tests
 
 ## check-tests
 
@@ -253,6 +254,11 @@ run-unit-tests: build-unit-tests
 run-python-tests: python-bindings concretecompiler
 	PYTHONPATH=${PYTHONPATH}:$(BUILD_DIR)/tools/concretelang/python_packages/concretelang_core LD_PRELOAD=$(BUILD_DIR)/lib/libConcretelangRuntime.so pytest -vs -m $(PYTHON_TESTS_MARKER) tests/python
 
+## rust-tests
+
+run-rust-tests: concrete-sys
+	cd lib/Bindings/Rust && cargo test
+
 test-compiler-file-output: concretecompiler
 	pytest -vs tests/test_compiler_file_output