feat(compiler): add concrete-sys rust bindings

compilers/concrete-compiler/compiler/Makefile

@@ -120,7 +120,7 @@ ifeq ($(BUILD_TYPE),Debug)
 	LIBOMP_LINK_TO_LIBSTDCXX_OPT=-DLIBOMP_USE_STDCPPLIB=ON
 endif
 
-all: concretecompiler python-bindings build-tests build-benchmarks doc
+all: concretecompiler python-bindings build-tests build-benchmarks doc concrete-sys
 
 # HPX #####################################################
 
@@ -154,7 +154,7 @@ $(BUILD_DIR)/configured.stamp:
 	-DCMAKE_BUILD_TYPE=$(BUILD_TYPE) \
 	-DLLVM_ENABLE_ASSERTIONS=ON \
 	-DLLVM_ENABLE_RTTI=ON \
-	-DLLVM_LINK_LLVM_DYLIB=on \
+	-DLLVM_LINK_LLVM_DYLIB=OFF \
 	-DMLIR_ENABLE_BINDINGS_PYTHON=$(BINDINGS_PYTHON_ENABLED) \
 	-DCONCRETELANG_BINDINGS_PYTHON_ENABLED=$(BINDINGS_PYTHON_ENABLED) \
 	-DCONCRETELANG_DATAFLOW_EXECUTION_ENABLED=$(DATAFLOW_EXECUTION_ENABLED) \
@@ -195,7 +195,8 @@ clientlib: build-initialized
 serverlib: build-initialized
 	cmake --build $(BUILD_DIR) --target ConcretelangServerLib
 
-
+concrete-sys: build-initialized
+	cmake --build $(BUILD_DIR) --target ConcreteSys
 
 GITHUB_URL=https://api.github.com/repos/zama-ai/concrete-compiler-internal
 GITHUB_URL_LIST_ARTIFACTS="${GITHUB_URL}/actions/artifacts?name=${KEYSETCACHENAME}&per_page=1"
@@ -233,7 +234,7 @@ endif
 
 build-tests: build-unit-tests build-end-to-end-tests
 
-run-tests: run-check-tests run-unit-tests run-end-to-end-tests run-random-end-to-end-tests-for-each-options run-python-tests
+run-tests: run-check-tests run-unit-tests run-end-to-end-tests run-random-end-to-end-tests-for-each-options run-python-tests run-rust-tests
 
 ## check-tests
 
@@ -253,6 +254,11 @@ run-unit-tests: build-unit-tests
 run-python-tests: python-bindings concretecompiler
 	PYTHONPATH=${PYTHONPATH}:$(BUILD_DIR)/tools/concretelang/python_packages/concretelang_core LD_PRELOAD=$(BUILD_DIR)/lib/libConcretelangRuntime.so pytest -vs -m $(PYTHON_TESTS_MARKER) tests/python
 
+## rust-tests
+
+run-rust-tests: concrete-sys
+	cd lib/Bindings/Rust && cargo test
+
 test-compiler-file-output: concretecompiler
 	pytest -vs tests/test_compiler_file_output
 

compilers/concrete-compiler/compiler/include/concretelang/Support/CompilerEngine.h

@@ -6,8 +6,6 @@
 #ifndef CONCRETELANG_SUPPORT_COMPILER_ENGINE_H
 #define CONCRETELANG_SUPPORT_COMPILER_ENGINE_H
 
-#include "capnp/message.h"
-#include "concrete-protocol.capnp.h"
 #include "concretelang/Common/Protocol.h"
 #include "concretelang/Conversion/Utils/GlobalFHEContext.h"
 #include "concretelang/Support/Encodings.h"

compilers/concrete-compiler/compiler/lib/Bindings/CMakeLists.txt

@@ -1,3 +1,5 @@
 if(CONCRETELANG_BINDINGS_PYTHON_ENABLED)
   add_subdirectory(Python)
 endif()
+
+add_subdirectory(Rust)

compilers/concrete-compiler/compiler/lib/Common/CMakeLists.txt

@@ -1,5 +1,12 @@
 add_compile_options(-fexceptions -fsized-deallocation -fno-rtti)
 
+add_library(
+    _ConcreteSys_ConcretelangCommon_Sec
+    STATIC
+    Security.cpp
+)
+target_include_directories(_ConcreteSys_ConcretelangCommon_Sec PUBLIC ${CONCRETE_CPU_INCLUDE_DIR})
+
 add_mlir_library(
   ConcretelangCommon
   Protocol.cpp
@@ -14,7 +21,6 @@ add_mlir_library(
   concrete-protocol
   LINK_LIBS
   PUBLIC
-  concrete_cpu
   kj
   capnp)
 

compilers/concrete-compiler/compiler/lib/Dialect/RT/Analysis/CMakeLists.txt

@@ -12,5 +12,4 @@ add_mlir_library(
   LINK_LIBS
   PUBLIC
   MLIRIR
-  RTDialect
-  ConcretelangRuntime)
+  RTDialect)

compilers/concrete-compiler/compiler/lib/Runtime/CMakeLists.txt

@@ -1,5 +1,13 @@
 add_compile_options(-fsized-deallocation)
 
+add_library(
+    _ConcreteSys_ConcretelangRuntime_Dfr
+    STATIC
+    DFRuntime.cpp
+    GPUDFG.cpp
+    utils.cpp
+)
+
 if(CONCRETELANG_CUDA_SUPPORT)
   add_library(
     ConcretelangRuntime SHARED
@@ -67,7 +75,15 @@ target_link_libraries(
          dl
          $<TARGET_OBJECTS:mlir_c_runner_utils>
          $<TARGET_OBJECTS:mlir_float16_utils>
-         $<TARGET_OBJECTS:MLIRSparseTensorRuntime>)
+         $<TARGET_OBJECTS:MLIRSparseTensorRuntime>
+)
+target_link_libraries(
+    ConcretelangRuntime
+    PRIVATE
+    LLVMSupport
+    LLVMTarget
+    LLVMAArch64CodeGen
+)
 
 if(CONCRETELANG_CUDA_SUPPORT)
   install(TARGETS ConcretelangRuntime omp tfhe_cuda_backend EXPORT ConcretelangRuntime)

compilers/concrete-compiler/compiler/lib/ServerLib/CMakeLists.txt

@@ -16,5 +16,4 @@ add_mlir_library(
   DEPENDS
   mlir-headers
   LINK_LIBS
-  ConcretelangRuntime
   ConcretelangCommon)

compilers/concrete-compiler/compiler/lib/Support/CMakeLists.txt

@@ -38,9 +38,9 @@ add_mlir_library(
   concrete_optimizer
   MLIRExecutionEngine
   ${LLVM_PTHREAD_LIB}
-  ConcretelangRuntime
-  ConcretelangClientLib
-  ConcretelangServerLib
+  concrete-protocol
+  capnp
+  kj
   TFHEDialectAnalysis
   ConcreteDialectAnalysis)
 

compilers/concrete-compiler/compiler/lib/Support/CompilationFeedback.cpp

@@ -13,6 +13,26 @@
 
 using concretelang::protocol::Message;
 
+
+// TODO: the two following functions are the only dependencies on `concrete-cpu`.
+// Unfortunately, having both `concrete-cpu` and `concrete-optimizer` linked statically does
+// not work (because of non-mangled private duplicate symbols clashing). This is in the process of
+// being solved on the rustc side, but it will likely take a bit of time.
+//
+// In the meantime, since the function is fairly straightforward, we copy it here.
+size_t concrete_cpu_bootstrap_key_size_u64(size_t decomposition_level_count,
+                                           size_t glwe_dimension,
+                                           size_t polynomial_size,
+                                           size_t input_lwe_dimension){
+    return input_lwe_dimension * decomposition_level_count * polynomial_size * (glwe_dimension + 1) * (glwe_dimension + 1);
+}
+
+size_t concrete_cpu_keyswitch_key_size_u64(size_t decomposition_level_count,
+                                           size_t input_dimension,
+                                           size_t output_dimension) {
+    return input_dimension * decomposition_level_count * (output_dimension + 1);
+}
+
 namespace mlir {
 namespace concretelang {
 

compilers/concrete-compiler/compiler/lib/Support/ProgramInfoGeneration.cpp

@@ -14,7 +14,6 @@
 #include "concrete-protocol.capnp.h"
 #include "concretelang/Common/Protocol.h"
 #include "concretelang/Common/Security.h"
-#include "concretelang/Common/Values.h"
 #include "concretelang/Conversion/Utils/GlobalFHEContext.h"
 #include "concretelang/Dialect/Concrete/IR/ConcreteTypes.h"
 #include "concretelang/Dialect/FHE/IR/FHETypes.h"
@@ -36,6 +35,22 @@
 #include "llvm/Config/abi-breaking.h"
 #include "llvm/Support/Error.h"
 
+size_t getCorrespondingPrecision(size_t originalPrecision) {
+  if (originalPrecision <= 8) {
+    return 8;
+  }
+  if (originalPrecision <= 16) {
+    return 16;
+  }
+  if (originalPrecision <= 32) {
+    return 32;
+  }
+  if (originalPrecision <= 64) {
+    return 64;
+  }
+  assert(false);
+}
+
 using concretelang::protocol::Message;
 
 namespace mlir {
@@ -152,14 +167,14 @@ generateGate(mlir::Type inputType,
     auto plaintextGateInfo = output.asBuilder().initTypeInfo().initPlaintext();
     plaintextGateInfo.setShape(inputShape);
     plaintextGateInfo.setIntegerPrecision(
-        ::concretelang::values::getCorrespondingPrecision(
+        getCorrespondingPrecision(
             inputType.getIntOrFloatBitWidth()));
     plaintextGateInfo.setIsSigned(inputType.isSignedInteger());
 
     auto rawInfo = output.asBuilder().initRawInfo();
     rawInfo.setShape(inputShape);
     rawInfo.setIntegerPrecision(
-        ::concretelang::values::getCorrespondingPrecision(
+        getCorrespondingPrecision(
             inputType.getIntOrFloatBitWidth()));
     rawInfo.setIsSigned(inputType.isSignedInteger());
   } else if (inputEncoding.hasIndex()) {

compilers/concrete-compiler/compiler/src/CMakeLists.txt

@@ -18,6 +18,7 @@ target_link_libraries(
           SDFGDialect
           OptimizerDialect
           ConcretelangSupport
+          ConcretelangRuntime
           ConcretelangTransforms
           MLIRIR
           MLIRLLVMIRTransforms

compilers/concrete-compiler/compiler/tests/end_to_end_benchmarks/CMakeLists.txt

@@ -1,7 +1,23 @@
 add_executable(end_to_end_benchmark end_to_end_benchmark.cpp)
-target_link_libraries(end_to_end_benchmark benchmark::benchmark ConcretelangSupport EndToEndFixture)
+target_link_libraries(
+    end_to_end_benchmark
+    benchmark::benchmark
+    ConcretelangSupport
+    ConcretelangClientLib
+    ConcretelangServerLib
+    ConcretelangRuntime
+    EndToEndFixture
+)
 set_source_files_properties(end_to_end_benchmark.cpp PROPERTIES COMPILE_FLAGS "-fno-rtti")
 
 add_executable(end_to_end_mlbench end_to_end_mlbench.cpp)
-target_link_libraries(end_to_end_mlbench benchmark::benchmark ConcretelangSupport EndToEndFixture)
+target_link_libraries(
+    end_to_end_mlbench
+    benchmark::benchmark
+    ConcretelangSupport
+    ConcretelangClientLib
+    ConcretelangServerLib
+    ConcretelangRuntime
+    EndToEndFixture
+)
 set_source_files_properties(end_to_end_mlbench.cpp PROPERTIES COMPILE_FLAGS "-fno-rtti -fsized-deallocation")

compilers/concrete-compiler/compiler/tests/end_to_end_tests/CMakeLists.txt

@@ -3,7 +3,16 @@ add_compile_options(-fexceptions)
 
 function(add_concretecompiler_unittest test_name)
   add_unittest(ConcreteCompilerUnitTests ${test_name} ${ARGN})
-  target_link_libraries(${test_name} PRIVATE ConcretelangSupport EndToEndFixture ${RPATH_FLAGS})
+  target_link_libraries(
+    ${test_name}
+    PRIVATE
+    ConcretelangSupport
+    ConcretelangClientLib
+    ConcretelangServerLib
+    ConcretelangRuntime
+    EndToEndFixture
+    ${RPATH_FLAGS}
+  )
   set_source_files_properties(${ARGN} PROPERTIES COMPILE_FLAGS "-fno-rtti")
 endfunction()
 

compilers/concrete-compiler/compiler/tests/unit_tests/concretelang/Dialect/TFHE/Transforms/CMakeLists.txt

@@ -5,5 +5,14 @@ add_dependencies(ConcretelangUnitTests ConcretelangTFHETransformsTests)
 add_unittest(ConcretelangTFHETransformsTests unit_tests_concretelang_tfhe_transforms Parametrization.cpp)
 
 target_link_libraries(
-  unit_tests_concretelang_tfhe_transforms PRIVATE TFHEDialectTransforms MLIRParser MLIRExecutionEngine
-                                                  TFHEGlobalParametrization ConcretelangSupport)
+    unit_tests_concretelang_tfhe_transforms
+    PRIVATE
+    TFHEDialectTransforms
+    MLIRParser
+    MLIRExecutionEngine
+    TFHEGlobalParametrization
+    ConcretelangSupport
+    ConcretelangClientLib
+    ConcretelangServerLib
+    ConcretelangRuntime
+)

compilers/concrete-compiler/compiler/tests/unit_tests/concretelang/Encodings/CMakeLists.txt

@@ -4,8 +4,15 @@ add_dependencies(ConcretelangUnitTests EncodingsUnitTests)
 
 function(add_concretecompiler_lib_test test_name)
   add_unittest(EncodingsUnitTests ${test_name} ${ARGN})
-  target_link_libraries(${test_name} PRIVATE ConcretelangSupport)
-  target_link_libraries(${test_name} PRIVATE ConcretelangCommon)
+  target_link_libraries(
+    ${test_name}
+    PRIVATE
+    ConcretelangSupport
+    ConcretelangCommon
+    ConcretelangClientLib
+    ConcretelangServerLib
+    ConcretelangRuntime
+  )
   set_source_files_properties(${ARGN} PROPERTIES COMPILE_FLAGS "-fno-rtti")
 endfunction()
 

compilers/concrete-compiler/compiler/tests/unit_tests/concretelang/SDFG/CMakeLists.txt

@@ -4,7 +4,14 @@ add_dependencies(ConcretelangUnitTests SDFGUnitTests)
 
 function(add_concretecompiler_lib_test test_name)
   add_unittest(SDFGUnitTests ${test_name} ${ARGN})
-  target_link_libraries(${test_name} PRIVATE ConcretelangSupport)
+  target_link_libraries(
+    ${test_name}
+    PRIVATE
+    ConcretelangSupport
+    ConcretelangClientLib
+    ConcretelangServerLib
+    ConcretelangRuntime
+  )
   set_source_files_properties(${ARGN} PROPERTIES COMPILE_FLAGS "-fno-rtti")
 endfunction()
 

compilers/concrete-compiler/compiler/tests/unit_tests/concretelang/TestLib/CMakeLists.txt

@@ -4,7 +4,14 @@ add_dependencies(ConcretelangUnitTests ConcreteCompilerLibTests)
 
 function(add_concretecompiler_lib_test test_name)
   add_unittest(ConcreteCompilerLibTests ${test_name} ${ARGN})
-  target_link_libraries(${test_name} PRIVATE ConcretelangSupport)
+  target_link_libraries(
+    ${test_name}
+    PRIVATE
+    ConcretelangSupport
+    ConcretelangClientLib
+    ConcretelangServerLib
+    ConcretelangRuntime
+  )
   set_source_files_properties(${ARGN} PROPERTIES COMPILE_FLAGS "-fno-rtti")
 endfunction()