diff --git a/.dockerignore b/.dockerignore index a9a67f955..85ee674e9 100644 --- a/.dockerignore +++ b/.dockerignore @@ -5,5 +5,6 @@ htmlcov dist **/__pycache__ cpp/build +cpp/memgraph/build Dockerfile* -.dockerignore \ No newline at end of file +.dockerignore diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 5b61332e2..12cba45ba 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -9,7 +9,8 @@ env: NEO4J_CONTAINER: "neo4j_test" OFFICIAL: "true" -on: [pull_request, workflow_dispatch] +# TODO(gitbuda): Revert +# on: [pull_request, workflow_dispatch] jobs: build: diff --git a/.github/workflows/test_no_ml.yml b/.github/workflows/test_no_ml.yml index b3d71b4dd..aba6b4f62 100644 --- a/.github/workflows/test_no_ml.yml +++ b/.github/workflows/test_no_ml.yml @@ -9,7 +9,8 @@ env: NEO4J_CONTAINER: "neo4j_test" OFFICIAL: "true" -on: [pull_request, workflow_dispatch] +# TODO(gitbuda): Revert +# on: [pull_request, workflow_dispatch] jobs: build: diff --git a/Dockerfile.cugraph b/Dockerfile.cugraph index 9f744d5af..82d5f8660 100644 --- a/Dockerfile.cugraph +++ b/Dockerfile.cugraph @@ -1,12 +1,13 @@ -ARG CUGRAPH_VERSION=22.02 -ARG CUDA_VERSION=11.5 -ARG CUDA_VERSION_MINOR=11.5.2 -ARG PY_VERSION=3.8 -ARG MG_VERSION=2.14.1 +ARG CUGRAPH_VERSION=24.06a +ARG CUDA_VERSION=12.2 +ARG CUDA_VERSION_MINOR=12.2.2 +ARG PY_VERSION=3.11 +ARG MG_VERSION=2.15.1 -FROM rapidsai/rapidsai:${CUGRAPH_VERSION}-cuda${CUDA_VERSION}-runtime-ubuntu20.04-py${PY_VERSION} as cugraph-dev +# TODO(gitbuda): Probably turn this into Dockerfile.only_cugraph -FROM nvidia/cuda:${CUDA_VERSION_MINOR}-devel-ubuntu20.04 AS dev +FROM rapidsai/base:${CUGRAPH_VERSION}-cuda${CUDA_VERSION}-py${PY_VERSION} as cugraph-dev +FROM nvidia/cuda:${CUDA_VERSION_MINOR}-devel-ubuntu22.04 AS dev USER root @@ -17,15 +18,12 @@ ENV MG_VERSION ${MG_VERSION} ENV PY_VERSION ${PY_VERSION} # Copy RAPIDS libraries -COPY --from=cugraph-dev /opt/conda/envs/rapids/lib/libcugraph.so /opt/conda/envs/rapids/lib/libcugraph.so -COPY --from=cugraph-dev /opt/conda/envs/rapids/include /opt/conda/envs/rapids/include +COPY --from=cugraph-dev /opt/conda/lib/libcugraph.so /opt/conda/lib/libcugraph.so +COPY --from=cugraph-dev /opt/conda/include /opt/conda/include # Prevent from linking the Conda environment ENV LD_LIBRARY_PATH=/usr/local/nvidia/lib:/usr/local/nvidia/lib64:/usr/local/cuda/lib64:/usr/local/lib -# NVIDIA key rotation -RUN rm /etc/apt/sources.list.d/cuda.list - # Essentials for production/dev RUN apt-get update && apt-get install -y \ libcurl4 `memgraph` \ @@ -62,29 +60,30 @@ EXPOSE 7687 # Copy and build MAGE WORKDIR /mage COPY . /mage +# TODO(gitbuda): In the meantime, we moved the whole memgraph repo under cpp -> init submodules. RUN curl https://sh.rustup.rs -sSf | sh -s -- -y && \ export PATH="/root/.cargo/bin:${PATH}" && \ python3 -m pip install -r /mage/python/requirements.txt && \ python3 -m pip install -r /mage/python/tests/requirements.txt && \ - python3 -m pip install dgl -f https://data.dgl.ai/wheels/repo.html && \ python3 /mage/setup build \ --gpu \ - --cpp-build-flags MAGE_CUGRAPH_ROOT=/opt/conda/envs/rapids/ CMAKE_BUILD_TYPE=Release \ + --cpp-build-flags MAGE_CUGRAPH_ROOT=/opt/conda/ CMAKE_BUILD_TYPE=Release \ -p /usr/lib/memgraph/query_modules/ + # python3 -m pip install dgl -f https://data.dgl.ai/wheels/repo.html && \ # TODO(gitbuda): Figure out DGL -#DGL build from source -RUN git clone --recurse-submodules -b 0.9.x https://github.com/dmlc/dgl.git \ - && cd dgl && mkdir build && cd build && cmake -DUSE_CUDA=ON .. \ - && make -j4 && cd ../python && python3 setup.py install +# # TODO(gitbuda): Figure out DGL +# # Build DGL from source. +# RUN git clone --recurse-submodules -b 2.1.x https://github.com/dmlc/dgl.git \ +# && cd dgl && mkdir build && cd build && cmake -DUSE_CUDA=ON .. \ +# && make -j4 && cd ../python && python3 setup.py install USER memgraph ENTRYPOINT ["/usr/lib/memgraph/memgraph"] -FROM nvidia/cuda:${CUDA_VERSION_MINOR}-runtime-ubuntu20.04 AS prod +FROM nvidia/cuda:${CUDA_VERSION_MINOR}-runtime-ubuntu22.04 AS prod USER root - ARG DEBIAN_FRONTEND=noninteractive ARG MG_VERSION ARG PY_VERSION @@ -94,9 +93,9 @@ ENV PY_VERSION ${PY_VERSION} # Copy modules COPY --from=dev /usr/lib/memgraph/query_modules/ /usr/lib/memgraph/query_modules/ # Copy cugraph library -COPY --from=dev /opt/conda/envs/rapids/lib/libcugraph.so /opt/conda/envs/rapids/lib/libcugraph.so +COPY --from=dev /opt/conda/lib/libcugraph.so /opt/conda/lib/libcugraph.so # Copy python build -COPY --from=dev /usr/local/lib/python${PY_VERSION}/ /usr/local/lib/python${PY_VERSION}/ +COPY --from=dev /usr/lib/python${PY_VERSION}/ /usr/lib/python${PY_VERSION}/ # NVIDIA key rotation RUN rm /etc/apt/sources.list.d/cuda.list @@ -112,15 +111,15 @@ RUN apt-get update && apt-get install -y \ libgomp1 `mage-memgraph` \ python3 `mage-memgraph` \ python3-setuptools `mage-memgraph` \ - && curl https://download.memgraph.com/memgraph/v${MG_VERSION}/ubuntu-20.04/memgraph_${MG_VERSION}-1_amd64.deb --output memgraph.deb \ + && curl https://download.memgraph.com/memgraph/v${MG_VERSION}/ubuntu-22.04/memgraph_${MG_VERSION}-1_amd64.deb --output memgraph.deb \ && dpkg -i memgraph.deb \ && rm memgraph.deb \ && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* -RUN export PATH="/usr/local/lib/python${PY_VERSION}:${PATH}" +RUN export PATH="/usr/lib/python${PY_VERSION}:${PATH}" RUN rm -rf /mage \ - && export PATH="/usr/local/lib/python${PY_VERSION}:${PATH}" \ + && export PATH="/usr/lib/python${PY_VERSION}:${PATH}" \ && apt-get -y --purge autoremove curl python3-dev \ && apt-get clean diff --git a/Dockerfile.experiment_full b/Dockerfile.experiment_full new file mode 100644 index 000000000..2f5077d41 --- /dev/null +++ b/Dockerfile.experiment_full @@ -0,0 +1,118 @@ +# It's important to be 3.10 on Ubuntu 22.04 because memgraph binary depends on exactly that version. +ARG PY_VERSION=3.10 +ARG CUDA_VERSION=12.2 +ARG CUDA_VERSION_MINOR=12.2.2 +ARG RAPIDS_VERSION=24.04a +ARG MG_VERSION=2.18.1 +# TODO(gitbuda): Try to build on ARM and adjust settings if needed. + +## COMPILATION +FROM rapidsai/base:${RAPIDS_VERSION}-cuda${CUDA_VERSION}-py${PY_VERSION} as rapids-dev +FROM nvidia/cuda:${CUDA_VERSION_MINOR}-devel-ubuntu22.04 AS cuda-dev +USER root +ARG DEBIAN_FRONTEND=noninteractive +ARG PY_VERSION +ENV PY_VERSION ${PY_VERSION} + +### SYSTEM SETUP +COPY --from=rapids-dev /opt/conda/lib/libcugraph.so /opt/conda/lib/libcugraph.so +COPY --from=rapids-dev /opt/conda/lib/libcugraph-ops++.so /opt/conda/lib/libcugraph-ops++.so +COPY --from=rapids-dev /opt/conda/lib/libraft.so /opt/conda/lib/libraft.so +COPY --from=rapids-dev /opt/conda/include /opt/conda/include +# Prevent from linking the Conda environment +ENV LD_LIBRARY_PATH=/usr/local/nvidia/lib:/usr/local/nvidia/lib64:/usr/local/cuda/lib64:/usr/local/lib +RUN apt-get update && apt-get install -y \ + vim `memgraph` \ + libcurl4 `memgraph` \ + libpython${PY_VERSION} `memgraph` \ + libssl3 `memgraph` \ + libssl-dev `memgraph` \ + openssl `memgraph` \ + build-essential `mage-memgraph` \ + cmake `mage-memgraph` \ + curl `mage-memgraph` \ + g++ `mage-memgraph` \ + python3 `mage-memgraph` \ + python3-pip `mage-memgraph` \ + python3-setuptools `mage-memgraph` \ + python3-dev `mage-memgraph` \ + clang `mage-memgraph` \ + git `mage-memgraph` \ + unixodbc-dev `mage-memgraph` \ + software-properties-common `mage-cugraph` \ + lsb-release `mage-cugraph` \ + wget `mage-cugraph` \ + --no-install-recommends \ + && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* \ + # Install newest CMake (cuGraph requires >= 20.01) + && wget -qO - https://apt.kitware.com/keys/kitware-archive-latest.asc | apt-key add - && \ + apt-add-repository "deb https://apt.kitware.com/ubuntu/ $(lsb_release -cs) main" && \ + apt-get install -y \ + cmake `mage-memgraph` \ + --no-install-recommends +ENV PATH=/usr/local/nvidia/bin:/usr/local/cuda/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/bin/cmake:/usr/lib/cmake +# TODO(gitbuda): How to cleanup rust installation cache? +RUN curl https://sh.rustup.rs -sSf | sh -s -- -y +ENV PATH="${PATH}:/root/.cargo/bin" +# # NOTE: miniconda will be installed under /root/miniconda3 +# RUN cd /root && \ +# curl -o miniconda.sh https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh && \ +# bash miniconda.sh -b +WORKDIR /mage +COPY . /mage +RUN python3 -m pip install -r /mage/python/requirements.txt && \ + python3 -m pip install -r /mage/python/tests/requirements.txt +# TODO(gitbuda): Experiment with pipx, the below works on debian-12 (on newer systems) +# RUN python3 -m pip install --break-system-packages -r /mage/python/requirements.txt && \ +# python3 -m pip install --break-system-packages -r /mage/python/tests/requirements.txt + +### BUILD +RUN python3 /mage/setup build \ + --gpu \ + --cpp-build-flags \ + CMAKE_BUILD_TYPE=RelWithDebInfo \ + MAGE_CUGRAPH_ROOT=/opt/conda/ \ + -p /usr/lib/memgraph/query_modules/ + +## RUNTIME +# TODO(gitbuda): Add the ability to inject local RelWithDebInfo package of Memgraph +FROM nvidia/cuda:${CUDA_VERSION_MINOR}-runtime-ubuntu22.04 AS cuda-prod +USER root +ARG DEBIAN_FRONTEND=noninteractive +ARG MG_VERSION +ENV MG_VERSION ${MG_VERSION} +ARG PY_VERSION +ENV PY_VERSION ${PY_VERSION} + +#### SYSTEM SETUP +COPY --from=cuda-dev /usr/lib/memgraph/query_modules/ /usr/lib/memgraph/query_modules/ +COPY --from=cuda-dev /opt/conda/lib/libcugraph.so /opt/conda/lib/libcugraph.so +COPY --from=cuda-dev /opt/conda/lib/libcugraph-ops++.so /opt/conda/lib/libcugraph-ops++.so +COPY --from=cuda-dev /opt/conda/lib/libraft.so /opt/conda/lib/libraft.so +COPY --from=cuda-dev /usr/lib/python${PY_VERSION}/ /usr/lib/python${PY_VERSION}/ +# TODO(gitbuda): Consider how to copy all the requirements from python -> because the above is not full, e.g. networkx is not working. +RUN rm -rf /etc/apt/sources.list.d/cuda* +RUN apt-get update && apt-get install -y \ + libcurl4 `memgraph` \ + libpython${PY_VERSION} `memgraph` \ + libssl3 `memgraph` \ + libssl-dev `memgraph` \ + openssl `memgraph` \ + curl `mage-memgraph` \ + libgomp1 `mage-memgraph` \ + python3 `mage-memgraph` \ + python3-setuptools `mage-memgraph` \ + && curl https://download.memgraph.com/memgraph/v${MG_VERSION}/ubuntu-22.04/memgraph_${MG_VERSION}-1_amd64.deb --output memgraph.deb \ + && dpkg -i memgraph.deb \ + && rm memgraph.deb \ + && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* +ENV PATH="/usr/lib/python${PY_VERSION}:${PATH}" +# TODO(gitbuda): extra packages python3-setuptools, python3-pip, cmake -> make sure the dev step wasn't wrongly cached. +# TODO(gitbuda): The rm part should be under the apt install RUN part (/mage is not present here) +RUN rm -rf /mage \ + && export PATH="/usr/lib/python${PY_VERSION}:${PATH}" \ + && apt-get -y --purge autoremove curl python3-dev \ + && apt-get clean + +USER memgraph +ENTRYPOINT ["/usr/lib/memgraph/memgraph"] diff --git a/Dockerfile.experiment_partial b/Dockerfile.experiment_partial new file mode 100644 index 000000000..fdac42ca4 --- /dev/null +++ b/Dockerfile.experiment_partial @@ -0,0 +1,13 @@ +ARG PY_VERSION=3.11 +ARG RAPIDS_VERSION=24.06a +ARG CUDA_VERSION=12.2 +ARG CUDA_VERSION_MINOR=12.2.2 +ARG MG_VERSION=2.15.1 + +FROM rapidsai/base:${RAPIDS_VERSION}-cuda${CUDA_VERSION}-py${PY_VERSION} as rapids-dev +RUN ls -alh /opt/ && ls -alh /opt/conda/ && ls -alh /opt/conda/lib/ | grep cugraph +RUN nvcc --version +# +# FROM nvidia/cuda:${CUDA_VERSION_MINOR}-devel-ubuntu22.04 AS cuda-dev +# RUN ls -alh /opt/ && ls -alh /opt/nvidia/ +# RUN nvcc --version diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index 4810cc39e..2e8607ac8 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -1,25 +1,40 @@ # Memgraph Mage C++ Query Modules CMake configuration. cmake_minimum_required(VERSION 3.14) set(MEMGRAPH_MAGE_PROJECT_NAME "memgraph-mage") + +if (MAGE_CUGRAPH_ENABLE) + # TODO(gitbuda): Duplicated under cugraph.cmake -> FIX. + set(MAGE_CUGRAPH_TAG "v24.04.00" CACHE STRING "cuGraph GIT tag to checkout" ) + # RAPIDS.cmake is here because rapids_cuda_init_architectures is required to + # properly set both CMAKE_CUDA_ARCHITECTURES and CUDA_ARCHITECTURES target + # property. + file(DOWNLOAD "https://raw.githubusercontent.com/rapidsai/rapids-cmake/${MAGE_CUGRAPH_TAG}a/RAPIDS.cmake" + ${CMAKE_BINARY_DIR}/RAPIDS.cmake) + include(${CMAKE_BINARY_DIR}/RAPIDS.cmake) + # NOTE: RAPIDS init is required to be called before project(). + include(rapids-cuda) + rapids_cuda_init_architectures("${MEMGRAPH_MAGE_PROJECT_NAME}") + # rapids_cuda_patch_toolkit() +endif() project("${MEMGRAPH_MAGE_PROJECT_NAME}" LANGUAGES C CXX) -include(FetchContent) +if (MAGE_CUGRAPH_ENABLE) + find_package(CUDAToolkit REQUIRED) + enable_language(CUDA) + add_definitions(-DLIBCUDACXX_ENABLE_EXPERIMENTAL_MEMORY_RESOURCE) +endif() +include(FetchContent) # setup CMake module path, defines path for include() and find_package() # https://cmake.org/cmake/help/latest/variable/CMAKE_MODULE_PATH.html set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} ${PROJECT_SOURCE_DIR}/cugraph_cmake) - # Export the compile commands so that we can use clang-tidy. Additional benefit # is easier debugging of compilation and linker flags. set(CMAKE_EXPORT_COMPILE_COMMANDS ON) - -# Required C++ standard. set(CMAKE_C_STANDARD 11) set(CMAKE_C_STANDARD_REQUIRED ON) set(CMAKE_CXX_STANDARD 20) set(CMAKE_CXX_STANDARD_REQUIRED ON) - - set(CMAKE_POSITION_INDEPENDENT_CODE ON) # Compile flags. @@ -116,9 +131,10 @@ add_subdirectory(mg_utility) FetchContent_Declare(fmt GIT_REPOSITORY https://github.com/fmtlib/fmt.git - GIT_TAG 9.1.0 + GIT_TAG 10.2.1 ) FetchContent_MakeAvailable(fmt) +add_compile_definitions(SPDLOG_FMT_EXTERNAL) function(add_query_module target_name version src) add_library(${target_name} SHARED ${src}) diff --git a/cpp/cugraph_cmake/cugraph.cmake b/cpp/cugraph_cmake/cugraph.cmake index ebbbab8d3..c1db938dc 100644 --- a/cpp/cugraph_cmake/cugraph.cmake +++ b/cpp/cugraph_cmake/cugraph.cmake @@ -27,37 +27,39 @@ # Order of the languages matters because cmake pics different compilers. # # Compiling cugraph takes ages and it's complex. -# TODO(gitbuda): Allow linking of an already compiled version of cugraph. +# -> Defining MAGE_CUGRAPH_ROOT will skip compilation. # # CUDA_ARCHITECTURES -> # https://arnon.dk/matching-sm-architectures-arch-and-gencode-for-various-nvidia-cards/ # Rapids CMake add NATIVE + ALL options as CUDA_ARCHITECTURES which simplifies # build configuration. +# +# CUDA + CMake -> https://developer.download.nvidia.com/video/gputechconf/gtc/2019/presentation/s9444-build-systems-exploring-modern-cmake-cuda-v2.pdf option(MAGE_CUGRAPH_ENABLE "Enable cuGraph build" OFF) message(STATUS "MAGE cuGraph build enabled: ${MAGE_CUGRAPH_ENABLE}") if (MAGE_CUGRAPH_ENABLE) # Version of cuGraph for local build - set(MAGE_CUGRAPH_TAG "v22.02.00" CACHE STRING "cuGraph GIT tag to checkout" ) + # set(MAGE_CUGRAPH_TAG "v24.04.00" CACHE STRING "cuGraph GIT tag to checkout" ) set(MAGE_CUGRAPH_REPO "https://github.com/rapidsai/cugraph.git" CACHE STRING "cuGraph GIT repo URL") # Custom MAGE_CUGRAPH_BUILD_TYPE. set(MAGE_CUGRAPH_BUILD_TYPE "Release" CACHE STRING "Passed to cuGraph as CMAKE_BUILD_TYPE") # NATIVE | ALL -> possible because cugraph calls rapids_cuda_init_architectures set(MAGE_CUDA_ARCHITECTURES "NATIVE" CACHE STRING "Passed to cuGraph as CMAKE_CUDA_ARCHITECTURES") - # RAPIDS.cmake is here because rapids_cuda_init_architectures is required to - # properly set both CMAKE_CUDA_ARCHITECTURES and CUDA_ARCHITECTURES target - # property. - file(DOWNLOAD "https://raw.githubusercontent.com/rapidsai/rapids-cmake/${MAGE_CUGRAPH_TAG}a/RAPIDS.cmake" - ${CMAKE_BINARY_DIR}/RAPIDS.cmake) - include(${CMAKE_BINARY_DIR}/RAPIDS.cmake) - - include(rapids-cuda) - rapids_cuda_init_architectures("${MEMGRAPH_MAGE_PROJECT_NAME}") - enable_language(CUDA) + # # RAPIDS.cmake is here because rapids_cuda_init_architectures is required to + # # properly set both CMAKE_CUDA_ARCHITECTURES and CUDA_ARCHITECTURES target + # # property. + # file(DOWNLOAD "https://raw.githubusercontent.com/rapidsai/rapids-cmake/${MAGE_CUGRAPH_TAG}a/RAPIDS.cmake" + # ${CMAKE_BINARY_DIR}/RAPIDS.cmake) + # include(${CMAKE_BINARY_DIR}/RAPIDS.cmake) + # + # include(rapids-cuda) + # rapids_cuda_init_architectures("${MEMGRAPH_MAGE_PROJECT_NAME}") + # enable_language(CUDA) - set(CMAKE_CUDA_STANDARD 17) + set(CMAKE_CUDA_STANDARD 20) set(CMAKE_CUDA_STANDARD_REQUIRED ON) message(STATUS "MAGE cuGraph root: ${MAGE_CUGRAPH_ROOT}") @@ -104,10 +106,14 @@ macro(target_mage_cugraph target_name) message(STATUS "Building with CUDA debugging flags") list(APPEND MAGE_CUDA_FLAGS -g -G -Xcompiler=-rdynamic) endif() + # NOTE: Ignore some of the warnings produced by nvcc: + # * 186 warning related to pointless comparison of unsigned integer with zero + # * 20200 remark about deprecation of gem.hpp under Nvidia RAFT library + list(APPEND MAGE_CUDA_FLAGS --diag-suppress 186,20200) target_compile_options("${target_name}" PRIVATE "$<$:${MAGE_CXX_FLAGS}>" "$<$:${MAGE_CUDA_FLAGS}>" ) - target_link_libraries("${target_name}" PRIVATE mage_cugraph) + target_link_libraries("${target_name}" PRIVATE mage_cugraph CUDA::toolkit) endif() endmacro() diff --git a/cpp/cugraph_module/CMakeLists.txt b/cpp/cugraph_module/CMakeLists.txt index 9166b69a1..62ebd80fd 100644 --- a/cpp/cugraph_module/CMakeLists.txt +++ b/cpp/cugraph_module/CMakeLists.txt @@ -46,4 +46,4 @@ target_include_directories(cugraph.hits PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}) add_query_module(cugraph.generator 1 algorithms/graph_generator.cu) target_mage_cugraph(cugraph.generator) target_link_libraries(cugraph.generator PRIVATE mg_utility) -target_include_directories(cugraph.generator PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}) \ No newline at end of file +target_include_directories(cugraph.generator PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}) diff --git a/cpp/cugraph_module/README.md b/cpp/cugraph_module/README.md index 190336e92..267c40c76 100644 --- a/cpp/cugraph_module/README.md +++ b/cpp/cugraph_module/README.md @@ -2,6 +2,15 @@ This is fairly simple installation setup (created for Ubuntu 20.04) to follow: +## TODOs + +* Test all results on small graphs (in an automated way!). +* Try to resolve all compile warnings. +* Give some pointers on how to manage nvcc (TODOs in the code). +* cugraph v24.04 docs is the code! + * https://github.com/rapidsai/cugraph/blob/v24.04.00/cpp/include/cugraph/graph_functions.hpp + * https://github.com/rapidsai/cugraph/blob/v24.04.00/cpp/include/cugraph/algorithms.hpp +* Try to benchmark again :eyes: #### 1. [IF NEEDED] Install the newer version of CMake (Version 20.0+) since it is requirement for cuGraph CMakeLists.txt. Example shows CMake 3.22. diff --git a/cpp/cugraph_module/algorithms/balanced_cut_clustering.cu b/cpp/cugraph_module/algorithms/balanced_cut_clustering.cu index f9167b166..a9ff43b0e 100644 --- a/cpp/cugraph_module/algorithms/balanced_cut_clustering.cu +++ b/cpp/cugraph_module/algorithms/balanced_cut_clustering.cu @@ -67,7 +67,6 @@ void BalancedCutClusteringProc(mgp_list *args, mgp_graph *graph, mgp_result *res auto mg_graph = mg_utility::GetWeightedGraphView(graph, result, memory, mg_graph::GraphType::kUndirectedGraph, weight_property, kDefaultWeight); if (mg_graph->Empty()) return; - auto n_vertices = mg_graph.get()->Nodes().size(); // Define handle and operation stream diff --git a/cpp/cugraph_module/algorithms/betweenness_centrality.cu b/cpp/cugraph_module/algorithms/betweenness_centrality.cu index da66ca1e3..a0e11be86 100644 --- a/cpp/cugraph_module/algorithms/betweenness_centrality.cu +++ b/cpp/cugraph_module/algorithms/betweenness_centrality.cu @@ -15,11 +15,9 @@ #include "mg_cugraph_utility.hpp" namespace { -// TODO: Check Betweenness instances. Update in new cuGraph API. -using vertex_t = int32_t; -using edge_t = int32_t; +using vertex_t = int64_t; +using edge_t = int64_t; using weight_t = double; -using result_t = double; constexpr char const *kProcedureBetweenness = "get"; @@ -58,25 +56,18 @@ void BetweennessProc(mgp_list *args, mgp_graph *graph, mgp_result *result, mgp_m raft::handle_t handle{}; auto stream = handle.get_stream(); - - auto mg_graph = mg_utility::GetWeightedGraphView( - graph, result, memory, directed ? mg_graph::GraphType::kDirectedGraph : mg_graph::GraphType::kUndirectedGraph, - weight_property, kDefaultWeight); + auto mg_graph_type = directed ? mg_graph::GraphType::kDirectedGraph : mg_graph::GraphType::kUndirectedGraph; + auto mg_graph = + mg_utility::GetWeightedGraphView(graph, result, memory, mg_graph_type, weight_property, kDefaultWeight); if (mg_graph->Empty()) return; - - auto n_vertices = mg_graph.get()->Nodes().size(); - - // IMPORTANT: Betweenness centrality cuGraph algorithm works only on legacy code - auto cu_graph_ptr = - mg_cugraph::CreateCugraphLegacyFromMemgraph(*mg_graph.get(), handle); - auto cu_graph_view = cu_graph_ptr->view(); - cu_graph_view.prop.directed = directed; - - rmm::device_uvector betweenness_result(n_vertices, stream); - // TODO: Add weights to the betweenness centrality algorithm - cugraph::betweenness_centrality(handle, cu_graph_view, betweenness_result.data(), - normalized, false, static_cast(nullptr)); - + auto cu_graph = mg_cugraph::CreateCugraphFromMemgraph( + *mg_graph.get(), mg_graph_type, handle); + auto cu_graph_view = cu_graph.view(); + + // Calling https://github.com/rapidsai/cugraph/blob/branch-24.04/cpp/include/cugraph/algorithms.hpp#L341 + // NOTE: The current implementation does not support a weighted graph. + auto betweenness_result = cugraph::betweenness_centrality( + handle, cu_graph_view, std::nullopt, std::nullopt); for (vertex_t node_id = 0; node_id < betweenness_result.size(); ++node_id) { auto rank = betweenness_result.element(node_id, stream); InsertBetweennessRecord(graph, result, memory, mg_graph->GetMemgraphNodeId(node_id), rank); diff --git a/cpp/cugraph_module/algorithms/hits.cu b/cpp/cugraph_module/algorithms/hits.cu index 0dfe1b7e5..5b74e9a9a 100644 --- a/cpp/cugraph_module/algorithms/hits.cu +++ b/cpp/cugraph_module/algorithms/hits.cu @@ -68,8 +68,9 @@ void HITSProc(mgp_list *args, mgp_graph *graph, mgp_result *result, mgp_memory * auto cu_graph = mg_cugraph::CreateCugraphFromMemgraph(*mg_graph.get(), graph_type, handle); auto cu_graph_view = cu_graph.view(); - rmm::device_uvector hubs_result(cu_graph_view.get_number_of_local_vertices(), stream); - rmm::device_uvector authorities_result(cu_graph_view.get_number_of_local_vertices(), stream); + // TODO(gitbuda): Inspect local vs non-local vertices change in hits + rmm::device_uvector hubs_result(cu_graph_view.number_of_vertices(), stream); + rmm::device_uvector authorities_result(cu_graph_view.number_of_vertices(), stream); cugraph::hits(handle, cu_graph_view, hubs_result.data(), authorities_result.data(), tolerance, max_iterations, false, normalize, false); diff --git a/cpp/cugraph_module/algorithms/katz_centrality.cu b/cpp/cugraph_module/algorithms/katz_centrality.cu index 6ca143069..196e84cda 100644 --- a/cpp/cugraph_module/algorithms/katz_centrality.cu +++ b/cpp/cugraph_module/algorithms/katz_centrality.cu @@ -69,7 +69,7 @@ void KatzCentralityProc(mgp_list *args, mgp_graph *graph, mgp_result *result, mg auto cu_graph = mg_cugraph::CreateCugraphFromMemgraph(*mg_graph.get(), graph_type, handle); auto cu_graph_view = cu_graph.view(); - auto n_vertices = cu_graph_view.get_number_of_vertices(); + auto n_vertices = cu_graph_view.number_of_vertices(); auto degrees = cu_graph_view.compute_in_degrees(handle); std::vector cu_degrees(degrees.size()); @@ -81,8 +81,12 @@ void KatzCentralityProc(mgp_list *args, mgp_graph *graph, mgp_result *result, mg result_t beta{beta_arg}; result_t epsilon{epsilon_arg}; rmm::device_uvector katz_results(n_vertices, stream); - cugraph::katz_centrality(handle, cu_graph_view, static_cast(nullptr), katz_results.data(), alpha, beta, - epsilon, max_iterations, false, normalized, false); + // TODO(gitbuda): pass the right weight to the katz centrality; size_t and other casts, -> second problem with nvcc + // is that, true, false template arguments missmatch are not obvious + cugraph::katz_centrality( + handle, cu_graph_view, std::nullopt, static_cast(nullptr), + static_cast(katz_results.data()), alpha, beta, epsilon, (size_t)max_iterations, false, normalized, + false); for (vertex_t node_id = 0; node_id < katz_results.size(); ++node_id) { auto rank = katz_results.element(node_id, stream); diff --git a/cpp/cugraph_module/algorithms/leiden.cu b/cpp/cugraph_module/algorithms/leiden.cu index dcca27875..ecb9f4009 100644 --- a/cpp/cugraph_module/algorithms/leiden.cu +++ b/cpp/cugraph_module/algorithms/leiden.cu @@ -15,9 +15,8 @@ #include "mg_cugraph_utility.hpp" namespace { -// TODO: Check Leiden instances. Update in new cuGraph API. -using vertex_t = int32_t; -using edge_t = int32_t; +using vertex_t = int64_t; +using edge_t = int64_t; using weight_t = double; constexpr char const *kProcedureLeiden = "get"; @@ -50,24 +49,23 @@ void LeidenProc(mgp_list *args, mgp_graph *graph, mgp_result *result, mgp_memory auto max_iterations = mgp::value_get_int(mgp::list_at(args, 0)); auto resulution = mgp::value_get_double(mgp::list_at(args, 1)); - auto mg_graph = mg_utility::GetGraphView(graph, result, memory, mg_graph::GraphType::kUndirectedGraph); - if (mg_graph->Empty()) return; - - auto n_vertices = mg_graph.get()->Nodes().size(); - - // Define handle and operation stream raft::handle_t handle{}; auto stream = handle.get_stream(); - - // IMPORTANT: Leiden cuGraph algorithm works only on legacy code - auto cu_graph_ptr = - mg_cugraph::CreateCugraphLegacyFromMemgraph(*mg_graph.get(), handle); - auto cu_graph_view = cu_graph_ptr->view(); - cu_graph_view.prop.directed = false; + auto mg_graph_type = mg_graph::GraphType::kUndirectedGraph; + auto mg_graph = mg_utility::GetGraphView(graph, result, memory, mg_graph_type); + if (mg_graph->Empty()) return; + // TODO(gitbuda): Inject the valid seed. + raft::random::RngState rng_state(0); + auto cu_graph = mg_cugraph::CreateCugraphFromMemgraph( + *mg_graph.get(), mg_graph_type, handle); + auto cu_graph_view = cu_graph.view(); + auto n_vertices = cu_graph_view.number_of_vertices(); rmm::device_uvector clustering_result(n_vertices, stream); - cugraph::leiden(handle, cu_graph_view, clustering_result.data(), max_iterations, resulution); - + // TODO(gitbuda): Leiden weights and other arguments. Add theta argument. + cugraph::leiden(handle, rng_state, cu_graph_view, std::nullopt, + clustering_result.data(), (size_t)max_iterations, resulution, + 1.0); for (vertex_t node_id = 0; node_id < clustering_result.size(); ++node_id) { auto partition = clustering_result.element(node_id, stream); InsertLeidenRecord(graph, result, memory, mg_graph->GetMemgraphNodeId(node_id), partition); diff --git a/cpp/cugraph_module/algorithms/louvain.cu b/cpp/cugraph_module/algorithms/louvain.cu index d55575263..8c4cf1965 100644 --- a/cpp/cugraph_module/algorithms/louvain.cu +++ b/cpp/cugraph_module/algorithms/louvain.cu @@ -63,10 +63,14 @@ void LouvainProc(mgp_list *args, mgp_graph *graph, mgp_result *result, mgp_memor auto cu_graph = mg_cugraph::CreateCugraphFromMemgraph(*mg_graph.get(), graph_type, handle); auto cu_graph_view = cu_graph.view(); - auto n_vertices = cu_graph_view.get_number_of_vertices(); + auto n_vertices = cu_graph_view.number_of_vertices(); rmm::device_uvector clustering_result(n_vertices, stream); - cugraph::louvain(handle, cu_graph_view, clustering_result.data(), max_iterations, resolution); + // TODO(gitbuda): louvain arguments have to passed correctly + // TODO(gitbuda): NOTE for the documentation -> nvcc is not great at telling what are the options, the solution here + // was to pass template arguments to pick the right function overload. + cugraph::louvain(handle, std::nullopt, cu_graph_view, std::nullopt, + clustering_result.data(), max_iterations, resolution, 0.0); for (vertex_t node_id = 0; node_id < clustering_result.size(); ++node_id) { auto partition = clustering_result.element(node_id, stream); diff --git a/cpp/cugraph_module/algorithms/pagerank.cu b/cpp/cugraph_module/algorithms/pagerank.cu index 42686c34e..37f680f08 100644 --- a/cpp/cugraph_module/algorithms/pagerank.cu +++ b/cpp/cugraph_module/algorithms/pagerank.cu @@ -67,16 +67,16 @@ void PagerankProc(mgp_list *args, mgp_graph *graph, mgp_result *result, mgp_memo auto cu_graph = mg_cugraph::CreateCugraphFromMemgraph(*mg_graph.get(), mg_graph::GraphType::kDirectedGraph, handle); auto cu_graph_view = cu_graph.view(); - auto n_vertices = cu_graph_view.get_number_of_vertices(); + auto n_vertices = cu_graph_view.number_of_vertices(); rmm::device_uvector pagerank_results(n_vertices, stream); // IMPORTANT: store_transposed has to be true because cugraph::pagerank // only accepts true. It's hard to detect/debug problem because nvcc error // messages contain only the top call details + graph_view has many // template paremeters. - cugraph::pagerank(handle, cu_graph_view, std::nullopt, std::nullopt, - std::nullopt, std::nullopt, pagerank_results.data(), - damping_factor, stop_epsilon, max_iterations); + cugraph::pagerank( + handle, cu_graph_view, std::nullopt, std::nullopt, std::nullopt, std::nullopt, std::nullopt, + pagerank_results.data(), damping_factor, stop_epsilon, max_iterations); for (vertex_t node_id = 0; node_id < pagerank_results.size(); ++node_id) { auto rank = pagerank_results.element(node_id, stream); diff --git a/cpp/cugraph_module/algorithms/personalized_pagerank.cu b/cpp/cugraph_module/algorithms/personalized_pagerank.cu index 021690da8..462fae2cc 100644 --- a/cpp/cugraph_module/algorithms/personalized_pagerank.cu +++ b/cpp/cugraph_module/algorithms/personalized_pagerank.cu @@ -71,7 +71,7 @@ void PersonalizedPagerankProc(mgp_list *args, mgp_graph *graph, mgp_result *resu auto cu_graph = mg_cugraph::CreateCugraphFromMemgraph(*mg_graph.get(), mg_graph::GraphType::kDirectedGraph, handle); auto cu_graph_view = cu_graph.view(); - auto n_vertices = cu_graph_view.get_number_of_vertices(); + auto n_vertices = cu_graph_view.number_of_vertices(); rmm::device_uvector pagerank_results(n_vertices, stream); // IMPORTANT: store_transposed has to be true because cugraph::pagerank @@ -97,9 +97,11 @@ void PersonalizedPagerankProc(mgp_list *args, mgp_graph *graph, mgp_result *resu raft::update_device(personalization_values.data(), v_personalization_values.data(), v_personalization_values.size(), stream); + // TODO(gitbuda): Here weight has to be passed correctly. cugraph::pagerank( - handle, cu_graph_view, std::nullopt, personalization_vertices.data(), personalization_values.data(), - v_personalization_vertices.size(), pagerank_results.data(), damping_factor, stop_epsilon, max_iterations); + handle, cu_graph_view, std::nullopt, std::nullopt, personalization_vertices.data(), + personalization_values.data(), v_personalization_vertices.size(), pagerank_results.data(), damping_factor, + stop_epsilon, max_iterations); for (vertex_t node_id = 0; node_id < pagerank_results.size(); ++node_id) { auto rank = pagerank_results.element(node_id, stream); diff --git a/cpp/cugraph_module/mg_cugraph_utility.hpp b/cpp/cugraph_module/mg_cugraph_utility.hpp index a022e705d..022f493a0 100644 --- a/cpp/cugraph_module/mg_cugraph_utility.hpp +++ b/cpp/cugraph_module/mg_cugraph_utility.hpp @@ -1,10 +1,10 @@ #include -#include // legacy coo_to_csr #include #include +#include // legacy coo_to_csr -#include -#include +#include +#include #include #include @@ -52,17 +52,26 @@ auto CreateCugraphFromMemgraph(const mg_graph::GraphView<> &mg_graph, const mg_g mg_weight.reserve(mg_edges.size()); std::vector mg_vertices; mg_vertices.reserve(mg_nodes.size()); + // TODO(gitbuda): Update types + std::vector mg_edge_ids; + mg_edge_ids.reserve(mg_edges.size()); + std::vector mg_edge_types; + mg_edge_types.reserve(mg_edges.size()); std::transform(mg_edges.begin(), mg_edges.end(), std::back_inserter(mg_src), [](const auto &edge) -> TVertexT { return edge.from; }); std::transform(mg_edges.begin(), mg_edges.end(), std::back_inserter(mg_dst), [](const auto &edge) -> TVertexT { return edge.to; }); - std::transform( mg_edges.begin(), mg_edges.end(), std::back_inserter(mg_weight), [&mg_graph](const auto &edge) -> TWeightT { return mg_graph.IsWeighted() ? mg_graph.GetWeight(edge.id) : 1.0; }); std::transform(mg_nodes.begin(), mg_nodes.end(), std::back_inserter(mg_vertices), [](const auto &node) -> TVertexT { return node.id; }); + std::transform(mg_edges.begin(), mg_edges.end(), std::back_inserter(mg_edge_ids), + [](const auto &edge) -> TVertexT { return edge.id; }); + // TODO(gitbuda): Add valid edge type (bigger changes because the mage util data structure has to change). + std::transform(mg_edges.begin(), mg_edges.end(), std::back_inserter(mg_edge_types), + [](const auto &edge) -> TVertexT { return 0; }); // Synchronize the data structures to the GPU auto stream = handle.get_stream(); @@ -75,19 +84,29 @@ auto CreateCugraphFromMemgraph(const mg_graph::GraphView<> &mg_graph, const mg_g rmm::device_uvector cu_vertices(mg_vertices.size(), stream); raft::update_device(cu_vertices.data(), mg_vertices.data(), mg_vertices.size(), stream); + // TODO(gitbuda): Update types and inject valid ids / edge types. + rmm::device_uvector cu_edge_ids(mg_edge_ids.size(), stream); + raft::update_device(cu_edge_ids.data(), mg_edge_ids.data(), mg_edge_ids.size(), stream); + rmm::device_uvector cu_edge_types(mg_edge_types.size(), stream); + raft::update_device(cu_edge_types.data(), mg_edge_types.data(), mg_edge_types.size(), stream); + // TODO: Deal_with/pass edge weights to CuGraph graph. // TODO: Allow for multigraphs - cugraph::graph_t cu_graph(handle); + // TODO(gitbuda): graph_t doesn't have weight anymore -> use std::optional> edge_weight_view + // for a given algorithm. + cugraph::graph_t cu_graph(handle); // NOTE: Renumbering is not required because graph coming from Memgraph is already correctly numbered. - std::tie(cu_graph, std::ignore) = - cugraph::create_graph_from_edgelist( + std::tie(cu_graph, std::ignore, std::ignore, std::ignore, std::ignore) = + cugraph::create_graph_from_edgelist( handle, std::move(cu_vertices), std::move(cu_src), std::move(cu_dst), std::move(cu_weight), - cugraph::graph_properties_t{false, false}, false, false); + std::move(cu_edge_ids), std::move(cu_edge_types), cugraph::graph_properties_t{false, false}, false, false); stream.synchronize_no_throw(); return std::move(cu_graph); } +// TODO(gitbuda): If CreateCugraphLegacy... is not used -> delete. /// ///@brief Create a cuGraph legacy graph object from a given Memgraph graph. This method generates the graph in the /// Compressed Sparse Row format that defines offsets and indices. Description is available at [Compressed Sparse @@ -158,9 +177,9 @@ auto CreateCugraphLegacyFromMemgraph(const mg_graph::GraphView<> &mg_graph, raft ///@param b Probability of the second partition ///@param c Probability of the third partition ///@param seed Random seed applied -///@param clip_and_flip Clip and flip +///@param clip_and_flip Clip and flip ///@param handle Handle for GPU communication -///@return Edges in edge list format +///@return Edges in edge list format /// template auto GenerateCugraphRMAT(std::size_t scale, std::size_t num_edges, double a, double b, double c, std::uint64_t seed, @@ -182,4 +201,4 @@ auto GenerateCugraphRMAT(std::size_t scale, std::size_t num_edges, double a, dou } return mg_edges; } -} // namespace mg_cugraph \ No newline at end of file +} // namespace mg_cugraph diff --git a/cpp/memgraph b/cpp/memgraph index a2a863b00..93962b1a7 160000 --- a/cpp/memgraph +++ b/cpp/memgraph @@ -1 +1 @@ -Subproject commit a2a863b009afebe7a0363e16f5bcccccbf326bf6 +Subproject commit 93962b1a79e34d8ebac8b7420da984610a5144b7 diff --git a/cpp/pagerank_module/algorithm/pagerank.cpp b/cpp/pagerank_module/algorithm/pagerank.cpp index b3b687e97..bf049f564 100644 --- a/cpp/pagerank_module/algorithm/pagerank.cpp +++ b/cpp/pagerank_module/algorithm/pagerank.cpp @@ -155,7 +155,7 @@ PageRankGraph::PageRankGraph(const std::uint64_t number_of_nodes, const std::uin : node_count_(number_of_nodes), edge_count_(number_of_edges), out_degree_(number_of_nodes) { AdjacencyList in_neighbours(number_of_nodes); - for (const auto [from, to] : edges) { + for (const auto &[from, to] : edges) { // Because PageRank needs a set of nodes that point to a given node. in_neighbours.AddAdjacentPair(from, to); out_degree_[from] += 1; diff --git a/e2e/_graphs/README.md b/e2e/_graphs/README.md new file mode 100644 index 000000000..9a8cf2ca3 --- /dev/null +++ b/e2e/_graphs/README.md @@ -0,0 +1,8 @@ +# Memgraph Mage End2End Graph Test Cases + +Just symlink any `.cyp` file from this directory under +`e2e/test/case/input.cyp`. + +`ipr_example*.cyp` are constracted based on the graph examples under [The +Google Pagerank Algorithm and How It +Works](https://www.khoury.northeastern.edu/home/vip/teach/IRcourse/4_webgraph/notes/Pagerank%20Explained%20Correctly%20with%20Examples.html) diff --git a/e2e/_graphs/ipr_example1.cyp b/e2e/_graphs/ipr_example1.cyp new file mode 100644 index 000000000..833f23d8d --- /dev/null +++ b/e2e/_graphs/ipr_example1.cyp @@ -0,0 +1,4 @@ +MERGE (a {id:"A"}) MERGE (b {id: "B"}) CREATE (a)-[:Links]->(b) +MERGE (a {id:"A"}) MERGE (c {id: "C"}) CREATE (a)-[:Links]->(c) +MERGE (c {id:"C"}) MERGE (a {id: "A"}) CREATE (c)-[:Links]->(a) +MERGE (b {id:"B"}) MERGE (c {id: "C"}) CREATE (b)-[:Links]->(c) diff --git a/e2e/pagerank_test/test_ipr1_cpu/input.cyp b/e2e/pagerank_test/test_ipr1_cpu/input.cyp new file mode 120000 index 000000000..61f53ff26 --- /dev/null +++ b/e2e/pagerank_test/test_ipr1_cpu/input.cyp @@ -0,0 +1 @@ +../../_graphs/ipr_example1.cyp \ No newline at end of file diff --git a/e2e/pagerank_test/test_ipr1_cpu/test.yml b/e2e/pagerank_test/test_ipr1_cpu/test.yml new file mode 100644 index 000000000..9e1ec6f15 --- /dev/null +++ b/e2e/pagerank_test/test_ipr1_cpu/test.yml @@ -0,0 +1,12 @@ +query: > + CALL pagerank.get() YIELD node, rank + RETURN node.id AS id, rank + ORDER BY id ASC; + +output: + - id: "A" + rank: 0.3878 + - id: "B" + rank: 0.2148 + - id: "C" + rank: 0.3973 diff --git a/e2e/pagerank_test/test_ipr1_cugraph/input.cyp b/e2e/pagerank_test/test_ipr1_cugraph/input.cyp new file mode 120000 index 000000000..61f53ff26 --- /dev/null +++ b/e2e/pagerank_test/test_ipr1_cugraph/input.cyp @@ -0,0 +1 @@ +../../_graphs/ipr_example1.cyp \ No newline at end of file diff --git a/e2e/pagerank_test/test_ipr1_cugraph/test.yml b/e2e/pagerank_test/test_ipr1_cugraph/test.yml new file mode 100644 index 000000000..2c7eda467 --- /dev/null +++ b/e2e/pagerank_test/test_ipr1_cugraph/test.yml @@ -0,0 +1,12 @@ +query: > + CALL cugraph.pagerank.get() YIELD node, pagerank + RETURN node.id AS id, pagerank AS rank + ORDER BY id ASC; + +output: + - id: "A" + rank: 0.3878 + - id: "B" + rank: 0.2148 + - id: "C" + rank: 0.3973 diff --git a/experiments_to_delete.sh b/experiments_to_delete.sh new file mode 100755 index 000000000..eec368f69 --- /dev/null +++ b/experiments_to_delete.sh @@ -0,0 +1,29 @@ +#!/bin/bash +set -Eeuo pipefail +SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" + +MAKE_VERBOSE="VERBOSE=1" +# MAKE_VERBOSE="" +# MAKE_PARALLEL="-j16" +MAKE_PARALLEL="" + +install_latest_cmake () { + # Latest CMake on Ubuntu 22.04 + wget -qO - https://apt.kitware.com/keys/kitware-archive-latest.asc | sudo apt-key add - + sudo apt-add-repository "deb https://apt.kitware.com/ubuntu/ $(lsb_release -cs) main" + sudo apt-get install -y cmake +} + +mkdir -p "$SCRIPT_DIR/build" && cd "$SCRIPT_DIR/build/" +# rm -rf dgl +# git clone --recurse-submodules -b 2.1.x https://github.com/dmlc/dgl.git +cd dgl && mkdir build -p && cd build +# rm -rf ./* +cmake -DUSE_CUDA=ON .. +# cmake -DUSE_CUDA=ON -DCMAKE_C_COMPILER=clang -DCMAKE_CXX_COMPILER=clang++ .. +make -j4 + +# # https://docs.dgl.ai/install/index.html#system-requirements +# # TODO(gitbuda): conda is required on the system +# bash script/create_dev_conda_env.sh -g 12.1 +# bash script/build_dgl.sh -g diff --git a/python/requirements.txt b/python/requirements.txt index ceecc9dfe..ba3585a43 100644 --- a/python/requirements.txt +++ b/python/requirements.txt @@ -2,12 +2,12 @@ gekko==0.2.8 networkx==2.6.2 python-Levenshtein==0.12.1 elasticsearch==8.4.3 -gensim==4.0.0 -torch==1.12.0 +gensim==4.3.2 +torch==1.13.0 six==1.16.0 torchmetrics==0.9.3 igraph==0.10.2 -scikit-learn==0.24.2 +scikit-learn==1.4.0 gqlalchemy==1.4.1 mysql-connector-python==8.0.32 oracledb==1.2.2 diff --git a/python/tests/requirements.txt b/python/tests/requirements.txt index 42f6a1190..9073a6e1c 100644 --- a/python/tests/requirements.txt +++ b/python/tests/requirements.txt @@ -1,6 +1,6 @@ pytest==7.0 -PyYAML==5.4.1 -black==22.3.0 +PyYAML==6.0.1 +black==24.3.0 flake8==3.9.2 pymgclient==1.3.1 pytest-pylint==0.21.0 @@ -9,4 +9,4 @@ pytest-cov==2.12.1 pytest-benchmark==3.4.1 pytest-flake8==1.0.7 pytest-black==0.3.12 -gqlalchemy==1.4.1 \ No newline at end of file +gqlalchemy==1.4.1 diff --git a/test_e2e b/test_e2e index ced4a2f84..f96524696 100755 --- a/test_e2e +++ b/test_e2e @@ -15,25 +15,26 @@ def parse_arguments(): parser.add_argument( "-k", help="Add filter to tests you want to run", type=str, required=False ) + parser.add_argument( + "-l", help="List available tests (also works with the filter)", action="store_true" + ) args = parser.parse_args() return args -################################################# -# End to end tests # -################################################# - - -def main(test_filter: str = None): +def main(test_filter: str = "", list_tests: bool = False): os.environ["PYTHONPATH"] = E2E_DIRECTORY os.chdir(E2E_DIRECTORY) command = ["python3", "-m", "pytest", ".", "-vv"] if test_filter: command.extend(["-k", test_filter]) + if list_tests: + command.extend(["--collect-only"]) subprocess.run(command) if __name__ == "__main__": args = parse_arguments() test_filter = args.k - main(test_filter=test_filter) + list_tests = args.l + main(test_filter=test_filter, list_tests=list_tests)