diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index a471ae0..5029414 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -72,21 +72,25 @@ jobs: shell: bash run: ci/scripts/build_example.sh $(pwd)/example windows: - name: AMD64 Windows 2019 - runs-on: windows-2019 + name: AMD64 Windows 2022 + runs-on: windows-2022 timeout-minutes: 30 steps: - name: Checkout iceberg-cpp uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: fetch-depth: 0 + - name: Install ZLIB + shell: cmd + run: | + vcpkg install zlib:x64-windows - name: Build Iceberg shell: cmd run: | - call "C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\VC\Auxiliary\Build\vcvarsall.bat" x64 - bash -c "ci/scripts/build_iceberg.sh $(pwd) + call "C:\Program Files\Microsoft Visual Studio\2022\Enterprise\VC\Auxiliary\Build\vcvarsall.bat" x64 + bash -c "ci/scripts/build_iceberg.sh $(pwd)" - name: Build Example shell: cmd run: | - call "C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\VC\Auxiliary\Build\vcvarsall.bat" x64 - bash -c "ci/scripts/build_example.sh $(pwd)/example + call "C:\Program Files\Microsoft Visual Studio\2022\Enterprise\VC\Auxiliary\Build\vcvarsall.bat" x64 + bash -c "ci/scripts/build_example.sh $(pwd)/example" diff --git a/CMakeLists.txt b/CMakeLists.txt index 3c29687..9b52e30 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -37,6 +37,7 @@ option(ICEBERG_BUILD_STATIC "Build static library" ON) option(ICEBERG_BUILD_SHARED "Build shared library" OFF) option(ICEBERG_BUILD_TESTS "Build tests" ON) option(ICEBERG_ARROW "Build Arrow" ON) +option(ICEBERG_AVRO "Build Avro" ON) include(GNUInstallDirs) include(FetchContent) diff --git a/ci/scripts/build_example.sh b/ci/scripts/build_example.sh index a09f9c9..eb33e12 100755 --- a/ci/scripts/build_example.sh +++ b/ci/scripts/build_example.sh @@ -25,10 +25,27 @@ build_dir=${1}/build mkdir ${build_dir} pushd ${build_dir} -cmake \ - -DCMAKE_PREFIX_PATH=${CMAKE_INSTALL_PREFIX:-${ICEBERG_HOME}} \ - ${source_dir} -cmake --build . +is_windows() { + [[ "${OSTYPE}" == "msys" || "${OSTYPE}" == "win32" ]] +} + +CMAKE_ARGS=( + "-DCMAKE_PREFIX_PATH=${CMAKE_INSTALL_PREFIX:-${ICEBERG_HOME}}" +) + +if is_windows; then + CMAKE_ARGS+=("-DCMAKE_TOOLCHAIN_FILE=C:/vcpkg/scripts/buildsystems/vcpkg.cmake") + CMAKE_ARGS+=("-DCMAKE_BUILD_TYPE=Release") +else + CMAKE_ARGS+=("-DCMAKE_BUILD_TYPE=Debug") +fi + +cmake "${CMAKE_ARGS[@]}" ${source_dir} +if is_windows; then + cmake --build . --config Release +else + cmake --build . +fi popd diff --git a/ci/scripts/build_iceberg.sh b/ci/scripts/build_iceberg.sh index d08e0b2..4351b0f 100755 --- a/ci/scripts/build_iceberg.sh +++ b/ci/scripts/build_iceberg.sh @@ -25,13 +25,31 @@ build_dir=${1}/build mkdir ${build_dir} pushd ${build_dir} -cmake \ - -DCMAKE_INSTALL_PREFIX=${CMAKE_INSTALL_PREFIX:-${ICEBERG_HOME}} \ - -DICEBERG_BUILD_STATIC=ON \ - -DICEBERG_BUILD_SHARED=ON \ - ${source_dir} -cmake --build . --target install -ctest --output-on-failure -C Debug +is_windows() { + [[ "${OSTYPE}" == "msys" || "${OSTYPE}" == "win32" ]] +} + +CMAKE_ARGS=( + "-DCMAKE_INSTALL_PREFIX=${CMAKE_INSTALL_PREFIX:-${ICEBERG_HOME}}" + "-DICEBERG_BUILD_STATIC=ON" + "-DICEBERG_BUILD_SHARED=ON" +) + +if is_windows; then + CMAKE_ARGS+=("-DCMAKE_TOOLCHAIN_FILE=C:/vcpkg/scripts/buildsystems/vcpkg.cmake") + CMAKE_ARGS+=("-DCMAKE_BUILD_TYPE=Release") +else + CMAKE_ARGS+=("-DCMAKE_BUILD_TYPE=Debug") +fi + +cmake "${CMAKE_ARGS[@]}" ${source_dir} +if is_windows; then + cmake --build . --config Release --target install + ctest --output-on-failure -C Release +else + cmake --build . --target install + ctest --output-on-failure +fi popd diff --git a/cmake_modules/IcebergThirdpartyToolchain.cmake b/cmake_modules/IcebergThirdpartyToolchain.cmake index 1b00134..83deb19 100644 --- a/cmake_modules/IcebergThirdpartyToolchain.cmake +++ b/cmake_modules/IcebergThirdpartyToolchain.cmake @@ -126,3 +126,77 @@ endfunction() if(ICEBERG_ARROW) resolve_arrow_dependency() endif() + +# ---------------------------------------------------------------------- +# Apache Avro + +function(resolve_avro_dependency) + prepare_fetchcontent() + + set(AVRO_USE_BOOST + OFF + CACHE BOOL "" FORCE) + + set(AVRO_BUILD_EXECUTABLES + OFF + CACHE BOOL "" FORCE) + + set(AVRO_BUILD_TESTS + OFF + CACHE BOOL "" FORCE) + + fetchcontent_declare(Avro + ${FC_DECLARE_COMMON_OPTIONS} + # TODO: switch to upstream once the PR below is merged + # https://github.com/apache/avro/pull/3299 + # Eventually, we should switch to Apache Avro 1.3.0. + GIT_REPOSITORY https://github.com/wgtmac/avro.git + GIT_TAG 0aa7adf87a9af6d472a3e9d5966c5e7f1d6baa7d + SOURCE_SUBDIR + lang/c++ + FIND_PACKAGE_ARGS + NAMES + Avro + CONFIG) + + fetchcontent_makeavailable(Avro) + + if(avro_SOURCE_DIR) + if(NOT TARGET Avro::avrocpp_static) + add_library(Avro::avrocpp_static INTERFACE IMPORTED) + target_link_libraries(Avro::avrocpp_static INTERFACE avrocpp_s) + target_include_directories(Avro::avrocpp_static + INTERFACE ${avro_BINARY_DIR} ${avro_SOURCE_DIR}/lang/c++) + endif() + + set(AVRO_VENDORED TRUE) + set_target_properties(avrocpp_s PROPERTIES OUTPUT_NAME "iceberg_vendored_avrocpp") + set_target_properties(avrocpp_s PROPERTIES POSITION_INDEPENDENT_CODE ON) + install(TARGETS avrocpp_s + EXPORT iceberg_targets + RUNTIME DESTINATION "${ICEBERG_INSTALL_BINDIR}" + ARCHIVE DESTINATION "${ICEBERG_INSTALL_LIBDIR}" + LIBRARY DESTINATION "${ICEBERG_INSTALL_LIBDIR}") + + # TODO: add vendored ZLIB and Snappy support + find_package(Snappy CONFIG) + if(Snappy_FOUND) + list(APPEND ICEBERG_SYSTEM_DEPENDENCIES Snappy) + endif() + list(APPEND ICEBERG_SYSTEM_DEPENDENCIES ZLIB) + else() + set(AVRO_VENDORED FALSE) + list(APPEND ICEBERG_SYSTEM_DEPENDENCIES Avro) + endif() + + set(ICEBERG_SYSTEM_DEPENDENCIES + ${ICEBERG_SYSTEM_DEPENDENCIES} + PARENT_SCOPE) + set(AVRO_VENDORED + ${AVRO_VENDORED} + PARENT_SCOPE) +endfunction() + +if(ICEBERG_AVRO) + resolve_avro_dependency() +endif() diff --git a/example/CMakeLists.txt b/example/CMakeLists.txt index 4da20ca..c2c7a9c 100644 --- a/example/CMakeLists.txt +++ b/example/CMakeLists.txt @@ -26,5 +26,6 @@ find_package(Iceberg CONFIG REQUIRED) add_executable(demo_example demo_example.cc) -target_link_libraries(demo_example PRIVATE Iceberg::iceberg_puffin_static - Iceberg::iceberg_arrow_static) +target_link_libraries(demo_example + PRIVATE Iceberg::iceberg_puffin_static + Iceberg::iceberg_arrow_static Iceberg::iceberg_avro_static) diff --git a/example/demo_example.cc b/example/demo_example.cc index f4801f1..f584339 100644 --- a/example/demo_example.cc +++ b/example/demo_example.cc @@ -20,6 +20,7 @@ #include #include "iceberg/arrow/demo_arrow.h" +#include "iceberg/avro/demo_avro.h" #include "iceberg/demo_table.h" #include "iceberg/puffin/demo_puffin.h" @@ -27,5 +28,6 @@ int main() { std::cout << iceberg::DemoTable().print() << std::endl; std::cout << iceberg::puffin::DemoPuffin().print() << std::endl; std::cout << iceberg::arrow::DemoArrow().print() << std::endl; + std::cout << iceberg::avro::DemoAvro().print() << std::endl; return 0; } diff --git a/src/iceberg/CMakeLists.txt b/src/iceberg/CMakeLists.txt index 4bebe4e..24563a5 100644 --- a/src/iceberg/CMakeLists.txt +++ b/src/iceberg/CMakeLists.txt @@ -29,6 +29,7 @@ install(FILES ${CMAKE_CURRENT_BINARY_DIR}/iceberg_export.h DESTINATION ${ICEBERG_INSTALL_INCLUDEDIR}/iceberg) add_subdirectory(arrow) +add_subdirectory(avro) add_subdirectory(puffin) iceberg_install_cmake_package(Iceberg iceberg_targets) diff --git a/src/iceberg/avro.h b/src/iceberg/avro.h new file mode 100644 index 0000000..9bd0160 --- /dev/null +++ b/src/iceberg/avro.h @@ -0,0 +1,34 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +#include + +#include "iceberg/iceberg_export.h" + +namespace iceberg { + +class ICEBERG_EXPORT Avro { + public: + virtual ~Avro() = default; + virtual std::string print() const = 0; +}; + +} // namespace iceberg diff --git a/src/iceberg/avro/CMakeLists.txt b/src/iceberg/avro/CMakeLists.txt new file mode 100644 index 0000000..6a17eb5 --- /dev/null +++ b/src/iceberg/avro/CMakeLists.txt @@ -0,0 +1,82 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +if(NOT ICEBERG_AVRO) + return() +endif() + +set(ICEBERG_AVRO_SOURCES demo_avro.cc) +set(ICEBERG_AVRO_INCLUDES "${ICEBERG_INCLUDES}") + +# Libraries to link with exported libiceberg_avro.{so,a}. +set(ICEBERG_AVRO_STATIC_BUILD_INTERFACE_LIBS) +set(ICEBERG_AVRO_SHARED_BUILD_INTERFACE_LIBS) +set(ICEBERG_AVRO_STATIC_INSTALL_INTERFACE_LIBS) +set(ICEBERG_AVRO_SHARED_INSTALL_INTERFACE_LIBS) + +list(APPEND + ICEBERG_AVRO_STATIC_BUILD_INTERFACE_LIBS + "$,iceberg_static,iceberg_shared>" + "$,Avro::avrocpp_static,Avro::avrocpp_shared>" +) +list(APPEND + ICEBERG_AVRO_SHARED_BUILD_INTERFACE_LIBS + "$,iceberg_shared,iceberg_static>" + "$,Avro::avrocpp_shared,Avro::avrocpp_static>" +) + +if(AVRO_VENDORED) + list(APPEND ICEBERG_AVRO_STATIC_INSTALL_INTERFACE_LIBS Iceberg::avrocpp_s) + list(APPEND ICEBERG_AVRO_SHARED_INSTALL_INTERFACE_LIBS Iceberg::avrocpp_s) +else() + list(APPEND + ICEBERG_AVRO_STATIC_INSTALL_INTERFACE_LIBS + "$,Avro::avrocpp_static,Avro::avrocpp_shared>" + ) + list(APPEND + ICEBERG_AVRO_SHARED_INSTALL_INTERFACE_LIBS + "$,Avro::avrocpp_shared,Avro::avrocpp_static>" + ) +endif() + +list(APPEND + ICEBERG_AVRO_STATIC_INSTALL_INTERFACE_LIBS + "$,Iceberg::iceberg_static,Iceberg::iceberg_shared>" +) +list(APPEND + ICEBERG_AVRO_SHARED_INSTALL_INTERFACE_LIBS + "$,Iceberg::iceberg_shared,Iceberg::iceberg_static>" +) + +add_iceberg_lib(iceberg_avro + SOURCES + ${ICEBERG_AVRO_SOURCES} + PRIVATE_INCLUDES + ${ICEBERG_AVRO_INCLUDES} + SHARED_LINK_LIBS + ${ICEBERG_AVRO_SHARED_BUILD_INTERFACE_LIBS} + STATIC_LINK_LIBS + ${ICEBERG_AVRO_STATIC_BUILD_INTERFACE_LIBS} + STATIC_INSTALL_INTERFACE_LIBS + ${ICEBERG_AVRO_STATIC_INSTALL_INTERFACE_LIBS} + SHARED_INSTALL_INTERFACE_LIBS + ${ICEBERG_AVRO_SHARED_INSTALL_INTERFACE_LIBS}) + +iceberg_install_all_headers(iceberg/avro) + +install(FILES ${CMAKE_CURRENT_BINARY_DIR}/iceberg_avro_export.h + DESTINATION ${ICEBERG_INSTALL_INCLUDEDIR}/iceberg/avro) diff --git a/src/iceberg/avro/demo_avro.cc b/src/iceberg/avro/demo_avro.cc new file mode 100644 index 0000000..b4bf00e --- /dev/null +++ b/src/iceberg/avro/demo_avro.cc @@ -0,0 +1,52 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "iceberg/avro/demo_avro.h" + +#include + +#include "avro/Compiler.hh" +#include "avro/ValidSchema.hh" +#include "iceberg/demo_table.h" + +namespace iceberg::avro { + +std::string DemoAvro::print() const { + std::string input = + "{\n\ + \"type\": \"record\",\n\ + \"name\": \"testrecord\",\n\ + \"fields\": [\n\ + {\n\ + \"name\": \"testbytes\",\n\ + \"type\": \"bytes\",\n\ + \"default\": \"\"\n\ + }\n\ + ]\n\ +}\n\ +"; + + ::avro::ValidSchema schema = ::avro::compileJsonSchemaFromString(input); + std::ostringstream actual; + schema.toJson(actual); + + return actual.str(); +} + +} // namespace iceberg::avro diff --git a/src/iceberg/avro/demo_avro.h b/src/iceberg/avro/demo_avro.h new file mode 100644 index 0000000..6abf969 --- /dev/null +++ b/src/iceberg/avro/demo_avro.h @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +#include + +#include "iceberg/avro.h" +#include "iceberg/avro/iceberg_avro_export.h" + +namespace iceberg::avro { + +class ICEBERG_AVRO_EXPORT DemoAvro : public Avro { + public: + DemoAvro() = default; + ~DemoAvro() override = default; + std::string print() const override; +}; + +} // namespace iceberg::avro diff --git a/src/iceberg/demo_table.cc b/src/iceberg/demo_table.cc index 29879b8..9e46bda 100644 --- a/src/iceberg/demo_table.cc +++ b/src/iceberg/demo_table.cc @@ -19,6 +19,7 @@ #include "iceberg/demo_table.h" +#include "iceberg/avro.h" // include to export symbols #include "iceberg/puffin.h" namespace iceberg { diff --git a/test/core/CMakeLists.txt b/test/core/CMakeLists.txt index 5512017..e6280ed 100644 --- a/test/core/CMakeLists.txt +++ b/test/core/CMakeLists.txt @@ -20,3 +20,11 @@ target_sources(core_unittest PRIVATE core_unittest.cc) target_link_libraries(core_unittest PRIVATE iceberg_static GTest::gtest_main) target_include_directories(core_unittest PRIVATE "${ICEBERG_INCLUDES}") add_test(NAME core_unittest COMMAND core_unittest) + +if(ICEBERG_AVRO) + add_executable(avro_unittest) + target_sources(avro_unittest PRIVATE avro_unittest.cc) + target_link_libraries(avro_unittest PRIVATE iceberg_avro_static GTest::gtest_main) + target_include_directories(avro_unittest PRIVATE "${ICEBERG_INCLUDES}") + add_test(NAME avro_unittest COMMAND avro_unittest) +endif() diff --git a/test/core/avro_unittest.cc b/test/core/avro_unittest.cc new file mode 100644 index 0000000..5bdcfca --- /dev/null +++ b/test/core/avro_unittest.cc @@ -0,0 +1,40 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include +#include + +TEST(AVROTest, TestDemoAvro) { + std::string expected = + "{\n\ + \"type\": \"record\",\n\ + \"name\": \"testrecord\",\n\ + \"fields\": [\n\ + {\n\ + \"name\": \"testbytes\",\n\ + \"type\": \"bytes\",\n\ + \"default\": \"\"\n\ + }\n\ + ]\n\ +}\n\ +"; + + auto avro = iceberg::avro::DemoAvro(); + EXPECT_EQ(avro.print(), expected); +}