Skip to content

Commit

Permalink
Add iceberg_avro interface (#34)
Browse files Browse the repository at this point in the history
Add iceberg_avro interface, this closes #17

---------

Signed-off-by: Junwang Zhao <[email protected]>
Co-authored-by: Gang Wu <[email protected]>
  • Loading branch information
zhjwpku and wgtmac authored Jan 30, 2025
1 parent 7e08c33 commit c0bf8ce
Show file tree
Hide file tree
Showing 15 changed files with 390 additions and 19 deletions.
16 changes: 10 additions & 6 deletions .github/workflows/test.yml
Original file line number Diff line number Diff line change
Expand Up @@ -72,21 +72,25 @@ jobs:
shell: bash
run: ci/scripts/build_example.sh $(pwd)/example
windows:
name: AMD64 Windows 2019
runs-on: windows-2019
name: AMD64 Windows 2022
runs-on: windows-2022
timeout-minutes: 30
steps:
- name: Checkout iceberg-cpp
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
with:
fetch-depth: 0
- name: Install ZLIB
shell: cmd
run: |
vcpkg install zlib:x64-windows
- name: Build Iceberg
shell: cmd
run: |
call "C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\VC\Auxiliary\Build\vcvarsall.bat" x64
bash -c "ci/scripts/build_iceberg.sh $(pwd)
call "C:\Program Files\Microsoft Visual Studio\2022\Enterprise\VC\Auxiliary\Build\vcvarsall.bat" x64
bash -c "ci/scripts/build_iceberg.sh $(pwd)"
- name: Build Example
shell: cmd
run: |
call "C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\VC\Auxiliary\Build\vcvarsall.bat" x64
bash -c "ci/scripts/build_example.sh $(pwd)/example
call "C:\Program Files\Microsoft Visual Studio\2022\Enterprise\VC\Auxiliary\Build\vcvarsall.bat" x64
bash -c "ci/scripts/build_example.sh $(pwd)/example"
1 change: 1 addition & 0 deletions CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,7 @@ option(ICEBERG_BUILD_STATIC "Build static library" ON)
option(ICEBERG_BUILD_SHARED "Build shared library" OFF)
option(ICEBERG_BUILD_TESTS "Build tests" ON)
option(ICEBERG_ARROW "Build Arrow" ON)
option(ICEBERG_AVRO "Build Avro" ON)

include(GNUInstallDirs)
include(FetchContent)
Expand Down
25 changes: 21 additions & 4 deletions ci/scripts/build_example.sh
Original file line number Diff line number Diff line change
Expand Up @@ -25,10 +25,27 @@ build_dir=${1}/build
mkdir ${build_dir}
pushd ${build_dir}

cmake \
-DCMAKE_PREFIX_PATH=${CMAKE_INSTALL_PREFIX:-${ICEBERG_HOME}} \
${source_dir}
cmake --build .
is_windows() {
[[ "${OSTYPE}" == "msys" || "${OSTYPE}" == "win32" ]]
}

CMAKE_ARGS=(
"-DCMAKE_PREFIX_PATH=${CMAKE_INSTALL_PREFIX:-${ICEBERG_HOME}}"
)

if is_windows; then
CMAKE_ARGS+=("-DCMAKE_TOOLCHAIN_FILE=C:/vcpkg/scripts/buildsystems/vcpkg.cmake")
CMAKE_ARGS+=("-DCMAKE_BUILD_TYPE=Release")
else
CMAKE_ARGS+=("-DCMAKE_BUILD_TYPE=Debug")
fi

cmake "${CMAKE_ARGS[@]}" ${source_dir}
if is_windows; then
cmake --build . --config Release
else
cmake --build .
fi

popd

Expand Down
32 changes: 25 additions & 7 deletions ci/scripts/build_iceberg.sh
Original file line number Diff line number Diff line change
Expand Up @@ -25,13 +25,31 @@ build_dir=${1}/build
mkdir ${build_dir}
pushd ${build_dir}

cmake \
-DCMAKE_INSTALL_PREFIX=${CMAKE_INSTALL_PREFIX:-${ICEBERG_HOME}} \
-DICEBERG_BUILD_STATIC=ON \
-DICEBERG_BUILD_SHARED=ON \
${source_dir}
cmake --build . --target install
ctest --output-on-failure -C Debug
is_windows() {
[[ "${OSTYPE}" == "msys" || "${OSTYPE}" == "win32" ]]
}

CMAKE_ARGS=(
"-DCMAKE_INSTALL_PREFIX=${CMAKE_INSTALL_PREFIX:-${ICEBERG_HOME}}"
"-DICEBERG_BUILD_STATIC=ON"
"-DICEBERG_BUILD_SHARED=ON"
)

if is_windows; then
CMAKE_ARGS+=("-DCMAKE_TOOLCHAIN_FILE=C:/vcpkg/scripts/buildsystems/vcpkg.cmake")
CMAKE_ARGS+=("-DCMAKE_BUILD_TYPE=Release")
else
CMAKE_ARGS+=("-DCMAKE_BUILD_TYPE=Debug")
fi

cmake "${CMAKE_ARGS[@]}" ${source_dir}
if is_windows; then
cmake --build . --config Release --target install
ctest --output-on-failure -C Release
else
cmake --build . --target install
ctest --output-on-failure
fi

popd

Expand Down
74 changes: 74 additions & 0 deletions cmake_modules/IcebergThirdpartyToolchain.cmake
Original file line number Diff line number Diff line change
Expand Up @@ -126,3 +126,77 @@ endfunction()
if(ICEBERG_ARROW)
resolve_arrow_dependency()
endif()

# ----------------------------------------------------------------------
# Apache Avro

function(resolve_avro_dependency)
prepare_fetchcontent()

set(AVRO_USE_BOOST
OFF
CACHE BOOL "" FORCE)

set(AVRO_BUILD_EXECUTABLES
OFF
CACHE BOOL "" FORCE)

set(AVRO_BUILD_TESTS
OFF
CACHE BOOL "" FORCE)

fetchcontent_declare(Avro
${FC_DECLARE_COMMON_OPTIONS}
# TODO: switch to upstream once the PR below is merged
# https://github.com/apache/avro/pull/3299
# Eventually, we should switch to Apache Avro 1.3.0.
GIT_REPOSITORY https://github.com/wgtmac/avro.git
GIT_TAG 0aa7adf87a9af6d472a3e9d5966c5e7f1d6baa7d
SOURCE_SUBDIR
lang/c++
FIND_PACKAGE_ARGS
NAMES
Avro
CONFIG)

fetchcontent_makeavailable(Avro)

if(avro_SOURCE_DIR)
if(NOT TARGET Avro::avrocpp_static)
add_library(Avro::avrocpp_static INTERFACE IMPORTED)
target_link_libraries(Avro::avrocpp_static INTERFACE avrocpp_s)
target_include_directories(Avro::avrocpp_static
INTERFACE ${avro_BINARY_DIR} ${avro_SOURCE_DIR}/lang/c++)
endif()

set(AVRO_VENDORED TRUE)
set_target_properties(avrocpp_s PROPERTIES OUTPUT_NAME "iceberg_vendored_avrocpp")
set_target_properties(avrocpp_s PROPERTIES POSITION_INDEPENDENT_CODE ON)
install(TARGETS avrocpp_s
EXPORT iceberg_targets
RUNTIME DESTINATION "${ICEBERG_INSTALL_BINDIR}"
ARCHIVE DESTINATION "${ICEBERG_INSTALL_LIBDIR}"
LIBRARY DESTINATION "${ICEBERG_INSTALL_LIBDIR}")

# TODO: add vendored ZLIB and Snappy support
find_package(Snappy CONFIG)
if(Snappy_FOUND)
list(APPEND ICEBERG_SYSTEM_DEPENDENCIES Snappy)
endif()
list(APPEND ICEBERG_SYSTEM_DEPENDENCIES ZLIB)
else()
set(AVRO_VENDORED FALSE)
list(APPEND ICEBERG_SYSTEM_DEPENDENCIES Avro)
endif()

set(ICEBERG_SYSTEM_DEPENDENCIES
${ICEBERG_SYSTEM_DEPENDENCIES}
PARENT_SCOPE)
set(AVRO_VENDORED
${AVRO_VENDORED}
PARENT_SCOPE)
endfunction()

if(ICEBERG_AVRO)
resolve_avro_dependency()
endif()
5 changes: 3 additions & 2 deletions example/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -26,5 +26,6 @@ find_package(Iceberg CONFIG REQUIRED)

add_executable(demo_example demo_example.cc)

target_link_libraries(demo_example PRIVATE Iceberg::iceberg_puffin_static
Iceberg::iceberg_arrow_static)
target_link_libraries(demo_example
PRIVATE Iceberg::iceberg_puffin_static
Iceberg::iceberg_arrow_static Iceberg::iceberg_avro_static)
2 changes: 2 additions & 0 deletions example/demo_example.cc
Original file line number Diff line number Diff line change
Expand Up @@ -20,12 +20,14 @@
#include <iostream>

#include "iceberg/arrow/demo_arrow.h"
#include "iceberg/avro/demo_avro.h"
#include "iceberg/demo_table.h"
#include "iceberg/puffin/demo_puffin.h"

int main() {
std::cout << iceberg::DemoTable().print() << std::endl;
std::cout << iceberg::puffin::DemoPuffin().print() << std::endl;
std::cout << iceberg::arrow::DemoArrow().print() << std::endl;
std::cout << iceberg::avro::DemoAvro().print() << std::endl;
return 0;
}
1 change: 1 addition & 0 deletions src/iceberg/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@ install(FILES ${CMAKE_CURRENT_BINARY_DIR}/iceberg_export.h
DESTINATION ${ICEBERG_INSTALL_INCLUDEDIR}/iceberg)

add_subdirectory(arrow)
add_subdirectory(avro)
add_subdirectory(puffin)

iceberg_install_cmake_package(Iceberg iceberg_targets)
34 changes: 34 additions & 0 deletions src/iceberg/avro.h
Original file line number Diff line number Diff line change
@@ -0,0 +1,34 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/

#pragma once

#include <string>

#include "iceberg/iceberg_export.h"

namespace iceberg {

class ICEBERG_EXPORT Avro {
public:
virtual ~Avro() = default;
virtual std::string print() const = 0;
};

} // namespace iceberg
82 changes: 82 additions & 0 deletions src/iceberg/avro/CMakeLists.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1,82 @@
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.

if(NOT ICEBERG_AVRO)
return()
endif()

set(ICEBERG_AVRO_SOURCES demo_avro.cc)
set(ICEBERG_AVRO_INCLUDES "${ICEBERG_INCLUDES}")

# Libraries to link with exported libiceberg_avro.{so,a}.
set(ICEBERG_AVRO_STATIC_BUILD_INTERFACE_LIBS)
set(ICEBERG_AVRO_SHARED_BUILD_INTERFACE_LIBS)
set(ICEBERG_AVRO_STATIC_INSTALL_INTERFACE_LIBS)
set(ICEBERG_AVRO_SHARED_INSTALL_INTERFACE_LIBS)

list(APPEND
ICEBERG_AVRO_STATIC_BUILD_INTERFACE_LIBS
"$<IF:$<TARGET_EXISTS:iceberg_static>,iceberg_static,iceberg_shared>"
"$<IF:$<TARGET_EXISTS:Avro::avrocpp_static>,Avro::avrocpp_static,Avro::avrocpp_shared>"
)
list(APPEND
ICEBERG_AVRO_SHARED_BUILD_INTERFACE_LIBS
"$<IF:$<TARGET_EXISTS:iceberg_shared>,iceberg_shared,iceberg_static>"
"$<IF:$<TARGET_EXISTS:Avro::avrocpp_shared>,Avro::avrocpp_shared,Avro::avrocpp_static>"
)

if(AVRO_VENDORED)
list(APPEND ICEBERG_AVRO_STATIC_INSTALL_INTERFACE_LIBS Iceberg::avrocpp_s)
list(APPEND ICEBERG_AVRO_SHARED_INSTALL_INTERFACE_LIBS Iceberg::avrocpp_s)
else()
list(APPEND
ICEBERG_AVRO_STATIC_INSTALL_INTERFACE_LIBS
"$<IF:$<TARGET_EXISTS:Avro::avrocpp_static>,Avro::avrocpp_static,Avro::avrocpp_shared>"
)
list(APPEND
ICEBERG_AVRO_SHARED_INSTALL_INTERFACE_LIBS
"$<IF:$<TARGET_EXISTS:Avro::avrocpp_shared>,Avro::avrocpp_shared,Avro::avrocpp_static>"
)
endif()

list(APPEND
ICEBERG_AVRO_STATIC_INSTALL_INTERFACE_LIBS
"$<IF:$<TARGET_EXISTS:Iceberg::iceberg_static>,Iceberg::iceberg_static,Iceberg::iceberg_shared>"
)
list(APPEND
ICEBERG_AVRO_SHARED_INSTALL_INTERFACE_LIBS
"$<IF:$<TARGET_EXISTS:Iceberg::iceberg_shared>,Iceberg::iceberg_shared,Iceberg::iceberg_static>"
)

add_iceberg_lib(iceberg_avro
SOURCES
${ICEBERG_AVRO_SOURCES}
PRIVATE_INCLUDES
${ICEBERG_AVRO_INCLUDES}
SHARED_LINK_LIBS
${ICEBERG_AVRO_SHARED_BUILD_INTERFACE_LIBS}
STATIC_LINK_LIBS
${ICEBERG_AVRO_STATIC_BUILD_INTERFACE_LIBS}
STATIC_INSTALL_INTERFACE_LIBS
${ICEBERG_AVRO_STATIC_INSTALL_INTERFACE_LIBS}
SHARED_INSTALL_INTERFACE_LIBS
${ICEBERG_AVRO_SHARED_INSTALL_INTERFACE_LIBS})

iceberg_install_all_headers(iceberg/avro)

install(FILES ${CMAKE_CURRENT_BINARY_DIR}/iceberg_avro_export.h
DESTINATION ${ICEBERG_INSTALL_INCLUDEDIR}/iceberg/avro)
52 changes: 52 additions & 0 deletions src/iceberg/avro/demo_avro.cc
Original file line number Diff line number Diff line change
@@ -0,0 +1,52 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/

#include "iceberg/avro/demo_avro.h"

#include <sstream>

#include "avro/Compiler.hh"
#include "avro/ValidSchema.hh"
#include "iceberg/demo_table.h"

namespace iceberg::avro {

std::string DemoAvro::print() const {
std::string input =
"{\n\
\"type\": \"record\",\n\
\"name\": \"testrecord\",\n\
\"fields\": [\n\
{\n\
\"name\": \"testbytes\",\n\
\"type\": \"bytes\",\n\
\"default\": \"\"\n\
}\n\
]\n\
}\n\
";

::avro::ValidSchema schema = ::avro::compileJsonSchemaFromString(input);
std::ostringstream actual;
schema.toJson(actual);

return actual.str();
}

} // namespace iceberg::avro
Loading

0 comments on commit c0bf8ce

Please sign in to comment.