Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add resource manager template arguments to class FilterBuffer. #4650

Closed
wants to merge 5 commits into from
Closed

Add resource manager template arguments to class FilterBuffer. #4650

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
d10651e
FilterBuffer
shaunrd0 Jan 17, 2024
e8ae691
Update references to FilterBuffer
shaunrd0 Jan 18, 2024
44e6fc5
Use ContextResources::resource_manager_type
shaunrd0 Jan 18, 2024
7f61da5
Update typedef
shaunrd0 Jan 19, 2024
1522e8a
WIP fix CI
shaunrd0 Jan 23, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
29 changes: 15 additions & 14 deletions test/src/unit-filter-buffer.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@

#include "tiledb/sm/filter/filter_buffer.h"
#include "tiledb/sm/filter/filter_storage.h"
#include "tiledb/sm/storage_manager/context.h"

#include <test/support/tdb_catch.h>
#include <iostream>
Expand All @@ -55,7 +56,7 @@ static void check_buf(const int* check, std::initializer_list<int> answer) {

TEST_CASE("FilterBuffer: Test init", "[filter][filter-buffer]") {
FilterStorage storage;
FilterBuffer fbuf(&storage);
FilterBuffer<Context::resource_manager_type> fbuf(&storage);

// Check reads and writes with a buffer not owned by the FilterBuffer.
const char data[] = {0, 1, 2, 3, 4, 5};
Expand Down Expand Up @@ -100,7 +101,7 @@ TEST_CASE("FilterBuffer: Test init", "[filter][filter-buffer]") {

TEST_CASE("FilterBuffer: Test prepend", "[filter][filter-buffer]") {
FilterStorage storage;
FilterBuffer fbuf(&storage);
FilterBuffer<Context::resource_manager_type> fbuf(&storage);

const char data[] = {0, 1, 2, 3, 4, 5};

Expand Down Expand Up @@ -176,7 +177,7 @@ TEST_CASE("FilterBuffer: Test prepend", "[filter][filter-buffer]") {
TEST_CASE(
"FilterBuffer: Test multiple reads/writes", "[filter][filter-buffer]") {
FilterStorage storage;
FilterBuffer fbuf(&storage);
FilterBuffer<Context::resource_manager_type> fbuf(&storage);

CHECK(fbuf.prepend_buffer(sizeof(int)).ok());
CHECK(fbuf.prepend_buffer(sizeof(int)).ok());
Expand All @@ -201,7 +202,7 @@ TEST_CASE(

TEST_CASE("FilterBuffer: Test write from other", "[filter][filter-buffer]") {
FilterStorage storage;
FilterBuffer fbuf(&storage), fbuf2(&storage);
FilterBuffer<Context::resource_manager_type> fbuf(&storage), fbuf2(&storage);

CHECK(fbuf.prepend_buffer(sizeof(int)).ok());
CHECK(fbuf.prepend_buffer(sizeof(int)).ok());
Expand All @@ -224,7 +225,7 @@ TEST_CASE("FilterBuffer: Test write from other", "[filter][filter-buffer]") {

TEST_CASE("FilterBuffer: Test get ConstBuffer", "[filter][filter-buffer]") {
FilterStorage storage;
FilterBuffer fbuf(&storage);
FilterBuffer<Context::resource_manager_type> fbuf(&storage);

CHECK(fbuf.prepend_buffer(sizeof(int)).ok());
CHECK(fbuf.prepend_buffer(sizeof(int)).ok());
Expand Down Expand Up @@ -255,7 +256,7 @@ TEST_CASE("FilterBuffer: Test get ConstBuffer", "[filter][filter-buffer]") {

TEST_CASE("FilterBuffer: Test clear", "[filter][filter-buffer]") {
FilterStorage storage;
FilterBuffer fbuf(&storage);
FilterBuffer<Context::resource_manager_type> fbuf(&storage);

const char init_data[] = {0, 1, 2};
Buffer buff;
Expand Down Expand Up @@ -295,7 +296,7 @@ TEST_CASE("FilterBuffer: Test clear", "[filter][filter-buffer]") {

TEST_CASE("FilterBuffer: Test copy_to", "[filter][filter-buffer]") {
FilterStorage storage;
FilterBuffer fbuf(&storage);
FilterBuffer<Context::resource_manager_type> fbuf(&storage);

const char init_data[] = {0, 1, 2};
Buffer buff;
Expand Down Expand Up @@ -328,14 +329,14 @@ TEST_CASE("FilterBuffer: Test copy_to", "[filter][filter-buffer]") {

TEST_CASE("FilterBuffer: Test append_view", "[filter][filter-buffer]") {
FilterStorage storage;
FilterBuffer fbuf(&storage);
FilterBuffer<Context::resource_manager_type> fbuf(&storage);

const char init_data[] = {0, 1, 2};
Buffer buff;
CHECK(buff.write(init_data, sizeof(init_data)).ok());
CHECK(fbuf.init(buff.data(), buff.size()).ok());

FilterBuffer fbuf2(&storage);
FilterBuffer<Context::resource_manager_type> fbuf2(&storage);
CHECK(fbuf2.append_view(&fbuf, 1, 2).ok());
char data_r[100];
fbuf2.reset_offset();
Expand Down Expand Up @@ -370,7 +371,7 @@ TEST_CASE("FilterBuffer: Test append_view", "[filter][filter-buffer]") {

TEST_CASE("FilterBuffer: Test view reclaim", "[filter][filter-buffer]") {
FilterStorage storage;
FilterBuffer fbuf(&storage), fbuf2(&storage);
FilterBuffer<Context::resource_manager_type> fbuf(&storage), fbuf2(&storage);

CHECK(storage.num_available() == 0);
CHECK(storage.num_in_use() == 0);
Expand Down Expand Up @@ -398,7 +399,7 @@ TEST_CASE("FilterBuffer: Test view reclaim", "[filter][filter-buffer]") {

TEST_CASE("FilterBuffer: Test fixed allocation", "[filter][filter-buffer]") {
FilterStorage storage;
FilterBuffer fbuf(&storage);
FilterBuffer<Context::resource_manager_type> fbuf(&storage);
Buffer fixed;
const unsigned nelts = 100;
CHECK(fixed.realloc(nelts * sizeof(unsigned)).ok());
Expand Down Expand Up @@ -440,7 +441,7 @@ TEST_CASE("FilterBuffer: Test fixed allocation", "[filter][filter-buffer]") {

SECTION("- Append view") {
// Set up data to view
FilterBuffer fbuf2(&storage);
FilterBuffer<Context::resource_manager_type> fbuf2(&storage);
CHECK(fbuf2.prepend_buffer(fbuf.size()).ok());
fbuf2.reset_offset();
for (unsigned i = 0; i < nelts; i++)
Expand Down Expand Up @@ -475,7 +476,7 @@ TEST_CASE(
"FilterBuffer: Test copy with reinterpret type",
"[filter][filter-buffer]") {
FilterStorage storage;
FilterBuffer fbuf(&storage), fbuf2(&storage);
FilterBuffer<Context::resource_manager_type> fbuf(&storage), fbuf2(&storage);

// Prepare some uint32 data in two separate buffers
const uint32_t a = 100, b = 101, c = 102, d = 200, e = 201, f = 202;
Expand Down Expand Up @@ -518,7 +519,7 @@ TEST_CASE(

TEST_CASE("FilterBuffer: Test read-only", "[filter][filter-buffer]") {
FilterStorage storage;
FilterBuffer fbuf(&storage);
FilterBuffer<Context::resource_manager_type> fbuf(&storage);

const char init_data[] = {0, 1, 2};
Buffer buff;
Expand Down
51 changes: 27 additions & 24 deletions test/src/unit-filter-pipeline.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -154,10 +154,11 @@ class Add1InPlace : public tiledb::sm::Filter {
Status run_forward(
const WriterTile&,
WriterTile* const,
FilterBuffer* input_metadata,
FilterBuffer* input,
FilterBuffer* output_metadata,
FilterBuffer* output) const override {
FilterBuffer<ContextResources::resource_manager_type>* input_metadata,
FilterBuffer<ContextResources::resource_manager_type>* input,
FilterBuffer<ContextResources::resource_manager_type>* output_metadata,
FilterBuffer<ContextResources::resource_manager_type>* output)
const override {
auto input_size = input->size();
RETURN_NOT_OK(output->append_view(input));
output->reset_offset();
Expand All @@ -178,10 +179,10 @@ class Add1InPlace : public tiledb::sm::Filter {
Status run_reverse(
const Tile&,
Tile*,
FilterBuffer* input_metadata,
FilterBuffer* input,
FilterBuffer* output_metadata,
FilterBuffer* output,
FilterBuffer<ContextResources::resource_manager_type>* input_metadata,
FilterBuffer<ContextResources::resource_manager_type>* input,
FilterBuffer<ContextResources::resource_manager_type>* output_metadata,
FilterBuffer<ContextResources::resource_manager_type>* output,
const tiledb::sm::Config& config) const override {
(void)config;

Expand Down Expand Up @@ -225,10 +226,11 @@ class Add1OutOfPlace : public tiledb::sm::Filter {
Status run_forward(
const WriterTile&,
WriterTile* const,
FilterBuffer* input_metadata,
FilterBuffer* input,
FilterBuffer* output_metadata,
FilterBuffer* output) const override {
FilterBuffer<ContextResources::resource_manager_type>* input_metadata,
FilterBuffer<ContextResources::resource_manager_type>* input,
FilterBuffer<ContextResources::resource_manager_type>* output_metadata,
FilterBuffer<ContextResources::resource_manager_type>* output)
const override {
auto input_size = input->size();
auto nelts = input_size / sizeof(uint64_t);

Expand Down Expand Up @@ -260,10 +262,10 @@ class Add1OutOfPlace : public tiledb::sm::Filter {
Status run_reverse(
const Tile&,
Tile*,
FilterBuffer* input_metadata,
FilterBuffer* input,
FilterBuffer* output_metadata,
FilterBuffer* output,
FilterBuffer<ContextResources::resource_manager_type>* input_metadata,
FilterBuffer<ContextResources::resource_manager_type>* input,
FilterBuffer<ContextResources::resource_manager_type>* output_metadata,
FilterBuffer<ContextResources::resource_manager_type>* output,
const tiledb::sm::Config& config) const override {
(void)config;

Expand Down Expand Up @@ -317,10 +319,11 @@ class PseudoChecksumFilter : public tiledb::sm::Filter {
Status run_forward(
const WriterTile&,
WriterTile* const,
FilterBuffer* input_metadata,
FilterBuffer* input,
FilterBuffer* output_metadata,
FilterBuffer* output) const override {
FilterBuffer<ContextResources::resource_manager_type>* input_metadata,
FilterBuffer<ContextResources::resource_manager_type>* input,
FilterBuffer<ContextResources::resource_manager_type>* output_metadata,
FilterBuffer<ContextResources::resource_manager_type>* output)
const override {
auto input_size = input->size();
auto nelts = input_size / sizeof(uint64_t);

Expand Down Expand Up @@ -348,10 +351,10 @@ class PseudoChecksumFilter : public tiledb::sm::Filter {
Status run_reverse(
const Tile&,
Tile*,
FilterBuffer* input_metadata,
FilterBuffer* input,
FilterBuffer* output_metadata,
FilterBuffer* output,
FilterBuffer<ContextResources::resource_manager_type>* input_metadata,
FilterBuffer<ContextResources::resource_manager_type>* input,
FilterBuffer<ContextResources::resource_manager_type>* output_metadata,
FilterBuffer<ContextResources::resource_manager_type>* output,
const tiledb::sm::Config& config) const override {
(void)config;

Expand Down
4 changes: 3 additions & 1 deletion tiledb/common/common.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,9 @@
*/
#include "common-std.h"

namespace tiledb::common {}
namespace tiledb::common {
using context_bypass_RM = void;
}
namespace tdb = tiledb::common;

/*
Expand Down
5 changes: 4 additions & 1 deletion tiledb/sm/compressors/util/tdb_gzip_embedded_data.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,7 @@
#include "tiledb/sm/filter/filter_buffer.h"
#include "tiledb/sm/filter/filter_storage.h"
#include "tiledb/sm/misc/types.h"
#include "tiledb/sm/storage_manager/context_resources.h"

#include "gzip_wrappers.h"

Expand All @@ -63,7 +64,9 @@ int main(int argc, char* argv[]) {
char fbuf[seg_sz];
auto filter_stg = make_shared<tiledb::sm::FilterStorage>(HERE());
auto inbuf = make_shared<tiledb::sm::Buffer>(HERE());
auto zipped_buf = make_shared<tiledb::sm::FilterBuffer>(HERE(), &*filter_stg);
auto zipped_buf = make_shared<tiledb::sm::FilterBuffer<
tiledb::sm::ContextResources::resource_manager_type>>(
HERE(), &*filter_stg);
uint64_t nread;
uint64_t cntread = 0;

Expand Down
45 changes: 25 additions & 20 deletions tiledb/sm/filter/bit_width_reduction_filter.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -109,10 +109,10 @@ bool BitWidthReductionFilter::accepts_input_datatype(Datatype datatype) const {
Status BitWidthReductionFilter::run_forward(
const WriterTile& tile,
WriterTile* const offsets_tile,
FilterBuffer* input_metadata,
FilterBuffer* input,
FilterBuffer* output_metadata,
FilterBuffer* output) const {
FilterBuffer<ContextResources::resource_manager_type>* input_metadata,
FilterBuffer<ContextResources::resource_manager_type>* input,
FilterBuffer<ContextResources::resource_manager_type>* output_metadata,
FilterBuffer<ContextResources::resource_manager_type>* output) const {
switch (filter_data_type_) {
case Datatype::INT16:
return run_forward<int16_t>(
Expand Down Expand Up @@ -178,10 +178,10 @@ template <typename T>
Status BitWidthReductionFilter::run_forward(
const WriterTile&,
WriterTile* const,
FilterBuffer* input_metadata,
FilterBuffer* input,
FilterBuffer* output_metadata,
FilterBuffer* output) const {
FilterBuffer<ContextResources::resource_manager_type>* input_metadata,
FilterBuffer<ContextResources::resource_manager_type>* input,
FilterBuffer<ContextResources::resource_manager_type>* output_metadata,
FilterBuffer<ContextResources::resource_manager_type>* output) const {
auto input_size = static_cast<uint32_t>(input->size());

// Compute the upper bound on the size of the output.
Expand Down Expand Up @@ -229,8 +229,9 @@ Status BitWidthReductionFilter::run_forward(
template <typename T>
Status BitWidthReductionFilter::compress_part(
ConstBuffer* input,
FilterBuffer* output,
FilterBuffer* output_metadata) const {
FilterBuffer<ContextResources::resource_manager_type>* output,
FilterBuffer<ContextResources::resource_manager_type>* output_metadata)
const {
// Compute window size in bytes as a multiple of the element width
auto input_bytes = static_cast<uint32_t>(input->size());
uint32_t window_size = std::min(input_bytes, max_window_size_);
Expand Down Expand Up @@ -281,10 +282,10 @@ Status BitWidthReductionFilter::compress_part(
Status BitWidthReductionFilter::run_reverse(
const Tile& tile,
Tile* const offsets_tile,
FilterBuffer* input_metadata,
FilterBuffer* input,
FilterBuffer* output_metadata,
FilterBuffer* output,
FilterBuffer<ContextResources::resource_manager_type>* input_metadata,
FilterBuffer<ContextResources::resource_manager_type>* input,
FilterBuffer<ContextResources::resource_manager_type>* output_metadata,
FilterBuffer<ContextResources::resource_manager_type>* output,
const Config&) const {
switch (filter_data_type_) {
case Datatype::INT16:
Expand Down Expand Up @@ -351,10 +352,10 @@ template <typename T>
Status BitWidthReductionFilter::run_reverse(
const Tile&,
Tile* const,
FilterBuffer* input_metadata,
FilterBuffer* input,
FilterBuffer* output_metadata,
FilterBuffer* output) const {
FilterBuffer<ContextResources::resource_manager_type>* input_metadata,
FilterBuffer<ContextResources::resource_manager_type>* input,
FilterBuffer<ContextResources::resource_manager_type>* output_metadata,
FilterBuffer<ContextResources::resource_manager_type>* output) const {
auto data_type_size = datatype_size(filter_data_type_);

uint32_t num_windows, orig_length;
Expand Down Expand Up @@ -439,7 +440,9 @@ uint8_t BitWidthReductionFilter::compute_bits_required(

template <typename T>
Status BitWidthReductionFilter::write_compressed_value(
FilterBuffer* buffer, T value, uint8_t num_bits) const {
FilterBuffer<ContextResources::resource_manager_type>* buffer,
T value,
uint8_t num_bits) const {
switch (num_bits) {
case 8: {
auto val = static_cast<
Expand Down Expand Up @@ -481,7 +484,9 @@ Status BitWidthReductionFilter::write_compressed_value(

template <typename T>
Status BitWidthReductionFilter::read_compressed_value(
FilterBuffer* buffer, uint8_t compressed_bits, T* value) const {
FilterBuffer<ContextResources::resource_manager_type>* buffer,
uint8_t compressed_bits,
T* value) const {
switch (compressed_bits) {
case 8: {
typename std::conditional<std::is_signed<T>::value, int8_t, uint8_t>::type
Expand Down
Loading
Loading