From 9ec8ebbd0f2873096dabc80d4362e86b2e80aefa Mon Sep 17 00:00:00 2001
From: "kevin.russ" <kevin.russ@accenture.com>
Date: Thu, 16 Jan 2025 14:49:21 +0100
Subject: [PATCH 1/2] Remove buf_redux
---
application/apps/indexer/Cargo.lock | 479 ++++++++----------
application/apps/indexer/Cargo.toml | 5 +-
.../apps/indexer/addons/text_grep/Cargo.toml | 2 +-
.../indexer/addons/text_grep/src/buffer.rs | 73 ++-
.../apps/indexer/addons/text_grep/src/lib.rs | 8 +-
application/apps/indexer/processor/Cargo.toml | 2 +-
.../indexer/processor/src/search/buffer.rs | 75 ++-
.../processor/src/search/searchers/mod.rs | 10 +-
.../apps/indexer/processor/src/text_source.rs | 9 +-
application/apps/indexer/sources/Cargo.toml | 2 +-
.../indexer/sources/src/binary/pcap/legacy.rs | 20 +-
.../indexer/sources/src/binary/pcap/ng.rs | 20 +-
.../apps/indexer/sources/src/binary/raw.rs | 14 +-
.../indexer/sources/src/command/process.rs | 18 +-
.../indexer/sources/src/serial/serialport.rs | 16 +-
.../apps/indexer/sources/src/socket/tcp.rs | 20 +-
.../apps/indexer/sources/src/socket/udp.rs | 16 +-
application/apps/protocol/Cargo.lock | 3 +-
.../apps/rustcore/rs-bindings/Cargo.lock | 57 +--
19 files changed, 390 insertions(+), 459 deletions(-)
diff --git a/application/apps/indexer/Cargo.lock b/application/apps/indexer/Cargo.lock
index e350e876fb..e913d1af94 100644
--- a/application/apps/indexer/Cargo.lock
+++ b/application/apps/indexer/Cargo.lock
@@ -116,19 +116,20 @@ dependencies = [
[[package]]
name = "anstyle-wincon"
-version = "3.0.6"
+version = "3.0.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2109dbce0e72be3ec00bed26e6a7479ca384ad226efdd66db8fa2e3a38c83125"
+checksum = "ca3534e77181a9cc07539ad51f2141fe32f6c3ffd4df76db8ad92346b003ae4e"
dependencies = [
"anstyle",
+ "once_cell",
"windows-sys 0.59.0",
]
[[package]]
name = "anyhow"
-version = "1.0.93"
+version = "1.0.95"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4c95c10ba0b00a02636238b814946408b1322d5ac4760326e6fb8ec956d85775"
+checksum = "34ac096ce696dc2fcabef30516bb13c0a68a11d30131d3df6f04711467681b04"
[[package]]
name = "arrayref"
@@ -227,7 +228,7 @@ version = "3.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ff6e472cdea888a4bd64f342f09b3f50e1886d32afe8df3d663c01140b811b18"
dependencies = [
- "event-listener 5.3.1",
+ "event-listener 5.4.0",
"event-listener-strategy",
"pin-project-lite",
]
@@ -245,7 +246,7 @@ dependencies = [
"async-task",
"blocking",
"cfg-if",
- "event-listener 5.3.1",
+ "event-listener 5.4.0",
"futures-lite",
"rustix",
"tracing",
@@ -315,7 +316,7 @@ checksum = "c7c24de15d275a1ecfd47a380fb4d5ec9bfe0933f309ed5e705b775596a3574d"
dependencies = [
"proc-macro2",
"quote",
- "syn 2.0.90",
+ "syn 2.0.96",
]
[[package]]
@@ -326,13 +327,13 @@ checksum = "8b75356056920673b02621b35afd0f7dda9306d03c79a30f5c56c44cf256e3de"
[[package]]
name = "async-trait"
-version = "0.1.83"
+version = "0.1.85"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "721cae7de5c34fbb2acd27e21e6d2cf7b886dce0c27388d46c4e6c47ea4318dd"
+checksum = "3f934833b4b7233644e5848f235df3f57ed8c80f1528a26c3dfa13d2147fa056"
dependencies = [
"proc-macro2",
"quote",
- "syn 2.0.90",
+ "syn 2.0.96",
]
[[package]]
@@ -384,18 +385,18 @@ dependencies = [
[[package]]
name = "bit-set"
-version = "0.5.3"
+version = "0.8.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0700ddab506f33b20a03b13996eccd309a48e5ff77d0d95926aa0210fb4e95f1"
+checksum = "08807e080ed7f9d5433fa9b275196cfc35414f66a0c79d864dc51a0d825231a3"
dependencies = [
"bit-vec",
]
[[package]]
name = "bit-vec"
-version = "0.6.3"
+version = "0.8.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "349f9b6a179ed607305526ca489b34ad0a41aed5f7980fa90eb03160b69598fb"
+checksum = "5e764a1d40d510daf35e07be9eb06e75770908c27d411ee6c92109c9840eaaf7"
[[package]]
name = "bitflags"
@@ -405,9 +406,9 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a"
[[package]]
name = "bitflags"
-version = "2.6.0"
+version = "2.8.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b048fb63fd8b5923fc5aa7b340d8e156aec7ec02f0c78fa8a6ddc2613f6f71de"
+checksum = "8f68f53c83ab957f72c32642f3868eec03eb974d1fb82e453128456482613d36"
[[package]]
name = "blake3"
@@ -437,9 +438,9 @@ dependencies = [
[[package]]
name = "bstr"
-version = "1.11.0"
+version = "1.11.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1a68f1f47cdf0ec8ee4b941b2eee2a80cb796db73118c0dd09ac63fbe405be22"
+checksum = "531a9155a481e2ee699d4f98f43c0ca4ff8ee1bfd55c31e9e98fb29d2b176fe0"
dependencies = [
"memchr",
"regex-automata 0.4.9",
@@ -447,25 +448,9 @@ dependencies = [
]
[[package]]
-name = "buf_redux"
-version = "0.8.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b953a6887648bb07a535631f2bc00fbdb2a2216f135552cb3f534ed136b9c07f"
-dependencies = [
- "memchr",
- "safemem",
- "slice-deque",
-]
-
-[[package]]
-name = "buf_redux"
-version = "0.8.4"
-source = "git+https://github.com/DmitryAstafyev/buf_redux.git#595d13446d3d90eb4834a3cee67c0f79e28f01d8"
-dependencies = [
- "memchr",
- "safemem",
- "slice-deque",
-]
+name = "bufread"
+version = "0.1.0"
+source = "git+https://github.com/kruss/bufread#5dec80376c0cbbbf3c4f864cf88b965cceb67c3b"
[[package]]
name = "bumpalo"
@@ -499,9 +484,9 @@ checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5"
[[package]]
name = "cc"
-version = "1.2.2"
+version = "1.2.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f34d93e62b03caf570cccc334cbc6c2fceca82f39211051345108adcba3eebdc"
+checksum = "c8293772165d9345bdaaa39b45b2109591e63fe5e6fbc23c6ff930a048aa310b"
dependencies = [
"shlex",
]
@@ -520,9 +505,9 @@ checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724"
[[package]]
name = "chrono"
-version = "0.4.38"
+version = "0.4.39"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a21f936df1771bf62b77f047b726c4625ff2e8aa607c01ec06e5a05bd8463401"
+checksum = "7e36cc9d416881d2e24f9a963be5fb1cd90966419ac844274161d10488b3e825"
dependencies = [
"android-tzdata",
"iana-time-zone",
@@ -603,18 +588,18 @@ dependencies = [
[[package]]
name = "clap"
-version = "4.5.21"
+version = "4.5.26"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fb3b4b9e5a7c7514dfa52869339ee98b3156b0bfb4e8a77c4ff4babb64b1604f"
+checksum = "a8eb5e908ef3a6efbe1ed62520fb7287959888c88485abe072543190ecc66783"
dependencies = [
"clap_builder",
]
[[package]]
name = "clap_builder"
-version = "4.5.21"
+version = "4.5.26"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b17a95aa67cc7b5ebd32aa5370189aa0d79069ef1c64ce893bd30fb24bff20ec"
+checksum = "96b01801b5fc6a0a232407abc821660c9c6d25a1cafc0d4f85f29fb8d9afc121"
dependencies = [
"anstyle",
"clap_lex",
@@ -622,9 +607,9 @@ dependencies = [
[[package]]
name = "clap_lex"
-version = "0.7.3"
+version = "0.7.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "afb84c814227b90d6895e01398aee0d8033c00e7466aca416fb6a8e0eb19d8a7"
+checksum = "f46ad14479a25103f283c0f10005961cf086d8dc42205bb44c46ac563475dca6"
[[package]]
name = "clipboard-win"
@@ -652,15 +637,15 @@ dependencies = [
[[package]]
name = "console"
-version = "0.15.8"
+version = "0.15.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0e1f83fc076bd6dd27517eacdf25fef6c4dfe5f1d7448bafaaf3a26f13b5e4eb"
+checksum = "ea3c6ecd8059b57859df5c69830340ed3c41d30e3da0c1cbed90a96ac853041b"
dependencies = [
"encode_unicode",
- "lazy_static",
"libc",
- "unicode-width 0.1.14",
- "windows-sys 0.52.0",
+ "once_cell",
+ "unicode-width 0.2.0",
+ "windows-sys 0.59.0",
]
[[package]]
@@ -700,7 +685,7 @@ dependencies = [
"anes",
"cast",
"ciborium",
- "clap 4.5.21",
+ "clap 4.5.26",
"criterion-plot",
"futures",
"is-terminal",
@@ -731,18 +716,18 @@ dependencies = [
[[package]]
name = "crossbeam-channel"
-version = "0.5.13"
+version = "0.5.14"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "33480d6946193aa8033910124896ca395333cae7e2d1113d1fef6c3272217df2"
+checksum = "06ba6d68e24814cb8de6bb986db8222d3a027d15872cabc0d18817bc3c0e4471"
dependencies = [
"crossbeam-utils",
]
[[package]]
name = "crossbeam-deque"
-version = "0.8.5"
+version = "0.8.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "613f8cc01fe9cf1a3eb3d7f488fd2fa8388403e97039e2f73692932e291a770d"
+checksum = "9dd111b7b7f7d55b72c0a6ae361660ee5853c9af73f70c3c2ef6858b950e2e51"
dependencies = [
"crossbeam-epoch",
"crossbeam-utils",
@@ -759,9 +744,9 @@ dependencies = [
[[package]]
name = "crossbeam-utils"
-version = "0.8.20"
+version = "0.8.21"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "22ec99545bb0ed0ea7bb9b8e1e9122ea386ff8a48c0922e43f36d45ab09e0e80"
+checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28"
[[package]]
name = "crunchy"
@@ -786,7 +771,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "32a2785755761f3ddc1492979ce1e48d2c00d09311c39e4466429188f3dd6501"
dependencies = [
"quote",
- "syn 2.0.90",
+ "syn 2.0.96",
]
[[package]]
@@ -819,7 +804,7 @@ dependencies = [
"proc-macro2",
"quote",
"strsim 0.11.1",
- "syn 2.0.90",
+ "syn 2.0.96",
]
[[package]]
@@ -830,7 +815,7 @@ checksum = "d336a2a514f6ccccaa3e09b02d41d35330c07ddf03a62165fcec10bb561c7806"
dependencies = [
"darling_core",
"quote",
- "syn 2.0.90",
+ "syn 2.0.96",
]
[[package]]
@@ -851,7 +836,7 @@ dependencies = [
"darling",
"proc-macro2",
"quote",
- "syn 2.0.90",
+ "syn 2.0.96",
]
[[package]]
@@ -861,7 +846,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ab63b0e2bf4d5928aff72e83a7dace85d7bba5fe12dcc3c5a572d78caffd3f3c"
dependencies = [
"derive_builder_core",
- "syn 2.0.90",
+ "syn 2.0.96",
]
[[package]]
@@ -874,7 +859,7 @@ dependencies = [
"proc-macro2",
"quote",
"rustc_version",
- "syn 2.0.90",
+ "syn 2.0.96",
]
[[package]]
@@ -907,10 +892,9 @@ dependencies = [
[[package]]
name = "dlt-core"
version = "0.18.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0b304e32f1164b8c2ef1dc746b32d321f25f88a32672f0f5bcba2df0f70a3b70"
+source = "git+https://github.com/kruss/dlt-core.git?branch=remove_buf_redux#87c7cea8be5224790ba8457ff531b4b6804c1298"
dependencies = [
- "buf_redux 0.8.4 (registry+https://github.com/rust-lang/crates.io-index)",
+ "bufread",
"byteorder",
"bytes",
"derive_more",
@@ -948,9 +932,9 @@ checksum = "60b1af1c220855b6ceac025d3f6ecdd2b7c4894bfe9cd9bda4fbb4bc7c0d4cf0"
[[package]]
name = "encode_unicode"
-version = "0.3.6"
+version = "1.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a357d28ed41a50f9c765dbfe56cbc04a64e53e5fc58ba79fbc34c10ef3df831f"
+checksum = "34aa73646ffb006b8f5147f3dc182bd4bcb190227ce861fc4a4844bf8e3cb2c0"
[[package]]
name = "encoding_rs"
@@ -978,9 +962,9 @@ checksum = "c34f04666d835ff5d62e058c3995147c06f42fe86ff053337632bca83e42702d"
[[package]]
name = "env_filter"
-version = "0.1.2"
+version = "0.1.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4f2c92ceda6ceec50f43169f9ee8424fe2db276791afde7b2cd8bc084cb376ab"
+checksum = "186e05a59d4c50738528153b83b0b0194d3a29507dfec16eccd4b342903397d0"
dependencies = [
"log",
"regex",
@@ -988,9 +972,9 @@ dependencies = [
[[package]]
name = "env_logger"
-version = "0.11.5"
+version = "0.11.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e13fa619b91fb2381732789fc5de83b45675e882f66623b7d8cb4f643017018d"
+checksum = "dcaee3d8e3cfc3fd92428d477bc97fc29ec8716d180c0d74c643bb26166660e0"
dependencies = [
"anstream",
"anstyle",
@@ -1050,9 +1034,9 @@ checksum = "0206175f82b8d6bf6652ff7d71a1e27fd2e4efde587fd368662814d6ec1d9ce0"
[[package]]
name = "event-listener"
-version = "5.3.1"
+version = "5.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6032be9bd27023a771701cc49f9f053c751055f71efb2e0ae5c15809093675ba"
+checksum = "3492acde4c3fc54c845eaab3eed8bd00c7a7d881f78bfc801e43a93dec1331ae"
dependencies = [
"concurrent-queue",
"parking",
@@ -1061,11 +1045,11 @@ dependencies = [
[[package]]
name = "event-listener-strategy"
-version = "0.5.2"
+version = "0.5.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0f214dc438f977e6d4e3500aaa277f5ad94ca83fbbd9b1a15713ce2344ccc5a1"
+checksum = "3c3e4e0dd3673c1139bf041f3008816d9cf2946bbfac2945c09e523b8d7b05b2"
dependencies = [
- "event-listener 5.3.1",
+ "event-listener 5.4.0",
"pin-project-lite",
]
@@ -1075,14 +1059,14 @@ version = "0.1.0"
dependencies = [
"proc-macro2",
"quote",
- "syn 2.0.90",
+ "syn 2.0.96",
]
[[package]]
name = "fastrand"
-version = "2.2.0"
+version = "2.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "486f806e73c5707928240ddc295403b1b93c96a02038563881c4a2fd84b81ac4"
+checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be"
[[package]]
name = "fd-lock"
@@ -1192,9 +1176,9 @@ checksum = "9e5c1b78ca4aae1ac06c48a526a655760685149f0d465d21f37abfe57ce075c6"
[[package]]
name = "futures-lite"
-version = "2.5.0"
+version = "2.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "cef40d21ae2c515b51041df9ed313ed21e572df340ea58a922a0aefe7e8891a1"
+checksum = "f5edaec856126859abb19ed65f39e90fea3a9574b9707f13539acf4abf7eb532"
dependencies = [
"fastrand",
"futures-core",
@@ -1211,7 +1195,7 @@ checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650"
dependencies = [
"proc-macro2",
"quote",
- "syn 2.0.90",
+ "syn 2.0.96",
]
[[package]]
@@ -1257,13 +1241,13 @@ dependencies = [
[[package]]
name = "ghost"
-version = "0.1.17"
+version = "0.1.18"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b0e085ded9f1267c32176b40921b9754c474f7dd96f7e808d4a982e48aa1e854"
+checksum = "39b697dbd8bfcc35d0ee91698aaa379af096368ba8837d279cc097b276edda45"
dependencies = [
"proc-macro2",
"quote",
- "syn 2.0.90",
+ "syn 2.0.96",
]
[[package]]
@@ -1363,11 +1347,11 @@ checksum = "fbf6a919d6cf397374f7dfeeea91d974c7c0a7221d0d0f4f20d859d329e53fcc"
[[package]]
name = "home"
-version = "0.5.9"
+version = "0.5.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e3d1354bf6b7235cb4a0576c2619fd4ed18183f689b12b006a0ee7329eeff9a5"
+checksum = "589533453244b0995c858700322199b2becb13b627df2851f64a2775d024abcf"
dependencies = [
- "windows-sys 0.52.0",
+ "windows-sys 0.59.0",
]
[[package]]
@@ -1411,7 +1395,7 @@ version = "0.1.0"
dependencies = [
"log",
"serde",
- "thiserror 2.0.3",
+ "thiserror 2.0.11",
]
[[package]]
@@ -1458,13 +1442,13 @@ dependencies = [
[[package]]
name = "insta"
-version = "1.41.1"
+version = "1.42.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7e9ffc4d4892617c50a928c52b2961cb5174b6fc6ebf252b2fac9d21955c48b8"
+checksum = "6513e4067e16e69ed1db5ab56048ed65db32d10ba5fc1217f5393f8f17d8b5a5"
dependencies = [
"console",
- "lazy_static",
"linked-hash-map",
+ "once_cell",
"serde",
"similar",
]
@@ -1544,9 +1528,9 @@ checksum = "d75a2a4b1b190afb6f5425f10f6a8f959d2ea0b9c2b1d79553551850539e4674"
[[package]]
name = "js-sys"
-version = "0.3.74"
+version = "0.3.77"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a865e038f7f6ed956f788f0d7d60c541fff74c7bd74272c5d4cf15c63743e705"
+checksum = "1cfaf33c695fc6e08064efbc1f72ec937429614f25eef83af942d0e227c3a28f"
dependencies = [
"once_cell",
"wasm-bindgen",
@@ -1569,15 +1553,9 @@ checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe"
[[package]]
name = "libc"
-version = "0.2.167"
+version = "0.2.169"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "09d6582e104315a817dff97f75133544b2e094ee22447d2acf4a74e189ba06fc"
-
-[[package]]
-name = "libm"
-version = "0.2.11"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8355be11b20d696c8f18f6cc018c4e372165b1fa8126cef092399c9951984ffa"
+checksum = "b5aba8db14291edd000dfcc4d620c7ebfb122c613afb886ca8803fa4e128a20a"
[[package]]
name = "libmimalloc-sys"
@@ -1595,7 +1573,7 @@ version = "0.1.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c0ff37bd590ca25063e35af745c343cb7a0271906fb7b37e4813e8f79f00268d"
dependencies = [
- "bitflags 2.6.0",
+ "bitflags 2.8.0",
"libc",
]
@@ -1627,9 +1605,9 @@ checksum = "0717cef1bc8b636c6e1c1bbdefc09e6322da8a9321966e8928ef80d20f7f770f"
[[package]]
name = "linux-raw-sys"
-version = "0.4.14"
+version = "0.4.15"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "78b3ae25bc7c8c38cec158d1f2757ee79e9b3740fbc7ccf0e59e4b08d793fa89"
+checksum = "d26c52dbd32dccf2d10cac7725f8eae5296885fb5703b261f7d0a0739ec807ab"
[[package]]
name = "lock_api"
@@ -1643,22 +1621,13 @@ dependencies = [
[[package]]
name = "log"
-version = "0.4.22"
+version = "0.4.25"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a7a70ba024b9dc04c27ea2f0c0548feb474ec5c54bba33a7f72f873a39d07b24"
+checksum = "04cbf5b083de1c7e0222a7a51dbfdba1cbe1c6ab0b15e29fff3f6c077fd9cd9f"
dependencies = [
"value-bag",
]
-[[package]]
-name = "mach"
-version = "0.2.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "86dd2487cdfea56def77b88438a2c915fb45113c5319bfe7e14306ca4cd0b0e1"
-dependencies = [
- "libc",
-]
-
[[package]]
name = "mach2"
version = "0.4.2"
@@ -1692,15 +1661,6 @@ dependencies = [
"libc",
]
-[[package]]
-name = "memoffset"
-version = "0.7.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5de893c32cde5f383baa4c04c5d6dbdd735cfd4a794b0debdb2bb1b421da5ff4"
-dependencies = [
- "autocfg",
-]
-
[[package]]
name = "merging"
version = "0.1.0"
@@ -1708,7 +1668,7 @@ dependencies = [
"log",
"serde",
"serde_json",
- "thiserror 2.0.3",
+ "thiserror 2.0.11",
]
[[package]]
@@ -1744,25 +1704,13 @@ checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a"
[[package]]
name = "miniz_oxide"
-version = "0.8.0"
+version = "0.8.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e2d80299ef12ff69b16a84bb182e3b9df68b5a91574d3d4fa6e41b65deec4df1"
+checksum = "b8402cab7aefae129c6977bb0ff1b8fd9a04eb5b51efc50a70bea51cda0c7924"
dependencies = [
"adler2",
]
-[[package]]
-name = "mio"
-version = "0.8.11"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a4a650543ca06a924e8b371db273b2756685faae30f8487da1b56505a8f78b0c"
-dependencies = [
- "libc",
- "log",
- "wasi",
- "windows-sys 0.48.0",
-]
-
[[package]]
name = "mio"
version = "1.0.3"
@@ -1770,19 +1718,20 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2886843bf800fba2e3377cff24abf6379b4c4d5c6681eaf9ea5b0d15090450bd"
dependencies = [
"libc",
+ "log",
"wasi",
"windows-sys 0.52.0",
]
[[package]]
name = "mio-serial"
-version = "5.0.5"
+version = "5.0.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "20a4c60ca5c9c0e114b3bd66ff4aa5f9b2b175442be51ca6c4365d687a97a2ac"
+checksum = "029e1f407e261176a983a6599c084efd322d9301028055c87174beac71397ba3"
dependencies = [
"log",
- "mio 0.8.11",
- "nix 0.26.4",
+ "mio",
+ "nix 0.29.0",
"serialport",
"winapi",
]
@@ -1805,8 +1754,6 @@ dependencies = [
"bitflags 1.3.2",
"cfg-if",
"libc",
- "memoffset",
- "pin-utils",
]
[[package]]
@@ -1815,7 +1762,7 @@ version = "0.29.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "71e2746dc3a24dd78b3cfcb7be93368c6de9963d30f43a6a73998a9cf4b17b46"
dependencies = [
- "bitflags 2.6.0",
+ "bitflags 2.8.0",
"cfg-if",
"cfg_aliases",
"libc",
@@ -1919,7 +1866,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841"
dependencies = [
"autocfg",
- "libm",
]
[[package]]
@@ -1930,9 +1876,9 @@ checksum = "830b246a0e5f20af87141b25c173cd1b609bd7779a4617d6ec582abaf90870f3"
[[package]]
name = "object"
-version = "0.36.5"
+version = "0.36.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "aedf0a2d09c573ed1d8d85b30c119153926a2b36dce0ab28322c09a117a4683e"
+checksum = "62948e14d923ea95ea2c7c86c71013138b66525b86bdc08d2dcc262bdb497b87"
dependencies = [
"memchr",
]
@@ -2018,7 +1964,7 @@ dependencies = [
"someip-payload",
"someip-tools",
"stringreader",
- "thiserror 2.0.3",
+ "thiserror 2.0.11",
]
[[package]]
@@ -2040,18 +1986,18 @@ dependencies = [
[[package]]
name = "phf"
-version = "0.11.2"
+version = "0.11.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ade2d8b8f33c7333b51bcf0428d37e217e9f32192ae4772156f65063b8ce03dc"
+checksum = "1fd6780a80ae0c52cc120a26a1a42c1ae51b247a253e4e06113d23d2c2edd078"
dependencies = [
"phf_shared",
]
[[package]]
name = "phf_codegen"
-version = "0.11.2"
+version = "0.11.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e8d39688d359e6b34654d328e262234662d16cc0f60ec8dcbe5e718709342a5a"
+checksum = "aef8048c789fa5e851558d709946d6d79a8ff88c0440c587967f8e94bfb1216a"
dependencies = [
"phf_generator",
"phf_shared",
@@ -2059,9 +2005,9 @@ dependencies = [
[[package]]
name = "phf_generator"
-version = "0.11.2"
+version = "0.11.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "48e4cc64c2ad9ebe670cb8fd69dd50ae301650392e81c05f9bfcb2d5bdbc24b0"
+checksum = "3c80231409c20246a13fddb31776fb942c38553c51e871f8cbd687a4cfb5843d"
dependencies = [
"phf_shared",
"rand",
@@ -2069,18 +2015,18 @@ dependencies = [
[[package]]
name = "phf_shared"
-version = "0.11.2"
+version = "0.11.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "90fcb95eef784c2ac79119d1dd819e162b5da872ce6f3c3abe1e8ca1c082f72b"
+checksum = "67eabc2ef2a60eb7faa00097bd1ffdb5bd28e62bf39990626a582201b7a754e5"
dependencies = [
"siphasher",
]
[[package]]
name = "pin-project-lite"
-version = "0.2.15"
+version = "0.2.16"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "915a1e146535de9163f3987b8944ed8cf49a18bb0056bcebcdcece385cece4ff"
+checksum = "3b3cff922bd51709b605d9ead9aa71031d81447142d828eb4a6eba76fe619f9b"
[[package]]
name = "pin-utils"
@@ -2199,9 +2145,9 @@ dependencies = [
[[package]]
name = "proc-macro2"
-version = "1.0.92"
+version = "1.0.93"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "37d3544b3f2748c54e147655edb5025752e2303145b5aefb3c3ea2c78b973bb0"
+checksum = "60946a68e5f9d28b0dc1c21bb8a97ee7d018a8b322fa57838ba31cc878e22d99"
dependencies = [
"unicode-ident",
]
@@ -2211,7 +2157,7 @@ name = "processor"
version = "0.2.0"
dependencies = [
"bincode",
- "buf_redux 0.8.4 (git+https://github.com/DmitryAstafyev/buf_redux.git)",
+ "bufread",
"bytecount",
"criterion",
"futures",
@@ -2229,20 +2175,20 @@ dependencies = [
"serde_json",
"stypes",
"tempfile",
- "thiserror 2.0.3",
+ "thiserror 2.0.11",
"tokio-util",
"uuid",
]
[[package]]
name = "proptest"
-version = "1.5.0"
+version = "1.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b4c2511913b88df1637da85cc8d96ec8e43a3f8bb8ccb71ee1ac240d6f3df58d"
+checksum = "14cae93065090804185d3b75f0bf93b8eeda30c7a9b4a33d3bdb3988d6229e50"
dependencies = [
"bit-set",
"bit-vec",
- "bitflags 2.6.0",
+ "bitflags 2.8.0",
"lazy_static",
"num-traits",
"rand",
@@ -2280,9 +2226,9 @@ dependencies = [
[[package]]
name = "quote"
-version = "1.0.37"
+version = "1.0.38"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b5b9d34b8991d19d98081b46eacdd8eb58c6f2b201139f7c5f643cc155a633af"
+checksum = "0e4dccaaaf89514f546c693ddc140f729f958c247918a13380cccc6078391acc"
dependencies = [
"proc-macro2",
]
@@ -2358,11 +2304,11 @@ dependencies = [
[[package]]
name = "redox_syscall"
-version = "0.5.7"
+version = "0.5.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9b6dfecf2c74bce2466cabf93f6664d6998a69eb21e39f4207930065b27b771f"
+checksum = "03a862b389f93e68874fbf580b9de08dd02facb9a788ebadaf4a3fd33cf58834"
dependencies = [
- "bitflags 2.6.0",
+ "bitflags 2.8.0",
]
[[package]]
@@ -2472,17 +2418,23 @@ dependencies = [
[[package]]
name = "rustix"
-version = "0.38.41"
+version = "0.38.43"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d7f649912bc1495e167a6edee79151c84b1bad49748cb4f1f1167f459f6224f6"
+checksum = "a78891ee6bf2340288408954ac787aa063d8e8817e9f53abb37c695c6d834ef6"
dependencies = [
- "bitflags 2.6.0",
+ "bitflags 2.8.0",
"errno",
"libc",
"linux-raw-sys",
- "windows-sys 0.52.0",
+ "windows-sys 0.59.0",
]
+[[package]]
+name = "rustversion"
+version = "1.0.19"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f7c45b9784283f1b2e7fb61b42047c2fd678ef0960d4f6f1eba131594cc369d4"
+
[[package]]
name = "rusty-fork"
version = "0.3.0"
@@ -2501,7 +2453,7 @@ version = "15.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2ee1e066dc922e513bda599c6ccb5f3bb2b0ea5870a579448f2622993f0a9a2f"
dependencies = [
- "bitflags 2.6.0",
+ "bitflags 2.8.0",
"cfg-if",
"clipboard-win",
"fd-lock",
@@ -2523,12 +2475,6 @@ version = "1.0.18"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f3cb5ba0dc43242ce17de99c180e96db90b235b8a9fdc9543c96d2209116bd9f"
-[[package]]
-name = "safemem"
-version = "0.3.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ef703b7cb59335eae2eb93ceb664c0eb7ea6bf567079d843e09420219668e072"
-
[[package]]
name = "same-file"
version = "1.0.6"
@@ -2546,35 +2492,35 @@ checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49"
[[package]]
name = "semver"
-version = "1.0.23"
+version = "1.0.24"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "61697e0a1c7e512e84a621326239844a24d8207b4669b41bc18b32ea5cbf988b"
+checksum = "3cb6eb87a131f756572d7fb904f6e7b68633f09cca868c5df1c4b8d1a694bbba"
[[package]]
name = "serde"
-version = "1.0.215"
+version = "1.0.217"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6513c1ad0b11a9376da888e3e0baa0077f1aed55c17f50e7b2397136129fb88f"
+checksum = "02fc4265df13d6fa1d00ecff087228cc0a2b5f3c0e87e258d8b94a156e984c70"
dependencies = [
"serde_derive",
]
[[package]]
name = "serde_derive"
-version = "1.0.215"
+version = "1.0.217"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ad1e866f866923f252f05c889987993144fb74e722403468a4ebd70c3cd756c0"
+checksum = "5a9bf7cf98d04a2b28aead066b7496853d4779c9cc183c440dbac457641e19a0"
dependencies = [
"proc-macro2",
"quote",
- "syn 2.0.90",
+ "syn 2.0.96",
]
[[package]]
name = "serde_json"
-version = "1.0.133"
+version = "1.0.135"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c7fceb2473b9166b2294ef05efcb65a3db80803f0b03ef86a5fc88a2b85ee377"
+checksum = "2b0d7ba2887406110130a978386c4e1befb98c674b4fba677954e4db976630d9"
dependencies = [
"itoa",
"memchr",
@@ -2584,11 +2530,11 @@ dependencies = [
[[package]]
name = "serialport"
-version = "4.6.1"
+version = "4.7.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "779e2977f0cc2ff39708fef48f96f3768ac8ddd8c6caaaab82e83bd240ef99b2"
+checksum = "5ecfc4858c2266c7695d8b8460bbd612fa81bd2e250f5f0dd16195e4b4f8b3d8"
dependencies = [
- "bitflags 2.6.0",
+ "bitflags 2.8.0",
"cfg-if",
"core-foundation",
"core-foundation-sys",
@@ -2627,7 +2573,7 @@ dependencies = [
"sources",
"stypes",
"tempfile",
- "thiserror 2.0.3",
+ "thiserror 2.0.11",
"tokio",
"tokio-stream",
"tokio-util",
@@ -2676,9 +2622,9 @@ checksum = "1de1d4f81173b03af4c0cbed3c898f6bff5b870e4a7f5d6f4057d62a7a4b686e"
[[package]]
name = "siphasher"
-version = "0.3.11"
+version = "1.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "38b58827f4464d87d377d175e90bf58eb00fd8716ff0a62f80356b5e61555d0d"
+checksum = "56199f7ddabf13fe5074ce809e7d3f42b42ae711800501b5b16ea82ad029c39d"
[[package]]
name = "slab"
@@ -2689,17 +2635,6 @@ dependencies = [
"autocfg",
]
-[[package]]
-name = "slice-deque"
-version = "0.2.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ffddf594f5f597f63533d897427a570dbaa9feabaaa06595b74b71b7014507d7"
-dependencies = [
- "libc",
- "mach",
- "winapi",
-]
-
[[package]]
name = "smallvec"
version = "1.13.2"
@@ -2745,7 +2680,7 @@ name = "someip-tools"
version = "0.1.0"
dependencies = [
"nom",
- "thiserror 2.0.3",
+ "thiserror 2.0.11",
]
[[package]]
@@ -2753,7 +2688,7 @@ name = "sources"
version = "0.1.0"
dependencies = [
"async-stream",
- "buf_redux 0.8.4 (registry+https://github.com/rust-lang/crates.io-index)",
+ "bufread",
"bytes",
"criterion",
"env_logger",
@@ -2769,7 +2704,7 @@ dependencies = [
"serde",
"shellexpand",
"stypes",
- "thiserror 2.0.3",
+ "thiserror 2.0.11",
"tikv-jemallocator",
"tokio",
"tokio-serial",
@@ -2846,7 +2781,7 @@ dependencies = [
"regex",
"remove_dir_all",
"serde",
- "thiserror 2.0.3",
+ "thiserror 2.0.11",
"tokio",
"ts-rs",
"uuid",
@@ -2866,9 +2801,9 @@ dependencies = [
[[package]]
name = "syn"
-version = "2.0.90"
+version = "2.0.96"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "919d3b74a5dd0ccd15aeb8f93e7006bd9e14c295087c9896a110f490752bcf31"
+checksum = "d5d0adab1ae378d7f53bdebc67a39f1f151407ef230f0ce2883572f5d8985c80"
dependencies = [
"proc-macro2",
"quote",
@@ -2877,12 +2812,13 @@ dependencies = [
[[package]]
name = "tempfile"
-version = "3.14.0"
+version = "3.15.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "28cce251fcbc87fac86a866eeb0d6c2d536fc16d06f184bb61aeae11aa4cee0c"
+checksum = "9a8a559c81686f576e8cd0290cd2a24a2a9ad80c98b3478856500fcbd7acd704"
dependencies = [
"cfg-if",
"fastrand",
+ "getrandom",
"once_cell",
"rustix",
"windows-sys 0.59.0",
@@ -2901,12 +2837,12 @@ dependencies = [
name = "text_grep"
version = "0.1.0"
dependencies = [
- "buf_redux 0.8.4 (registry+https://github.com/rust-lang/crates.io-index)",
+ "bufread",
"grep-regex",
"grep-searcher",
"regex",
"tempfile",
- "thiserror 2.0.3",
+ "thiserror 2.0.11",
"tokio",
"tokio-util",
]
@@ -2931,11 +2867,11 @@ dependencies = [
[[package]]
name = "thiserror"
-version = "2.0.3"
+version = "2.0.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c006c85c7651b3cf2ada4584faa36773bd07bac24acfb39f3c431b36d7e667aa"
+checksum = "d452f284b73e6d76dd36758a0c8684b1d5be31f92b89d07fd5822175732206fc"
dependencies = [
- "thiserror-impl 2.0.3",
+ "thiserror-impl 2.0.11",
]
[[package]]
@@ -2946,18 +2882,18 @@ checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1"
dependencies = [
"proc-macro2",
"quote",
- "syn 2.0.90",
+ "syn 2.0.96",
]
[[package]]
name = "thiserror-impl"
-version = "2.0.3"
+version = "2.0.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f077553d607adc1caf65430528a576c757a71ed73944b66ebb58ef2bbd243568"
+checksum = "26afc1baea8a989337eeb52b6e72a039780ce45c3edfcc9c5b9d112feeb173c2"
dependencies = [
"proc-macro2",
"quote",
- "syn 2.0.90",
+ "syn 2.0.96",
]
[[package]]
@@ -3002,14 +2938,14 @@ dependencies = [
[[package]]
name = "tokio"
-version = "1.41.1"
+version = "1.43.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "22cfb5bee7a6a52939ca9224d6ac897bb669134078daa8735560897f69de4d33"
+checksum = "3d61fa4ffa3de412bfea335c6ecff681de2b609ba3c77ef3e00e521813a9ed9e"
dependencies = [
"backtrace",
"bytes",
"libc",
- "mio 1.0.3",
+ "mio",
"parking_lot",
"pin-project-lite",
"signal-hook-registry",
@@ -3020,33 +2956,34 @@ dependencies = [
[[package]]
name = "tokio-macros"
-version = "2.4.0"
+version = "2.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "693d596312e88961bc67d7f1f97af8a70227d9f90c31bba5806eec004978d752"
+checksum = "6e06d43f1345a3bcd39f6a56dbb7dcab2ba47e68e8ac134855e7e2bdbaf8cab8"
dependencies = [
"proc-macro2",
"quote",
- "syn 2.0.90",
+ "syn 2.0.96",
]
[[package]]
name = "tokio-serial"
-version = "5.4.4"
+version = "5.4.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "aa6e2e4cf0520a99c5f87d5abb24172b5bd220de57c3181baaaa5440540c64aa"
+checksum = "aa1d5427f11ba7c5e6384521cfd76f2d64572ff29f3f4f7aa0f496282923fdc8"
dependencies = [
"cfg-if",
"futures",
"log",
"mio-serial",
+ "serialport",
"tokio",
]
[[package]]
name = "tokio-stream"
-version = "0.1.16"
+version = "0.1.17"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4f4e6ce100d0eb49a2734f8c0812bcd324cf357d21810932c5df6b96ef2b86f1"
+checksum = "eca58d7bba4a75707817a2c44174253f9236b2d5fbd055602e9d5c07c139a047"
dependencies = [
"futures-core",
"pin-project-lite",
@@ -3055,9 +2992,9 @@ dependencies = [
[[package]]
name = "tokio-util"
-version = "0.7.12"
+version = "0.7.13"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "61e7c3654c13bcd040d4a03abee2c75b1d14a37b423cf5a813ceae1cc903ec6a"
+checksum = "d7fcaa8d55a2bdd6b83ace262b016eca0d79ee02818c5c1bcdf0305114081078"
dependencies = [
"bytes",
"futures-core",
@@ -3089,7 +3026,7 @@ checksum = "395ae124c09f9e6918a2310af6038fba074bcf474ac352496d5910dd59a2226d"
dependencies = [
"proc-macro2",
"quote",
- "syn 2.0.90",
+ "syn 2.0.96",
]
[[package]]
@@ -3115,9 +3052,9 @@ dependencies = [
[[package]]
name = "tracing-subscriber"
-version = "0.3.18"
+version = "0.3.19"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ad0f048c97dbd9faa9b7df56362b8ebcaa52adb06b498c050d2f4e32f90a7a8b"
+checksum = "e8189decb5ac0fa7bc8b96b7cb9b2701d60d48805aca84a238004d665fcc4008"
dependencies = [
"matchers",
"nu-ansi-term",
@@ -3138,7 +3075,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e640d9b0964e9d39df633548591090ab92f7a4567bc31d3891af23471a3365c6"
dependencies = [
"lazy_static",
- "thiserror 2.0.3",
+ "thiserror 2.0.11",
"ts-rs-macros",
"uuid",
]
@@ -3151,7 +3088,7 @@ checksum = "0e9d8656589772eeec2cf7a8264d9cda40fb28b9bc53118ceb9e8c07f8f38730"
dependencies = [
"proc-macro2",
"quote",
- "syn 2.0.90",
+ "syn 2.0.96",
"termcolor",
]
@@ -3172,9 +3109,9 @@ dependencies = [
[[package]]
name = "unicase"
-version = "2.8.0"
+version = "2.8.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7e51b68083f157f853b6379db119d1c1be0e6e4dec98101079dec41f6f5cf6df"
+checksum = "75b844d17643ee918803943289730bec8aac480150456169e647ed0b576ba539"
[[package]]
name = "unicode-ident"
@@ -3208,9 +3145,9 @@ checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821"
[[package]]
name = "uuid"
-version = "1.11.0"
+version = "1.12.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f8c5f0a0af699448548ad1a2fbf920fb4bee257eae39953ba95cb84891a0446a"
+checksum = "744018581f9a3454a9e15beb8a33b017183f1e7c0cd170232a2d1453b23a51c4"
dependencies = [
"getrandom",
"serde",
@@ -3284,47 +3221,48 @@ checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423"
[[package]]
name = "wasm-bindgen"
-version = "0.2.97"
+version = "0.2.100"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d15e63b4482863c109d70a7b8706c1e364eb6ea449b201a76c5b89cedcec2d5c"
+checksum = "1edc8929d7499fc4e8f0be2262a241556cfc54a0bea223790e71446f2aab1ef5"
dependencies = [
"cfg-if",
"once_cell",
+ "rustversion",
"wasm-bindgen-macro",
]
[[package]]
name = "wasm-bindgen-backend"
-version = "0.2.97"
+version = "0.2.100"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8d36ef12e3aaca16ddd3f67922bc63e48e953f126de60bd33ccc0101ef9998cd"
+checksum = "2f0a0651a5c2bc21487bde11ee802ccaf4c51935d0d3d42a6101f98161700bc6"
dependencies = [
"bumpalo",
"log",
- "once_cell",
"proc-macro2",
"quote",
- "syn 2.0.90",
+ "syn 2.0.96",
"wasm-bindgen-shared",
]
[[package]]
name = "wasm-bindgen-futures"
-version = "0.4.45"
+version = "0.4.50"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "cc7ec4f8827a71586374db3e87abdb5a2bb3a15afed140221307c3ec06b1f63b"
+checksum = "555d470ec0bc3bb57890405e5d4322cc9ea83cebb085523ced7be4144dac1e61"
dependencies = [
"cfg-if",
"js-sys",
+ "once_cell",
"wasm-bindgen",
"web-sys",
]
[[package]]
name = "wasm-bindgen-macro"
-version = "0.2.97"
+version = "0.2.100"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "705440e08b42d3e4b36de7d66c944be628d579796b8090bfa3471478a2260051"
+checksum = "7fe63fc6d09ed3792bd0897b314f53de8e16568c2b3f7982f468c0bf9bd0b407"
dependencies = [
"quote",
"wasm-bindgen-macro-support",
@@ -3332,28 +3270,31 @@ dependencies = [
[[package]]
name = "wasm-bindgen-macro-support"
-version = "0.2.97"
+version = "0.2.100"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "98c9ae5a76e46f4deecd0f0255cc223cfa18dc9b261213b8aa0c7b36f61b3f1d"
+checksum = "8ae87ea40c9f689fc23f209965b6fb8a99ad69aeeb0231408be24920604395de"
dependencies = [
"proc-macro2",
"quote",
- "syn 2.0.90",
+ "syn 2.0.96",
"wasm-bindgen-backend",
"wasm-bindgen-shared",
]
[[package]]
name = "wasm-bindgen-shared"
-version = "0.2.97"
+version = "0.2.100"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6ee99da9c5ba11bd675621338ef6fa52296b76b83305e9b6e5c77d4c286d6d49"
+checksum = "1a05d73b933a847d6cccdda8f838a22ff101ad9bf93e33684f39c1f5f0eece3d"
+dependencies = [
+ "unicode-ident",
+]
[[package]]
name = "web-sys"
-version = "0.3.74"
+version = "0.3.77"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a98bc3c33f0fe7e59ad7cd041b89034fa82a7c2d4365ca538dda6cdaf513863c"
+checksum = "33b6dd2ef9186f1f2072e409e99cd22a975331a6b3591b12c764e0e55c60d5d2"
dependencies = [
"js-sys",
"wasm-bindgen",
@@ -3581,5 +3522,5 @@ checksum = "fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e"
dependencies = [
"proc-macro2",
"quote",
- "syn 2.0.90",
+ "syn 2.0.96",
]
diff --git a/application/apps/indexer/Cargo.toml b/application/apps/indexer/Cargo.toml
index 02852d740a..8bd797d89e 100644
--- a/application/apps/indexer/Cargo.toml
+++ b/application/apps/indexer/Cargo.toml
@@ -24,11 +24,12 @@ thiserror = "2.0"
lazy_static = "1.5"
tokio = { version = "1", features = ["full"] }
tokio-stream = "0.1"
-dlt-core = "0.18.0"
+#dlt-core = "0.18.0"
+dlt-core = { git = "https://github.com/kruss/dlt-core.git", branch = "remove_buf_redux" }
crossbeam-channel = "0.5"
futures = "0.3"
tokio-util = "0.7"
-buf_redux = "0.8"
+bufread = { git = "https://github.com/kruss/bufread" }
regex = "1"
grep-regex = "0.1"
rand = "0.8"
diff --git a/application/apps/indexer/addons/text_grep/Cargo.toml b/application/apps/indexer/addons/text_grep/Cargo.toml
index 23847f116d..903cfcf353 100644
--- a/application/apps/indexer/addons/text_grep/Cargo.toml
+++ b/application/apps/indexer/addons/text_grep/Cargo.toml
@@ -11,7 +11,7 @@ path = "src/lib.rs"
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
[dependencies]
tokio = { workspace = true, features = ["full"] }
-buf_redux.workspace = true
+bufread.workspace = true
tokio-util.workspace = true
tempfile.workspace = true
grep-searcher.workspace = true
diff --git a/application/apps/indexer/addons/text_grep/src/buffer.rs b/application/apps/indexer/addons/text_grep/src/buffer.rs
index e8930aca58..13ce4083d4 100644
--- a/application/apps/indexer/addons/text_grep/src/buffer.rs
+++ b/application/apps/indexer/addons/text_grep/src/buffer.rs
@@ -1,39 +1,62 @@
-use buf_redux::{
- do_read,
- policy::{DoRead, ReaderPolicy},
- Buffer,
-};
+// TODO this duplicates: application/apps/indexer/processor/src/search/buffer.rs
+use bufread::BufReader;
use tokio_util::sync::CancellationToken;
+use std::io::{BufRead, Read, Result, Seek, SeekFrom, Error, ErrorKind};
-pub const REDUX_READER_CAPACITY: usize = 1024 * 1024;
-pub const REDUX_MIN_BUFFER_SPACE: usize = 10 * 1024;
+const BIN_READER_CAPACITY: usize = 1024 * 1024;
+const BIN_MIN_BUFFER_SPACE: usize = 10 * 1024;
-#[derive(Debug)]
-pub struct CancallableMinBuffered(pub (usize, CancellationToken));
+pub struct CancellableBufReader<R> {
+ buffer: BufReader<R>,
+ cancel: CancellationToken
+}
-impl CancallableMinBuffered {
- /// Set the number of bytes to ensure are in the buffer.
- pub fn set_min(&mut self, min: usize) {
- self.0 .0 = min;
+impl<R> CancellableBufReader<R> {
+ pub fn new(reader: R, cancel: CancellationToken) -> Self {
+ CancellableBufReader {
+ buffer: BufReader::new(
+ BIN_READER_CAPACITY,
+ BIN_MIN_BUFFER_SPACE,
+ reader),
+ cancel
+ }
}
}
-impl ReaderPolicy for CancallableMinBuffered {
- fn before_read(&mut self, buffer: &mut Buffer) -> DoRead {
- // do nothing if we have enough data
- if buffer.len() >= self.0 .0 {
- do_read!(false)
+impl<R: Read> Read for CancellableBufReader<R> {
+ fn read(&mut self, buffer: &mut [u8]) -> Result<usize> {
+ if self.cancel.is_cancelled() {
+ return Ok(0);
}
+
+ self.buffer.read(buffer)
+ }
+}
- let cap = buffer.capacity();
+impl<R: Read> BufRead for CancellableBufReader<R> {
+ fn fill_buf(&mut self) -> Result<&[u8]> {
+ if self.cancel.is_cancelled() {
+ return Ok(&[][..]);
+ }
+
+ self.buffer.fill_buf()
+ }
- // if there's enough room but some of it's stuck after the head
- if buffer.usable_space() < self.0 .0 && buffer.free_space() >= self.0 .0 {
- buffer.make_room();
- } else if cap < self.0 .0 {
- buffer.reserve(self.0 .0 - cap);
+ fn consume(&mut self, size: usize) {
+ if self.cancel.is_cancelled() {
+ return;
}
- DoRead(true)
+ self.buffer.consume(size)
}
}
+
+impl<R: Seek> Seek for CancellableBufReader<R> {
+ fn seek(&mut self, pos: SeekFrom) -> Result<u64> {
+ if self.cancel.is_cancelled() {
+ return Err(Error::from(ErrorKind::NotFound));
+ }
+
+ self.buffer.seek(pos)
+ }
+}
\ No newline at end of file
diff --git a/application/apps/indexer/addons/text_grep/src/lib.rs b/application/apps/indexer/addons/text_grep/src/lib.rs
index 831181b805..7047c2fb27 100644
--- a/application/apps/indexer/addons/text_grep/src/lib.rs
+++ b/application/apps/indexer/addons/text_grep/src/lib.rs
@@ -65,7 +65,6 @@
//!
//! # Dependencies
//!
-//! - `buf_redux`: Provides buffered I/O functionality.
//! - `grep_regex`: Facilitates regular expression searching.
//! - `grep_searcher`: Implements file searching capabilities.
//! - `regex`: Provides regular expression support.
@@ -81,8 +80,7 @@
//! - Both case-sensitive and case-insensitive searches are supported based on user preference.
pub mod buffer;
-use crate::buffer::{CancallableMinBuffered, REDUX_MIN_BUFFER_SPACE, REDUX_READER_CAPACITY};
-use buf_redux::BufReader;
+use crate::buffer::CancellableBufReader;
use grep_regex::{RegexMatcher, RegexMatcherBuilder};
use grep_searcher::{sinks::UTF8, Searcher};
use regex::Regex;
@@ -214,9 +212,7 @@ fn process_file(
) -> Result<SearchResult, GrepError> {
let mut pattern_counts = HashMap::new();
let file = File::open(file_path)?;
- let reader = BufReader::with_capacity(REDUX_READER_CAPACITY, file).set_policy(
- CancallableMinBuffered((REDUX_MIN_BUFFER_SPACE, cancel_token.clone())),
- );
+ let reader = CancellableBufReader::new(file, cancel_token.clone());
let mut searcher = Searcher::new();
searcher
.search_reader(
diff --git a/application/apps/indexer/processor/Cargo.toml b/application/apps/indexer/processor/Cargo.toml
index 8c1c812df7..dd0de53467 100644
--- a/application/apps/indexer/processor/Cargo.toml
+++ b/application/apps/indexer/processor/Cargo.toml
@@ -6,7 +6,7 @@ edition = "2021"
[dependencies]
bincode = "1.3"
-buf_redux = { git = "https://github.com/DmitryAstafyev/buf_redux.git" }
+bufread.workspace = true
bytecount = "0.6"
futures.workspace = true
grep-regex.workspace = true
diff --git a/application/apps/indexer/processor/src/search/buffer.rs b/application/apps/indexer/processor/src/search/buffer.rs
index b43ea9c79f..f3cd2e1b30 100644
--- a/application/apps/indexer/processor/src/search/buffer.rs
+++ b/application/apps/indexer/processor/src/search/buffer.rs
@@ -1,43 +1,62 @@
-use buf_redux::{
- do_read,
- policy::{DoRead, ReaderPolicy},
- Buffer,
-};
+// TODO this duplicates: application/apps/indexer/addons/text_grep/src/buffer.rs
+use bufread::BufReader;
use tokio_util::sync::CancellationToken;
+use std::io::{BufRead, Read, Result, Seek, SeekFrom, Error, ErrorKind};
-pub const REDUX_READER_CAPACITY: usize = 1024 * 1024;
-pub const REDUX_MIN_BUFFER_SPACE: usize = 10 * 1024;
+const BIN_READER_CAPACITY: usize = 1024 * 1024;
+const BIN_MIN_BUFFER_SPACE: usize = 10 * 1024;
-#[derive(Debug)]
-pub struct CancallableMinBuffered(pub (usize, CancellationToken));
+pub struct CancellableBufReader<R> {
+ buffer: BufReader<R>,
+ cancel: CancellationToken
+}
-impl CancallableMinBuffered {
- /// Set the number of bytes to ensure are in the buffer.
- pub fn set_min(&mut self, min: usize) {
- self.0 .0 = min;
+impl<R> CancellableBufReader<R> {
+ pub fn new(reader: R, cancel: CancellationToken) -> Self {
+ CancellableBufReader {
+ buffer: BufReader::new(
+ BIN_READER_CAPACITY,
+ BIN_MIN_BUFFER_SPACE,
+ reader),
+ cancel
+ }
}
}
-impl ReaderPolicy for CancallableMinBuffered {
- fn before_read(&mut self, buffer: &mut Buffer) -> DoRead {
- // do nothing if we have enough data
- if buffer.len() >= self.0 .0 {
- do_read!(false)
+impl<R: Read> Read for CancellableBufReader<R> {
+ fn read(&mut self, buffer: &mut [u8]) -> Result<usize> {
+ if self.cancel.is_cancelled() {
+ return Ok(0);
}
+
+ self.buffer.read(buffer)
+ }
+}
- let cap = buffer.capacity();
+impl<R: Read> BufRead for CancellableBufReader<R> {
+ fn fill_buf(&mut self) -> Result<&[u8]> {
+ if self.cancel.is_cancelled() {
+ return Ok(&[][..]);
+ }
+
+ self.buffer.fill_buf()
+ }
- // if there's enough room but some of it's stuck after the head
- if buffer.usable_space() < self.0 .0 && buffer.free_space() >= self.0 .0 {
- buffer.make_room();
- } else if cap < self.0 .0 {
- buffer.reserve(self.0 .0 - cap);
+ fn consume(&mut self, size: usize) {
+ if self.cancel.is_cancelled() {
+ return;
}
- DoRead(true)
+ self.buffer.consume(size)
}
+}
- fn is_paused(&mut self) -> bool {
- self.0 .1.is_cancelled()
+impl<R: Seek> Seek for CancellableBufReader<R> {
+ fn seek(&mut self, pos: SeekFrom) -> Result<u64> {
+ if self.cancel.is_cancelled() {
+ return Err(Error::from(ErrorKind::NotFound));
+ }
+
+ self.buffer.seek(pos)
}
-}
+}
\ No newline at end of file
diff --git a/application/apps/indexer/processor/src/search/searchers/mod.rs b/application/apps/indexer/processor/src/search/searchers/mod.rs
index f1339e2e95..737bb6b5c5 100644
--- a/application/apps/indexer/processor/src/search/searchers/mod.rs
+++ b/application/apps/indexer/processor/src/search/searchers/mod.rs
@@ -1,11 +1,10 @@
use crate::{
grabber::GrabError,
search::{
- buffer::{CancallableMinBuffered, REDUX_MIN_BUFFER_SPACE, REDUX_READER_CAPACITY},
+ buffer::CancellableBufReader,
error::SearchError,
},
};
-use buf_redux::BufReader as ReduxReader;
use grep_regex::RegexMatcher;
use grep_searcher::{sinks::UTF8, Searcher};
use std::{
@@ -68,7 +67,7 @@ impl<State: SearchState> BaseSearcher<State> {
&mut self,
rows_count: u64,
read_bytes: u64,
- cancallation: CancellationToken,
+ cancel_token: CancellationToken,
mut f: F,
) -> Result<Range<usize>, SearchError>
where
@@ -101,10 +100,7 @@ impl<State: SearchState> BaseSearcher<State> {
let in_file = File::open(&self.file_path).map_err(|_| {
GrabError::IoOperation(format!("Could not open file {:?}", self.file_path))
})?;
- let mut in_file_reader =
- ReduxReader::with_capacity(REDUX_READER_CAPACITY, in_file).set_policy(
- CancallableMinBuffered((REDUX_MIN_BUFFER_SPACE, cancallation)),
- );
+ let mut in_file_reader = CancellableBufReader::new(in_file, cancel_token);
in_file_reader
.seek(SeekFrom::Start(self.bytes_read))
.map_err(|_| {
diff --git a/application/apps/indexer/processor/src/text_source.rs b/application/apps/indexer/processor/src/text_source.rs
index afcbb2ff4d..6bc39c61e9 100644
--- a/application/apps/indexer/processor/src/text_source.rs
+++ b/application/apps/indexer/processor/src/text_source.rs
@@ -2,7 +2,7 @@ use crate::grabber::{
identify_byte_range, ByteRange, ComputationResult, FilePart, GrabError, GrabMetadata,
LineRange, Slot,
};
-use buf_redux::{policy::MinBuffered, BufReader as ReduxReader};
+use bufread::BufReader;
use log::{debug, error};
use std::{
fs,
@@ -12,8 +12,8 @@ use std::{
};
use tokio_util::sync::CancellationToken;
-const REDUX_READER_CAPACITY: usize = 1024 * 32;
-const REDUX_MIN_BUFFER_SPACE: usize = 10 * 1024;
+const BIN_READER_CAPACITY: usize = 1024 * 32;
+const BIN_MIN_BUFFER_SPACE: usize = 10 * 1024;
#[derive(Debug)]
pub struct TextFileSource {
@@ -135,8 +135,7 @@ impl TextFileSource {
&self.path, byte_offset
))
})?;
- let mut reader = ReduxReader::with_capacity(REDUX_READER_CAPACITY, f)
- .set_policy(MinBuffered(REDUX_MIN_BUFFER_SPACE));
+ let mut reader = BufReader::new(BIN_READER_CAPACITY, BIN_MIN_BUFFER_SPACE, f);
let mut pending: Option<Slot> = None;
loop {
if let Some(shutdown_token) = &shutdown_token {
diff --git a/application/apps/indexer/sources/Cargo.toml b/application/apps/indexer/sources/Cargo.toml
index aeeb5b7b0b..bf1a74a1dd 100644
--- a/application/apps/indexer/sources/Cargo.toml
+++ b/application/apps/indexer/sources/Cargo.toml
@@ -6,7 +6,7 @@ edition = "2021"
[dependencies]
async-stream = "0.3"
-buf_redux.workspace = true
+bufread.workspace = true
bytes = "1.3"
etherparse = "0.16"
futures.workspace = true
diff --git a/application/apps/indexer/sources/src/binary/pcap/legacy.rs b/application/apps/indexer/sources/src/binary/pcap/legacy.rs
index fdb45c8a19..9ac0b20788 100644
--- a/application/apps/indexer/sources/src/binary/pcap/legacy.rs
+++ b/application/apps/indexer/sources/src/binary/pcap/legacy.rs
@@ -2,7 +2,7 @@ use crate::{
binary::pcap::debug_block, ByteSource, Error as SourceError, ReloadInfo, SourceFilter,
TransportProtocol,
};
-use buf_redux::Buffer;
+use bufread::DeqBuffer;
use etherparse::{SlicedPacket, TransportSlice};
use log::{debug, error, trace};
use pcap_parser::{traits::PcapReaderIterator, LegacyPcapReader, PcapBlockOwned, PcapError};
@@ -10,7 +10,7 @@ use std::io::Read;
pub struct PcapLegacyByteSource<R: Read> {
pcap_reader: LegacyPcapReader<R>,
- buffer: Buffer,
+ buffer: DeqBuffer,
last_know_timestamp: Option<u64>,
total: usize,
}
@@ -20,7 +20,7 @@ impl<R: Read> PcapLegacyByteSource<R> {
Ok(Self {
pcap_reader: LegacyPcapReader::new(65536, reader)
.map_err(|e| SourceError::Setup(format!("{e}")))?,
- buffer: Buffer::new(),
+ buffer: DeqBuffer::new(8192),
last_know_timestamp: None,
total: 0,
})
@@ -104,8 +104,8 @@ impl<R: Read + Send + Sync> ByteSource for PcapLegacyByteSource<R> {
}),
) => {
let actual_tp: TransportProtocol = actual.into();
- let received_bytes = self.buffer.copy_from_slice(payload);
- let available_bytes = self.buffer.len();
+ let received_bytes = self.buffer.write_from(payload);
+ let available_bytes = self.buffer.read_available();
if actual_tp == *wanted {
Ok(Some(ReloadInfo::new(
received_bytes,
@@ -123,8 +123,8 @@ impl<R: Read + Send + Sync> ByteSource for PcapLegacyByteSource<R> {
}
}
_ => {
- let copied = self.buffer.copy_from_slice(payload);
- let available_bytes = self.buffer.len();
+ let copied = self.buffer.write_from(payload);
+ let available_bytes = self.buffer.read_available();
Ok(Some(ReloadInfo::new(
copied,
available_bytes,
@@ -145,15 +145,15 @@ impl<R: Read + Send + Sync> ByteSource for PcapLegacyByteSource<R> {
}
fn current_slice(&self) -> &[u8] {
- self.buffer.buf()
+ self.buffer.read_slice()
}
fn consume(&mut self, offset: usize) {
- self.buffer.consume(offset);
+ self.buffer.read_done(offset);
}
fn len(&self) -> usize {
- self.buffer.len()
+ self.buffer.read_available()
}
}
diff --git a/application/apps/indexer/sources/src/binary/pcap/ng.rs b/application/apps/indexer/sources/src/binary/pcap/ng.rs
index 027ba043bb..ea4eca85ee 100644
--- a/application/apps/indexer/sources/src/binary/pcap/ng.rs
+++ b/application/apps/indexer/sources/src/binary/pcap/ng.rs
@@ -2,7 +2,7 @@ use crate::{
binary::pcap::debug_block, ByteSource, Error as SourceError, ReloadInfo, SourceFilter,
TransportProtocol,
};
-use buf_redux::Buffer;
+use bufread::DeqBuffer;
use etherparse::{SlicedPacket, TransportSlice};
use log::{debug, error, trace};
use pcap_parser::{traits::PcapReaderIterator, PcapBlockOwned, PcapError, PcapNGReader};
@@ -10,7 +10,7 @@ use std::io::Read;
pub struct PcapngByteSource<R: Read> {
pcapng_reader: PcapNGReader<R>,
- buffer: Buffer,
+ buffer: DeqBuffer,
last_know_timestamp: Option<u64>,
total: usize,
}
@@ -20,7 +20,7 @@ impl<R: Read> PcapngByteSource<R> {
Ok(Self {
pcapng_reader: PcapNGReader::new(65536, reader)
.map_err(|e| SourceError::Setup(format!("{e}")))?,
- buffer: Buffer::new(),
+ buffer: DeqBuffer::new(8192),
last_know_timestamp: None,
total: 0,
})
@@ -108,8 +108,8 @@ impl<R: Read + Send + Sync> ByteSource for PcapngByteSource<R> {
}),
) => {
let actual_tp: TransportProtocol = actual.into();
- let received_bytes = self.buffer.copy_from_slice(payload);
- let available_bytes = self.buffer.len();
+ let received_bytes = self.buffer.write_from(payload);
+ let available_bytes = self.buffer.read_available();
if actual_tp == *wanted {
Ok(Some(ReloadInfo::new(
received_bytes,
@@ -127,8 +127,8 @@ impl<R: Read + Send + Sync> ByteSource for PcapngByteSource<R> {
}
}
_ => {
- let copied = self.buffer.copy_from_slice(payload);
- let available_bytes = self.buffer.len();
+ let copied = self.buffer.write_from(payload);
+ let available_bytes = self.buffer.read_available();
Ok(Some(ReloadInfo::new(
copied,
available_bytes,
@@ -149,15 +149,15 @@ impl<R: Read + Send + Sync> ByteSource for PcapngByteSource<R> {
}
fn current_slice(&self) -> &[u8] {
- self.buffer.buf()
+ self.buffer.read_slice()
}
fn consume(&mut self, offset: usize) {
- self.buffer.consume(offset);
+ self.buffer.read_done(offset);
}
fn len(&self) -> usize {
- self.buffer.len()
+ self.buffer.read_available()
}
}
diff --git a/application/apps/indexer/sources/src/binary/raw.rs b/application/apps/indexer/sources/src/binary/raw.rs
index c6ca5779cd..164cc85fcf 100644
--- a/application/apps/indexer/sources/src/binary/raw.rs
+++ b/application/apps/indexer/sources/src/binary/raw.rs
@@ -2,14 +2,14 @@ use crate::{
ByteSource, Error as SourceError, ReloadInfo, SourceFilter, DEFAULT_MIN_BUFFER_SPACE,
DEFAULT_READER_CAPACITY,
};
-use buf_redux::{policy::MinBuffered, BufReader as ReduxReader};
+use bufread::BufReader;
use std::io::{BufRead, Read, Seek};
pub struct BinaryByteSource<R>
where
R: Read + Seek,
{
- reader: ReduxReader<R, MinBuffered>,
+ reader: BufReader<R>,
}
impl<R> BinaryByteSource<R>
@@ -18,8 +18,7 @@ where
{
/// create a new `BinaryByteSource` with default buffer settings for reading
pub fn new(input: R) -> BinaryByteSource<R> {
- let reader = ReduxReader::with_capacity(DEFAULT_READER_CAPACITY, input)
- .set_policy(MinBuffered(DEFAULT_MIN_BUFFER_SPACE));
+ let reader = BufReader::new(DEFAULT_READER_CAPACITY, DEFAULT_MIN_BUFFER_SPACE, input);
BinaryByteSource { reader }
}
@@ -27,15 +26,14 @@ where
/// the `total_capacity` specifies how big the underlying used buffers should be at least
/// the `min_space` will make sure that the buffer is filled with at least that many bytes
pub fn custom(input: R, total_capacity: usize, min_space: usize) -> BinaryByteSource<R> {
- let reader =
- ReduxReader::with_capacity(total_capacity, input).set_policy(MinBuffered(min_space));
+ let reader = BufReader::new(total_capacity, min_space, input);
BinaryByteSource { reader }
}
}
impl<R: Read + Send + Sync + Seek> ByteSource for BinaryByteSource<R> {
async fn load(&mut self, _: Option<&SourceFilter>) -> Result<Option<ReloadInfo>, SourceError> {
- let initial_buf_len = self.reader.buf_len();
+ let initial_buf_len = self.reader.len();
trace!(
"before: capacity: {} (buf_len: {})",
self.reader.capacity(),
@@ -75,7 +73,7 @@ impl<R: Read + Send + Sync + Seek> ByteSource for BinaryByteSource<R> {
}
fn len(&self) -> usize {
- self.reader.buf_len()
+ self.reader.len()
}
}
diff --git a/application/apps/indexer/sources/src/command/process.rs b/application/apps/indexer/sources/src/command/process.rs
index 759b0e1818..187852c4bd 100644
--- a/application/apps/indexer/sources/src/command/process.rs
+++ b/application/apps/indexer/sources/src/command/process.rs
@@ -1,5 +1,5 @@
use crate::{ByteSource, Error as SourceError, ReloadInfo, SourceFilter};
-use buf_redux::Buffer;
+use bufread::DeqBuffer;
use regex::{Captures, Regex};
use shellexpand::tilde;
use std::{collections::HashMap, ffi::OsString, path::PathBuf, process::Stdio};
@@ -30,7 +30,7 @@ pub enum ProcessError {
pub struct ProcessSource {
process: Child,
- buffer: Buffer,
+ buffer: DeqBuffer,
stdout: FramedRead<ChildStdout, LinesCodec>,
stderr: FramedRead<ChildStderr, LinesCodec>,
stdin: ChildStdin,
@@ -144,7 +144,7 @@ impl ProcessSource {
.ok_or_else(|| ProcessError::Setup(String::from("Fail to get stdin handle")))?;
Ok(Self {
process,
- buffer: Buffer::new(),
+ buffer: DeqBuffer::new(8192),
stdout,
stderr,
stdin,
@@ -183,9 +183,9 @@ impl ByteSource for ProcessSource {
}
if let Some(Ok(line)) = output {
let stored = line.len() + 1;
- self.buffer.copy_from_slice(line.as_bytes());
- self.buffer.copy_from_slice(b"\n");
- let available_bytes = self.buffer.len();
+ self.buffer.write_from(line.as_bytes());
+ self.buffer.write_from(b"\n");
+ let available_bytes = self.buffer.read_available();
Ok(Some(ReloadInfo::new(stored, available_bytes, 0, None)))
} else if let Some(Err(err)) = output {
Err(SourceError::Unrecoverable(format!("{err}")))
@@ -195,15 +195,15 @@ impl ByteSource for ProcessSource {
}
fn current_slice(&self) -> &[u8] {
- self.buffer.buf()
+ self.buffer.read_slice()
}
fn consume(&mut self, offset: usize) {
- self.buffer.consume(offset);
+ self.buffer.read_done(offset);
}
fn len(&self) -> usize {
- self.buffer.len()
+ self.buffer.read_available()
}
fn is_empty(&self) -> bool {
diff --git a/application/apps/indexer/sources/src/serial/serialport.rs b/application/apps/indexer/sources/src/serial/serialport.rs
index b5f9b8a20f..3dc1cf5d62 100644
--- a/application/apps/indexer/sources/src/serial/serialport.rs
+++ b/application/apps/indexer/sources/src/serial/serialport.rs
@@ -1,5 +1,5 @@
use crate::{ByteSource, Error as SourceError, ReloadInfo, SourceFilter};
-use buf_redux::Buffer;
+use bufread::DeqBuffer;
use bytes::{BufMut, BytesMut};
use futures::{
stream::{SplitSink, SplitStream, StreamExt},
@@ -79,7 +79,7 @@ fn stop_bits(stop_bits: &u8) -> StopBits {
pub struct SerialSource {
write_stream: SplitSink<Framed<SerialStream, LineCodec>, Vec<u8>>,
read_stream: SplitStream<Framed<SerialStream, LineCodec>>,
- buffer: Buffer,
+ buffer: DeqBuffer,
amount: usize,
send_data_delay: u8,
}
@@ -115,7 +115,7 @@ impl SerialSource {
Ok(Self {
write_stream,
read_stream,
- buffer: Buffer::new(),
+ buffer: DeqBuffer::new(8192),
amount: 0,
send_data_delay: config.send_data_delay,
})
@@ -141,7 +141,7 @@ impl ByteSource for SerialSource {
if self.amount == 0 {
return Ok(None);
}
- self.buffer.copy_from_slice(received.as_bytes());
+ self.buffer.write_from(received.as_bytes());
}
Err(err) => {
return Err(SourceError::Setup(format!("Failed to read stream: {err}")));
@@ -154,20 +154,20 @@ impl ByteSource for SerialSource {
}
}
- let available_bytes = self.buffer.len();
+ let available_bytes = self.buffer.read_available();
Ok(Some(ReloadInfo::new(self.amount, available_bytes, 0, None)))
}
fn current_slice(&self) -> &[u8] {
- self.buffer.buf()
+ self.buffer.read_slice()
}
fn consume(&mut self, offset: usize) {
- self.buffer.consume(offset);
+ self.buffer.read_done(offset);
}
fn len(&self) -> usize {
- self.buffer.len()
+ self.buffer.read_available()
}
fn is_empty(&self) -> bool {
diff --git a/application/apps/indexer/sources/src/socket/tcp.rs b/application/apps/indexer/sources/src/socket/tcp.rs
index 72f2afa75e..e6e381683d 100644
--- a/application/apps/indexer/sources/src/socket/tcp.rs
+++ b/application/apps/indexer/sources/src/socket/tcp.rs
@@ -1,13 +1,13 @@
use crate::{
socket::ReconnectStateMsg, ByteSource, Error as SourceError, ReloadInfo, SourceFilter,
};
-use buf_redux::Buffer;
+use bufread::DeqBuffer;
use tokio::{net::TcpStream, task::yield_now};
use super::{ReconnectInfo, ReconnectResult, ReconnectToServer};
pub struct TcpSource {
- buffer: Buffer,
+ buffer: DeqBuffer,
socket: TcpStream,
tmp_buffer: Vec<u8>,
binding_address: String,
@@ -23,8 +23,8 @@ impl TcpSource {
) -> Result<Self, std::io::Error> {
let binding_address: String = addr.into();
Ok(Self {
- buffer: Buffer::new(),
- socket: TcpStream::connect(&binding_address).await?,
+ buffer: DeqBuffer::new(8192),
+ socket: TcpStream::connect(binding_address).await?,
tmp_buffer: vec![0u8; MAX_DATAGRAM_SIZE],
binding_address,
reconnect_info,
@@ -114,8 +114,10 @@ impl ByteSource for TcpSource {
}
};
}
- self.buffer.copy_from_slice(&self.tmp_buffer[..len]);
- let available_bytes = self.buffer.len();
+ if len > 0 {
+ self.buffer.write_from(&self.tmp_buffer[..len]);
+ }
+ let available_bytes = self.buffer.read_available();
return Ok(Some(ReloadInfo::new(len, available_bytes, 0, None)));
}
Err(ref e) if e.kind() == std::io::ErrorKind::WouldBlock => {
@@ -149,15 +151,15 @@ impl ByteSource for TcpSource {
}
fn current_slice(&self) -> &[u8] {
- self.buffer.buf()
+ self.buffer.read_slice()
}
fn consume(&mut self, offset: usize) {
- self.buffer.consume(offset)
+ self.buffer.read_done(offset);
}
fn len(&self) -> usize {
- self.buffer.len()
+ self.buffer.read_available()
}
}
diff --git a/application/apps/indexer/sources/src/socket/udp.rs b/application/apps/indexer/sources/src/socket/udp.rs
index 6e32000d5b..9e4bfbe7dc 100644
--- a/application/apps/indexer/sources/src/socket/udp.rs
+++ b/application/apps/indexer/sources/src/socket/udp.rs
@@ -1,5 +1,5 @@
use crate::{ByteSource, Error as SourceError, ReloadInfo, SourceFilter};
-use buf_redux::Buffer;
+use bufread::DeqBuffer;
use log::trace;
use std::net::{IpAddr, Ipv4Addr};
use thiserror::Error;
@@ -20,7 +20,7 @@ pub enum UdpSourceError {
}
pub struct UdpSource {
- buffer: Buffer,
+ buffer: DeqBuffer,
socket: UdpSocket,
tmp_buffer: Vec<u8>,
}
@@ -66,7 +66,7 @@ impl UdpSource {
}
Ok(Self {
- buffer: Buffer::new(),
+ buffer: DeqBuffer::new(8192),
socket,
tmp_buffer: vec![0u8; MAX_DATAGRAM_SIZE],
})
@@ -91,23 +91,23 @@ impl ByteSource for UdpSource {
String::from_utf8_lossy(&self.tmp_buffer[..len])
);
if len > 0 {
- self.buffer.copy_from_slice(&self.tmp_buffer[..len]);
+ self.buffer.write_from(&self.tmp_buffer[..len]);
}
- let available_bytes = self.buffer.len();
+ let available_bytes = self.buffer.read_available();
Ok(Some(ReloadInfo::new(len, available_bytes, 0, None)))
}
fn current_slice(&self) -> &[u8] {
- self.buffer.buf()
+ self.buffer.read_slice()
}
fn consume(&mut self, offset: usize) {
- self.buffer.consume(offset)
+ self.buffer.read_done(offset);
}
fn len(&self) -> usize {
- self.buffer.len()
+ self.buffer.read_available()
}
}
diff --git a/application/apps/protocol/Cargo.lock b/application/apps/protocol/Cargo.lock
index e28ca0e757..251582e50c 100644
--- a/application/apps/protocol/Cargo.lock
+++ b/application/apps/protocol/Cargo.lock
@@ -66,8 +66,7 @@ dependencies = [
[[package]]
name = "dlt-core"
version = "0.18.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0b304e32f1164b8c2ef1dc746b32d321f25f88a32672f0f5bcba2df0f70a3b70"
+source = "git+https://github.com/kruss/dlt-core.git?branch=remove_buf_redux#87c7cea8be5224790ba8457ff531b4b6804c1298"
dependencies = [
"byteorder",
"bytes",
diff --git a/application/apps/rustcore/rs-bindings/Cargo.lock b/application/apps/rustcore/rs-bindings/Cargo.lock
index 96297c41eb..c122744999 100644
--- a/application/apps/rustcore/rs-bindings/Cargo.lock
+++ b/application/apps/rustcore/rs-bindings/Cargo.lock
@@ -345,25 +345,9 @@ dependencies = [
]
[[package]]
-name = "buf_redux"
-version = "0.8.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b953a6887648bb07a535631f2bc00fbdb2a2216f135552cb3f534ed136b9c07f"
-dependencies = [
- "memchr",
- "safemem",
- "slice-deque",
-]
-
-[[package]]
-name = "buf_redux"
-version = "0.8.4"
-source = "git+https://github.com/DmitryAstafyev/buf_redux.git#595d13446d3d90eb4834a3cee67c0f79e28f01d8"
-dependencies = [
- "memchr",
- "safemem",
- "slice-deque",
-]
+name = "bufread"
+version = "0.1.0"
+source = "git+https://github.com/kruss/bufread#5dec80376c0cbbbf3c4f864cf88b965cceb67c3b"
[[package]]
name = "bumpalo"
@@ -647,10 +631,9 @@ dependencies = [
[[package]]
name = "dlt-core"
version = "0.18.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0b304e32f1164b8c2ef1dc746b32d321f25f88a32672f0f5bcba2df0f70a3b70"
+source = "git+https://github.com/kruss/dlt-core.git?branch=remove_buf_redux#87c7cea8be5224790ba8457ff531b4b6804c1298"
dependencies = [
- "buf_redux 0.8.4 (registry+https://github.com/rust-lang/crates.io-index)",
+ "bufread",
"byteorder",
"bytes",
"derive_more",
@@ -1281,15 +1264,6 @@ dependencies = [
"winapi",
]
-[[package]]
-name = "mach"
-version = "0.2.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "86dd2487cdfea56def77b88438a2c915fb45113c5319bfe7e14306ca4cd0b0e1"
-dependencies = [
- "libc",
-]
-
[[package]]
name = "mach2"
version = "0.4.2"
@@ -1831,7 +1805,7 @@ name = "processor"
version = "0.2.0"
dependencies = [
"bincode",
- "buf_redux 0.8.4 (git+https://github.com/DmitryAstafyev/buf_redux.git)",
+ "bufread",
"bytecount",
"futures",
"grep-regex",
@@ -2026,12 +2000,6 @@ version = "1.0.18"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f3cb5ba0dc43242ce17de99c180e96db90b235b8a9fdc9543c96d2209116bd9f"
-[[package]]
-name = "safemem"
-version = "0.3.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ef703b7cb59335eae2eb93ceb664c0eb7ea6bf567079d843e09420219668e072"
-
[[package]]
name = "same-file"
version = "1.0.6"
@@ -2239,17 +2207,6 @@ dependencies = [
"autocfg",
]
-[[package]]
-name = "slice-deque"
-version = "0.2.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ffddf594f5f597f63533d897427a570dbaa9feabaaa06595b74b71b7014507d7"
-dependencies = [
- "libc",
- "mach",
- "winapi",
-]
-
[[package]]
name = "smallvec"
version = "1.13.2"
@@ -2303,7 +2260,7 @@ name = "sources"
version = "0.1.0"
dependencies = [
"async-stream",
- "buf_redux 0.8.4 (registry+https://github.com/rust-lang/crates.io-index)",
+ "bufread",
"bytes",
"etherparse",
"futures",
From b6b1a64fcd26fe0b79fb3b4d88d9772524a49332 Mon Sep 17 00:00:00 2001
From: "kevin.russ" <kevin.russ@accenture.com>
Date: Mon, 27 Jan 2025 11:05:05 +0100
Subject: [PATCH 2/2] Add bufread as an addon
---
application/apps/indexer/Cargo.lock | 50 ++-
application/apps/indexer/Cargo.toml | 6 +-
.../apps/indexer/addons/bufread/Cargo.toml | 19 +
.../apps/indexer/addons/bufread/README.md | 2 +
.../addons/bufread/benches/bench_config.rs | 11 +
.../addons/bufread/benches/buffer_benches.rs | 50 +++
.../addons/bufread/benches/reader_benches.rs | 34 ++
.../apps/indexer/addons/bufread/src/lib.rs | 403 ++++++++++++++++++
.../indexer/addons/bufread/tests/proto.rs | 138 ++++++
.../addons/bufread/tests/proto_tests.rs | 46 ++
.../addons/bufread/tests/reader_proptest.rs | 41 ++
.../apps/indexer/addons/text_grep/Cargo.toml | 2 +-
.../indexer/addons/text_grep/src/buffer.rs | 22 +-
application/apps/indexer/processor/Cargo.toml | 2 +-
.../indexer/processor/src/search/buffer.rs | 22 +-
.../processor/src/search/searchers/mod.rs | 5 +-
application/apps/indexer/sources/Cargo.toml | 2 +-
.../apps/indexer/sources/src/socket/tcp.rs | 2 +-
application/apps/indexer/stypes/Cargo.toml | 2 +-
application/apps/protocol/Cargo.lock | 3 +-
.../apps/rustcore/rs-bindings/Cargo.lock | 45 +-
21 files changed, 860 insertions(+), 47 deletions(-)
create mode 100644 application/apps/indexer/addons/bufread/Cargo.toml
create mode 100644 application/apps/indexer/addons/bufread/README.md
create mode 100644 application/apps/indexer/addons/bufread/benches/bench_config.rs
create mode 100644 application/apps/indexer/addons/bufread/benches/buffer_benches.rs
create mode 100644 application/apps/indexer/addons/bufread/benches/reader_benches.rs
create mode 100644 application/apps/indexer/addons/bufread/src/lib.rs
create mode 100644 application/apps/indexer/addons/bufread/tests/proto.rs
create mode 100644 application/apps/indexer/addons/bufread/tests/proto_tests.rs
create mode 100644 application/apps/indexer/addons/bufread/tests/reader_proptest.rs
diff --git a/application/apps/indexer/Cargo.lock b/application/apps/indexer/Cargo.lock
index e913d1af94..a8c5802ca5 100644
--- a/application/apps/indexer/Cargo.lock
+++ b/application/apps/indexer/Cargo.lock
@@ -447,10 +447,25 @@ dependencies = [
"serde",
]
+[[package]]
+name = "buf_redux"
+version = "0.8.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b953a6887648bb07a535631f2bc00fbdb2a2216f135552cb3f534ed136b9c07f"
+dependencies = [
+ "memchr",
+ "safemem",
+ "slice-deque",
+]
+
[[package]]
name = "bufread"
-version = "0.1.0"
-source = "git+https://github.com/kruss/bufread#5dec80376c0cbbbf3c4f864cf88b965cceb67c3b"
+version = "0.1.1"
+dependencies = [
+ "criterion",
+ "proptest",
+ "rand",
+]
[[package]]
name = "bumpalo"
@@ -892,9 +907,10 @@ dependencies = [
[[package]]
name = "dlt-core"
version = "0.18.0"
-source = "git+https://github.com/kruss/dlt-core.git?branch=remove_buf_redux#87c7cea8be5224790ba8457ff531b4b6804c1298"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0b304e32f1164b8c2ef1dc746b32d321f25f88a32672f0f5bcba2df0f70a3b70"
dependencies = [
- "bufread",
+ "buf_redux",
"byteorder",
"bytes",
"derive_more",
@@ -1628,6 +1644,15 @@ dependencies = [
"value-bag",
]
+[[package]]
+name = "mach"
+version = "0.2.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "86dd2487cdfea56def77b88438a2c915fb45113c5319bfe7e14306ca4cd0b0e1"
+dependencies = [
+ "libc",
+]
+
[[package]]
name = "mach2"
version = "0.4.2"
@@ -2475,6 +2500,12 @@ version = "1.0.18"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f3cb5ba0dc43242ce17de99c180e96db90b235b8a9fdc9543c96d2209116bd9f"
+[[package]]
+name = "safemem"
+version = "0.3.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ef703b7cb59335eae2eb93ceb664c0eb7ea6bf567079d843e09420219668e072"
+
[[package]]
name = "same-file"
version = "1.0.6"
@@ -2635,6 +2666,17 @@ dependencies = [
"autocfg",
]
+[[package]]
+name = "slice-deque"
+version = "0.2.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ffddf594f5f597f63533d897427a570dbaa9feabaaa06595b74b71b7014507d7"
+dependencies = [
+ "libc",
+ "mach",
+ "winapi",
+]
+
[[package]]
name = "smallvec"
version = "1.13.2"
diff --git a/application/apps/indexer/Cargo.toml b/application/apps/indexer/Cargo.toml
index 8bd797d89e..1b04260d23 100644
--- a/application/apps/indexer/Cargo.toml
+++ b/application/apps/indexer/Cargo.toml
@@ -6,6 +6,7 @@ members = [
"addons/someip-tools",
"addons/file-tools",
"addons/text_grep",
+ "addons/bufread",
"indexer_base",
"indexer_cli",
"merging",
@@ -24,12 +25,10 @@ thiserror = "2.0"
lazy_static = "1.5"
tokio = { version = "1", features = ["full"] }
tokio-stream = "0.1"
-#dlt-core = "0.18.0"
-dlt-core = { git = "https://github.com/kruss/dlt-core.git", branch = "remove_buf_redux" }
+dlt-core = "0.18.0"
crossbeam-channel = "0.5"
futures = "0.3"
tokio-util = "0.7"
-bufread = { git = "https://github.com/kruss/bufread" }
regex = "1"
grep-regex = "0.1"
rand = "0.8"
@@ -45,6 +44,7 @@ envvars = "0.1"
# Support for `html_reports` needs running the benchmarks via `cargo-criterion` tool.
criterion = { version = "0.5", features = ["html_reports"] }
insta = { version = "1.41", features = ["yaml"] }
+proptest = "1.6"
# `insta` crate and its dependency `similar` will be compiled once and run many times,
# therefore it's suggested to compile them with more optimizations for faster runs.
diff --git a/application/apps/indexer/addons/bufread/Cargo.toml b/application/apps/indexer/addons/bufread/Cargo.toml
new file mode 100644
index 0000000000..64cda961be
--- /dev/null
+++ b/application/apps/indexer/addons/bufread/Cargo.toml
@@ -0,0 +1,19 @@
+[package]
+name = "bufread"
+version = "0.1.1"
+edition = "2021"
+
+[dependencies]
+
+[dev-dependencies]
+criterion.workspace = true
+proptest.workspace = true
+rand.workspace = true
+
+[[bench]]
+name = "buffer_benches"
+harness = false
+
+[[bench]]
+name = "reader_benches"
+harness = false
diff --git a/application/apps/indexer/addons/bufread/README.md b/application/apps/indexer/addons/bufread/README.md
new file mode 100644
index 0000000000..901eec8c50
--- /dev/null
+++ b/application/apps/indexer/addons/bufread/README.md
@@ -0,0 +1,2 @@
+# bufread
+A buffered reader implementation in Rust with special behavior.
diff --git a/application/apps/indexer/addons/bufread/benches/bench_config.rs b/application/apps/indexer/addons/bufread/benches/bench_config.rs
new file mode 100644
index 0000000000..31bb35889d
--- /dev/null
+++ b/application/apps/indexer/addons/bufread/benches/bench_config.rs
@@ -0,0 +1,11 @@
+use criterion::Criterion;
+use std::time::Duration;
+
+pub fn bench_config(sample_size: usize) -> Criterion {
+ Criterion::default()
+ .warm_up_time(Duration::from_secs(5))
+ .measurement_time(Duration::from_secs(10))
+ .sample_size(sample_size)
+ .significance_level(0.01)
+ .noise_threshold(0.05)
+}
diff --git a/application/apps/indexer/addons/bufread/benches/buffer_benches.rs b/application/apps/indexer/addons/bufread/benches/buffer_benches.rs
new file mode 100644
index 0000000000..a21ee8c8aa
--- /dev/null
+++ b/application/apps/indexer/addons/bufread/benches/buffer_benches.rs
@@ -0,0 +1,50 @@
+use criterion::{criterion_group, criterion_main, Criterion};
+use std::hint::black_box;
+
+#[path = "./bench_config.rs"]
+mod bench_config;
+use bench_config::bench_config;
+
+use bufread::DeqBuffer;
+
+fn write_buffer(buffer: &mut DeqBuffer, src: &[u8]) {
+ while buffer.write_available() >= src.len() {
+ buffer.write_from(src);
+ }
+}
+
+fn write_buffer_benchmark(c: &mut Criterion) {
+ let mut buffer = DeqBuffer::new(50 * 1000);
+ let src: [u8; 100] = [1; 100];
+
+ c.bench_function("write_buffer", |b| {
+ b.iter(|| write_buffer(black_box(&mut buffer), black_box(&src)))
+ });
+}
+
+fn write_read_buffer(buffer: &mut DeqBuffer, src: &[u8], dst: &mut [u8]) {
+ while buffer.write_available() >= src.len() {
+ buffer.write_from(src);
+ }
+ while buffer.read_available() != 0 {
+ buffer.read_to(dst);
+ }
+}
+
+fn write_read_buffer_benchmark(c: &mut Criterion) {
+ let mut buffer = DeqBuffer::new(50 * 1000);
+ let src: [u8; 100] = [1; 100];
+ let mut dst: [u8; 100] = [1; 100];
+
+ c.bench_function("write_read_buffer", |b| {
+ b.iter(|| write_read_buffer(black_box(&mut buffer), black_box(&src), black_box(&mut dst)))
+ });
+}
+
+criterion_group! {
+ name = benches;
+ config = bench_config(1000);
+ targets = write_buffer_benchmark, write_read_buffer_benchmark
+}
+
+criterion_main!(benches);
diff --git a/application/apps/indexer/addons/bufread/benches/reader_benches.rs b/application/apps/indexer/addons/bufread/benches/reader_benches.rs
new file mode 100644
index 0000000000..16945cec50
--- /dev/null
+++ b/application/apps/indexer/addons/bufread/benches/reader_benches.rs
@@ -0,0 +1,34 @@
+#![allow(dead_code)]
+
+use criterion::{criterion_group, criterion_main, Criterion};
+use std::hint::black_box;
+
+#[path = "./bench_config.rs"]
+mod bench_config;
+use bench_config::bench_config;
+
+#[path = "../tests/proto.rs"]
+mod proto;
+
+use bufread::BufReader;
+use proto::{Parser, Source, MAX_PACKET_LEN};
+
+fn proto_benchmark(c: &mut Criterion) {
+ let source_min_size = 100 * MAX_PACKET_LEN;
+ let buffer_max_size = 3 * MAX_PACKET_LEN;
+ let buffer_min_size = MAX_PACKET_LEN;
+
+ let source = Source::fixed(source_min_size);
+ let reader = BufReader::new(buffer_max_size, buffer_min_size, source.data());
+ let mut parser = Parser::new(reader);
+
+ c.bench_function("proto", |b| b.iter(|| Parser::run(black_box(&mut parser))));
+}
+
+criterion_group! {
+ name = benches;
+ config = bench_config(1000);
+ targets = proto_benchmark
+}
+
+criterion_main!(benches);
diff --git a/application/apps/indexer/addons/bufread/src/lib.rs b/application/apps/indexer/addons/bufread/src/lib.rs
new file mode 100644
index 0000000000..bb468e6f9d
--- /dev/null
+++ b/application/apps/indexer/addons/bufread/src/lib.rs
@@ -0,0 +1,403 @@
+use std::{
+ cmp::min,
+ io::{BufRead, Read, Result, Seek, SeekFrom},
+ ptr::copy,
+};
+
+/// A buffered reader that could refill while still some remaining data is stored.
+pub struct BufReader<R> {
+ reader: R,
+ buffer: DeqBuffer,
+ min: usize,
+}
+
+impl<R> BufReader<R> {
+ /// Creates a new reader with the given maximum space and minimum bytes to be buffered ahead.
+ pub fn new(max: usize, min: usize, reader: R) -> Self {
+ BufReader {
+ reader,
+ buffer: DeqBuffer::new(max),
+ min,
+ }
+ }
+
+ /// Returns the total capacity of the inner buffer.
+ pub fn capacity(&self) -> usize {
+ self.buffer.capacity()
+ }
+
+ /// Returns the number of currently available bytes of the inner buffer.
+ pub fn len(&self) -> usize {
+ self.buffer.read_available()
+ }
+
+ /// Answers if the inner buffer is currently empty.
+ pub fn is_empty(&self) -> bool {
+ self.len() == 0
+ }
+
+ /// Returns the current slice to read from of the inner buffer.
+ pub fn buffer(&self) -> &[u8] {
+ self.buffer.read_slice()
+ }
+}
+
+impl<R: Read> Read for BufReader<R> {
+ fn read(&mut self, buffer: &mut [u8]) -> Result<usize> {
+ if self.buffer.write_available() < self.min {
+ self.buffer.flush();
+ }
+
+ let size = self.fill_buf()?.read(buffer)?;
+ self.consume(size);
+ Ok(size)
+ }
+}
+
+impl<R: Read> BufRead for BufReader<R> {
+ fn fill_buf(&mut self) -> Result<&[u8]> {
+ if self.buffer.read_available() < self.min {
+ self.buffer.flush();
+ }
+
+ if let Ok(size) = self.reader.read(self.buffer.write_slice()) {
+ self.buffer.write_done(size);
+ }
+
+ Ok(self.buffer.read_slice())
+ }
+
+ fn consume(&mut self, size: usize) {
+ self.buffer.read_done(size);
+ }
+}
+
+impl<R: Seek> Seek for BufReader<R> {
+ fn seek(&mut self, pos: SeekFrom) -> Result<u64> {
+ let result: u64;
+
+ if let SeekFrom::Current(pos) = pos {
+ result = self
+ .reader
+ .seek(SeekFrom::Current(pos - self.len() as i64))?;
+ } else {
+ result = self.reader.seek(pos)?;
+ }
+
+ self.buffer.clear();
+ Ok(result)
+ }
+}
+
+/// A fixed buffer that could be used as a byte dequeue.
+pub struct DeqBuffer {
+ slice: Box<[u8]>,
+ start: usize,
+ end: usize,
+}
+
+impl DeqBuffer {
+ /// Creates a new buffer with the given maximum space.
+ ///
+ /// # Safety
+ ///
+ /// The buffer needs to be written to before read from.
+ pub fn new(size: usize) -> Self {
+ let mut vec = Vec::with_capacity(size);
+ let cap = vec.capacity();
+
+ // SAFETY: Force the length of the vector to its capacity
+ // instead of using 'resize' for performance reasons.
+ unsafe {
+ vec.set_len(cap);
+ }
+
+ DeqBuffer {
+ slice: vec.into(),
+ start: 0,
+ end: 0,
+ }
+ }
+
+ /// Returns the total capacity of the buffer.
+ pub fn capacity(&self) -> usize {
+ self.slice.len()
+ }
+
+ /// Clears the buffer by consuming all available bytes
+ /// and returns the number of bytes being cleared.
+ pub fn clear(&mut self) -> usize {
+ let size = self.read_available();
+ self.read_done(size)
+ }
+
+ /// Reads from this buffer into the given output
+ /// and returns the number of bytes being read.
+ pub fn read_to(&mut self, buffer: &mut [u8]) -> usize {
+ let size = min(self.read_available(), buffer.len());
+ buffer[..size].copy_from_slice(&self.read_slice()[..size]);
+ self.read_done(size)
+ }
+
+ /// Returns the number of currently available bytes for reading.
+ pub fn read_available(&self) -> usize {
+ self.end - self.start
+ }
+
+ /// Returns the current slice to read from.
+ pub fn read_slice(&self) -> &[u8] {
+ &self.slice[self.start..self.end]
+ }
+
+ /// Signals the amount of newly read bytes from the buffer
+ /// and returns the number of bytes being updated.
+ pub fn read_done(&mut self, size: usize) -> usize {
+ let before = self.start;
+
+ self.start = min(self.start + size, self.slice.len());
+ let diff = self.start - before;
+
+ if self.start == self.end {
+ self.start = 0;
+ self.end = 0;
+ }
+
+ diff
+ }
+
+ /// Writes from the given input into this buffer
+ /// and returns the number of bytes being written.
+ pub fn write_from(&mut self, buffer: &[u8]) -> usize {
+ let size = min(self.write_available(), buffer.len());
+ self.write_slice()[..size].copy_from_slice(&buffer[..size]);
+ self.write_done(size)
+ }
+
+ /// Returns the number of currently available bytes for writing.
+ pub fn write_available(&self) -> usize {
+ self.slice.len() - self.end
+ }
+
+ /// Returns the current slice to write to.
+ pub fn write_slice(&mut self) -> &mut [u8] {
+ &mut self.slice[self.end..]
+ }
+
+ /// Signals the amount of newly written bytes to the buffer
+ /// and returns the number of bytes being updated.
+ pub fn write_done(&mut self, size: usize) -> usize {
+ let before = self.end;
+
+ self.end = min(self.end + size, self.slice.len());
+ self.end - before
+ }
+
+ /// Moves any remaining bytes within the buffer to its front
+ /// and returns the number of bytes being moved.
+ pub fn flush(&mut self) -> usize {
+ let before = self.start;
+
+ if self.start != 0 {
+ // SAFETY: Copying the bytes is safe as pointers will keep aliasing restrictions.
+ unsafe {
+ copy(
+ self.read_slice().as_ptr(),
+ self.slice.as_mut_ptr(),
+ self.read_available(),
+ );
+ }
+
+ self.end -= self.start;
+ self.start = 0;
+ }
+
+ before
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+ use std::io::Cursor;
+
+ #[test]
+ fn test_buffer() {
+ let max_size = 1000;
+ let chunk_size = 100;
+
+ let chunk1: [u8; 100] = [1; 100];
+ let chunk2: [u8; 100] = [2; 100];
+ let chunk3: [u8; 100] = [3; 100];
+ let mut temp: [u8; 100] = [0; 100];
+
+ let mut buffer = DeqBuffer::new(max_size);
+ assert_eq!(max_size, buffer.capacity());
+ assert_eq!(max_size, buffer.write_available());
+ assert_eq!(0, buffer.read_available());
+
+ // write first chunk
+ assert_eq!(chunk_size, buffer.write_from(&chunk1));
+ assert_eq!(max_size - chunk_size, buffer.write_available());
+ assert_eq!(chunk_size, buffer.read_available());
+
+ // write second chunk
+ assert_eq!(chunk_size, buffer.write_from(&chunk2));
+ assert_eq!(max_size - chunk_size * 2, buffer.write_available());
+ assert_eq!(chunk_size * 2, buffer.read_available());
+
+ // read first chunk
+ assert_eq!(chunk_size, buffer.read_to(&mut temp));
+ assert_eq!(chunk1, temp);
+ assert_eq!(max_size - chunk_size * 2, buffer.write_available());
+ assert_eq!(chunk_size, buffer.read_available());
+
+ // read second chunk will reset buffer
+ assert_eq!(chunk_size, buffer.read_to(&mut temp));
+ assert_eq!(chunk2, temp);
+ assert_eq!(max_size, buffer.write_available());
+ assert_eq!(0, buffer.read_available());
+
+ // write first chunk
+ assert_eq!(chunk_size, buffer.write_from(&chunk1));
+ assert_eq!(max_size - chunk_size, buffer.write_available());
+ assert_eq!(chunk_size, buffer.read_available());
+
+ // write second chunk
+ assert_eq!(chunk_size, buffer.write_from(&chunk2));
+ assert_eq!(max_size - chunk_size * 2, buffer.write_available());
+ assert_eq!(chunk_size * 2, buffer.read_available());
+
+ // read first chunk
+ assert_eq!(chunk_size, buffer.read_to(&mut temp));
+ assert_eq!(chunk1, temp);
+ assert_eq!(max_size - chunk_size * 2, buffer.write_available());
+ assert_eq!(chunk_size, buffer.read_available());
+
+ // write third chunk
+ assert_eq!(chunk_size, buffer.write_from(&chunk3));
+ assert_eq!(max_size - chunk_size * 3, buffer.write_available());
+ assert_eq!(chunk_size * 2, buffer.read_available());
+
+ // flush with rest
+ assert_eq!(chunk_size, buffer.flush());
+ assert_eq!(max_size - chunk_size * 2, buffer.write_available());
+ assert_eq!(chunk_size * 2, buffer.read_available());
+
+ // read second chunk
+ assert_eq!(chunk_size, buffer.read_to(&mut temp));
+ assert_eq!(chunk2, temp);
+ assert_eq!(max_size - chunk_size * 2, buffer.write_available());
+ assert_eq!(chunk_size, buffer.read_available());
+
+ // clear buffer
+ assert_eq!(chunk_size, buffer.clear());
+ assert_eq!(max_size, buffer.write_available());
+ assert_eq!(0, buffer.read_available());
+
+ // write all
+ for i in 0..max_size {
+ buffer.write_slice()[i] = (i % 255) as u8;
+ }
+ assert_eq!(max_size, buffer.write_done(max_size));
+ assert_eq!(0, buffer.write_available());
+ assert_eq!(max_size, buffer.read_available());
+
+ // buffer is full
+ assert_eq!(0, buffer.write_done(1));
+
+ // read one
+ assert_eq!(1, buffer.read_done(1));
+ assert_eq!(0, buffer.write_available());
+ assert_eq!(max_size - 1, buffer.read_available());
+
+ // flush rest
+ assert_eq!(1, buffer.flush());
+ assert_eq!(1, buffer.write_available());
+ assert_eq!(max_size - 1, buffer.read_available());
+
+ // read rest will reset buffer
+ for i in 1..(max_size) {
+ assert_eq!((i % 255) as u8, buffer.read_slice()[i - 1]);
+ }
+ assert_eq!(max_size - 1, buffer.read_done(max_size - 1));
+ assert_eq!(max_size, buffer.write_available());
+ assert_eq!(0, buffer.read_available());
+
+ // buffer is empty
+ assert_eq!(0, buffer.flush());
+ assert_eq!(max_size, buffer.write_available());
+ assert_eq!(0, buffer.read_available());
+ }
+
+ #[test]
+ fn test_reader() {
+ let input: &[u8] = &[1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15];
+ let max_size = 5;
+ let min_size = 2;
+
+ let mut reader = BufReader::new(max_size, min_size, input);
+ assert_eq!(max_size, reader.capacity());
+ assert!(reader.is_empty());
+
+ let mut output = [0, 0, 0];
+
+ // read first chunk fills buffer
+ assert_eq!(reader.read(&mut output).unwrap(), 3);
+ assert_eq!(output, [1, 2, 3]);
+
+ // read second chunk flushes buffer
+ assert_eq!(reader.read(&mut output).unwrap(), 3);
+ assert_eq!(output, [4, 5, 6]);
+
+ // refill buffer if below minimum
+ assert_eq!(reader.fill_buf().unwrap(), [7, 8]);
+ reader.consume(1);
+ assert_eq!(reader.fill_buf().unwrap(), [8, 9, 10, 11, 12]);
+
+ // read third chunk from buffer
+ assert_eq!(reader.read(&mut output).unwrap(), 3);
+ assert_eq!(output, [8, 9, 10]);
+
+ // drop some bytes and refill buffer
+ reader.consume(1);
+ assert_eq!(reader.read(&mut output).unwrap(), 3);
+ assert_eq!(output, [12, 13, 14]);
+
+ // read until end
+ let mut rest = [0, 0, 0];
+ assert_eq!(reader.read(&mut rest).unwrap(), 1);
+ assert_eq!(rest, [15, 0, 0]);
+ assert_eq!(reader.fill_buf().unwrap(), []);
+ }
+
+ #[test]
+ fn test_seek() {
+ let input: &[u8] = &[1, 2, 3, 4, 5, 6, 7, 8, 9];
+ let mut reader = BufReader::new(2, 1, Cursor::new(input));
+
+ // Seek from start
+ assert_eq!(reader.seek(SeekFrom::Start(2)).unwrap(), 2);
+ assert_eq!(reader.fill_buf().unwrap(), &[3, 4][..]);
+
+ // Seek from current
+ assert_eq!(reader.seek(SeekFrom::Current(1)).unwrap(), 3);
+ assert_eq!(reader.fill_buf().unwrap(), &[4, 5][..]);
+
+ // Seek empty
+ assert_eq!(reader.seek(SeekFrom::Current(0)).unwrap(), 3);
+ assert_eq!(reader.fill_buf().unwrap(), &[4, 5][..]);
+
+ // Seek reverse
+ assert_eq!(reader.seek(SeekFrom::Current(-1)).unwrap(), 2);
+ assert_eq!(reader.fill_buf().unwrap(), &[3, 4][..]);
+
+ // Seek after consuming bytes
+ reader.consume(1);
+ assert_eq!(reader.seek(SeekFrom::Current(1)).unwrap(), 4);
+ assert_eq!(reader.fill_buf().unwrap(), &[5, 6][..]);
+
+ // Seek to end
+ assert_eq!(reader.seek(SeekFrom::Start(9)).unwrap(), 9);
+ assert_eq!(reader.fill_buf().unwrap(), &[][..]);
+ }
+}
diff --git a/application/apps/indexer/addons/bufread/tests/proto.rs b/application/apps/indexer/addons/bufread/tests/proto.rs
new file mode 100644
index 0000000000..43e676e2af
--- /dev/null
+++ b/application/apps/indexer/addons/bufread/tests/proto.rs
@@ -0,0 +1,138 @@
+/*
+ Contains a pseudo protocol for testing, with packages consisting of
+ an u16 header with the length of the payload after the header
+ providing a total packet length within the range of
+ size_of::<u16>() to size_of::<u16>() + u16::MAX bytes.
+*/
+
+use bufread::BufReader;
+use rand::prelude::*;
+use std::{
+ io::{BufRead, Result},
+ mem::size_of,
+};
+
+/// The fixed header length of the protocol.
+pub const HEADER_LEN: usize = size_of::<u16>();
+/// The maximum payload length of the protocol.
+const MAX_PAYLOAD_LEN: usize = u16::MAX as usize;
+/// The maximum packet length of the protocol.
+pub const MAX_PACKET_LEN: usize = HEADER_LEN + MAX_PAYLOAD_LEN;
+
+/// A source for pseudo protocol packages.
+pub struct Source {
+ data: Vec<u8>,
+ num_packets: usize,
+}
+
+impl Source {
+ /// Creates a new source with a deterministic package layout and adds packages
+ /// until reaching at least the given minimal data size.
+ pub fn fixed(min_size: usize) -> Self {
+ let mut data = Vec::new();
+ let mut num_packets = 0;
+
+ while data.len() < min_size {
+ let payload_len = num_packets % 1024;
+ data.append(&mut Self::create_packet(payload_len));
+ num_packets += 1;
+ }
+
+ Source { data, num_packets }
+ }
+
+ /// Creates a new source with a random package layout and adds packages
+ /// until reaching at least the given minimal data size.
+ pub fn random(min_size: usize) -> Self {
+ let mut data = Vec::new();
+ let mut num_packets = 0;
+
+ while data.len() < min_size {
+ let payload_len: usize = rand::thread_rng().gen_range(0..MAX_PAYLOAD_LEN);
+ data.append(&mut Self::create_packet(payload_len));
+ num_packets += 1;
+ }
+
+ Source { data, num_packets }
+ }
+
+ /// Creates a new packet with the given payload length.
+ fn create_packet(payload_len: usize) -> Vec<u8> {
+ assert!(payload_len <= MAX_PAYLOAD_LEN);
+
+ let packet_len = HEADER_LEN + payload_len;
+ let mut packet: Vec<u8> = vec![0; packet_len];
+
+ let header = (payload_len as u16).to_be_bytes().to_vec();
+ packet[0] = header[0];
+ packet[1] = header[1];
+
+ rand::thread_rng().fill_bytes(&mut packet[HEADER_LEN..]);
+ packet
+ }
+
+ /// Returns the contained data slice.
+ pub fn data(&self) -> &[u8] {
+ self.data.as_slice()
+ }
+
+ /// Returns the length of the contained data.
+ pub fn data_len(&self) -> usize {
+ self.data.len()
+ }
+
+ /// Returns the number of the contained packages.
+ pub fn num_packets(&self) -> usize {
+ self.num_packets
+ }
+}
+
+/// A parser for pseudo protocol packages.
+pub struct Parser<'a> {
+ reader: BufReader<&'a [u8]>,
+}
+
+impl<'a> Parser<'a> {
+ /// Creates a new parser for the given source.
+ pub fn new(reader: BufReader<&'a [u8]>) -> Self {
+ Parser { reader }
+ }
+
+ /// Parses the next package from the source, if available.
+ ///
+ /// Returns the total length of the package being parsed, or
+ /// a zero-length if at EOF.
+ pub fn next(&mut self) -> Result<usize> {
+ let buffer = self.reader.fill_buf()?;
+ if buffer.is_empty() {
+ return Ok(0);
+ }
+
+ let mut header = [0; HEADER_LEN];
+ header[0] = buffer[0];
+ header[1] = buffer[1];
+
+ let payload_len = u16::from_be_bytes(header) as usize;
+ let packet_len = HEADER_LEN + payload_len;
+ self.reader.consume(packet_len);
+
+ Ok(packet_len)
+ }
+
+ /// Runs a parser and returns the total number of packets and bytes being read.
+ pub fn run(parser: &mut Parser) -> Result<(usize, usize)> {
+ let mut result: (usize, usize) = (0, 0);
+
+ loop {
+ let size = parser.next()?;
+ if size == 0 {
+ break;
+ }
+
+ result.0 += 1;
+ result.1 += size;
+ }
+
+ Ok(result)
+ }
+}
diff --git a/application/apps/indexer/addons/bufread/tests/proto_tests.rs b/application/apps/indexer/addons/bufread/tests/proto_tests.rs
new file mode 100644
index 0000000000..ff849bb46f
--- /dev/null
+++ b/application/apps/indexer/addons/bufread/tests/proto_tests.rs
@@ -0,0 +1,46 @@
+#[path = "./proto.rs"]
+mod proto;
+
+#[cfg(test)]
+mod tests {
+ use super::proto::{Parser, Source, MAX_PACKET_LEN};
+ use bufread::BufReader;
+
+ const SOURCE_MIN_SIZE: usize = 10 * MAX_PACKET_LEN;
+ const BUFFER_MAX_SIZE: usize = 3 * MAX_PACKET_LEN;
+ const BUFFER_MIN_SIZE: usize = MAX_PACKET_LEN;
+
+ #[test]
+ fn test_fixed_source() {
+ let source = Source::fixed(SOURCE_MIN_SIZE);
+ let reader = BufReader::new(BUFFER_MAX_SIZE, BUFFER_MIN_SIZE, source.data());
+ let mut parser = Parser::new(reader);
+
+ match Parser::run(&mut parser) {
+ Ok(result) => {
+ assert_eq!(source.num_packets(), result.0);
+ assert_eq!(source.data_len(), result.1);
+ }
+ Err(error) => {
+ panic!("{}", error);
+ }
+ }
+ }
+
+ #[test]
+ fn test_random_source() {
+ let source = Source::random(SOURCE_MIN_SIZE);
+ let reader = BufReader::new(BUFFER_MAX_SIZE, BUFFER_MIN_SIZE, source.data());
+ let mut parser = Parser::new(reader);
+
+ match Parser::run(&mut parser) {
+ Ok(result) => {
+ assert_eq!(source.num_packets(), result.0);
+ assert_eq!(source.data_len(), result.1);
+ }
+ Err(error) => {
+ panic!("{}", error);
+ }
+ }
+ }
+}
diff --git a/application/apps/indexer/addons/bufread/tests/reader_proptest.rs b/application/apps/indexer/addons/bufread/tests/reader_proptest.rs
new file mode 100644
index 0000000000..4212bfcdb6
--- /dev/null
+++ b/application/apps/indexer/addons/bufread/tests/reader_proptest.rs
@@ -0,0 +1,41 @@
+#![allow(dead_code)]
+
+#[path = "./proto.rs"]
+mod proto;
+
+#[cfg(test)]
+mod tests {
+ use super::proto::{Parser, Source, MAX_PACKET_LEN};
+ use bufread::BufReader;
+ use proptest::prelude::*;
+ use proptest::test_runner::FileFailurePersistence;
+
+ proptest! {
+ #![proptest_config(ProptestConfig::with_failure_persistence(FileFailurePersistence::Off))]
+ #[test]
+ fn reader_proptest(
+ source_min_size in (MAX_PACKET_LEN / 2)..(3 * MAX_PACKET_LEN),
+ buffer_max_size in MAX_PACKET_LEN..(2 * MAX_PACKET_LEN)
+ ) {
+ let buffer_min_size = MAX_PACKET_LEN;
+
+ let source = Source::fixed(source_min_size);
+ let reader = BufReader::new(buffer_max_size, buffer_min_size, source.data());
+ let mut parser = Parser::new(reader);
+
+ match Parser::run(&mut parser) {
+ Ok(result) => {
+ if source.num_packets() != result.0 {
+ panic!("num packets does not match: {} != {}", source.num_packets(), result.0);
+ }
+ if source.data_len() != result.1 {
+ panic!("source len does not match: {} != {}", source.data_len(), result.1);
+ }
+ }
+ Err(error) => {
+ panic!("{}", error);
+ }
+ }
+ }
+ }
+}
diff --git a/application/apps/indexer/addons/text_grep/Cargo.toml b/application/apps/indexer/addons/text_grep/Cargo.toml
index 903cfcf353..08a9ae0d5c 100644
--- a/application/apps/indexer/addons/text_grep/Cargo.toml
+++ b/application/apps/indexer/addons/text_grep/Cargo.toml
@@ -11,7 +11,7 @@ path = "src/lib.rs"
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
[dependencies]
tokio = { workspace = true, features = ["full"] }
-bufread.workspace = true
+bufread = { path = "../bufread" }
tokio-util.workspace = true
tempfile.workspace = true
grep-searcher.workspace = true
diff --git a/application/apps/indexer/addons/text_grep/src/buffer.rs b/application/apps/indexer/addons/text_grep/src/buffer.rs
index 13ce4083d4..1517a3b46a 100644
--- a/application/apps/indexer/addons/text_grep/src/buffer.rs
+++ b/application/apps/indexer/addons/text_grep/src/buffer.rs
@@ -1,24 +1,20 @@
-// TODO this duplicates: application/apps/indexer/processor/src/search/buffer.rs
use bufread::BufReader;
+use std::io::{BufRead, Error, ErrorKind, Read, Result, Seek, SeekFrom};
use tokio_util::sync::CancellationToken;
-use std::io::{BufRead, Read, Result, Seek, SeekFrom, Error, ErrorKind};
const BIN_READER_CAPACITY: usize = 1024 * 1024;
const BIN_MIN_BUFFER_SPACE: usize = 10 * 1024;
pub struct CancellableBufReader<R> {
buffer: BufReader<R>,
- cancel: CancellationToken
+ cancel: CancellationToken,
}
impl<R> CancellableBufReader<R> {
pub fn new(reader: R, cancel: CancellationToken) -> Self {
CancellableBufReader {
- buffer: BufReader::new(
- BIN_READER_CAPACITY,
- BIN_MIN_BUFFER_SPACE,
- reader),
- cancel
+ buffer: BufReader::new(BIN_READER_CAPACITY, BIN_MIN_BUFFER_SPACE, reader),
+ cancel,
}
}
}
@@ -28,7 +24,7 @@ impl<R: Read> Read for CancellableBufReader<R> {
if self.cancel.is_cancelled() {
return Ok(0);
}
-
+
self.buffer.read(buffer)
}
}
@@ -36,9 +32,9 @@ impl<R: Read> Read for CancellableBufReader<R> {
impl<R: Read> BufRead for CancellableBufReader<R> {
fn fill_buf(&mut self) -> Result<&[u8]> {
if self.cancel.is_cancelled() {
- return Ok(&[][..]);
+ return Ok(&[]);
}
-
+
self.buffer.fill_buf()
}
@@ -56,7 +52,7 @@ impl<R: Seek> Seek for CancellableBufReader<R> {
if self.cancel.is_cancelled() {
return Err(Error::from(ErrorKind::NotFound));
}
-
+
self.buffer.seek(pos)
}
-}
\ No newline at end of file
+}
diff --git a/application/apps/indexer/processor/Cargo.toml b/application/apps/indexer/processor/Cargo.toml
index dd0de53467..96f494db3c 100644
--- a/application/apps/indexer/processor/Cargo.toml
+++ b/application/apps/indexer/processor/Cargo.toml
@@ -6,7 +6,7 @@ edition = "2021"
[dependencies]
bincode = "1.3"
-bufread.workspace = true
+bufread = { path = "../addons/bufread" }
bytecount = "0.6"
futures.workspace = true
grep-regex.workspace = true
diff --git a/application/apps/indexer/processor/src/search/buffer.rs b/application/apps/indexer/processor/src/search/buffer.rs
index f3cd2e1b30..1517a3b46a 100644
--- a/application/apps/indexer/processor/src/search/buffer.rs
+++ b/application/apps/indexer/processor/src/search/buffer.rs
@@ -1,24 +1,20 @@
-// TODO this duplicates: application/apps/indexer/addons/text_grep/src/buffer.rs
use bufread::BufReader;
+use std::io::{BufRead, Error, ErrorKind, Read, Result, Seek, SeekFrom};
use tokio_util::sync::CancellationToken;
-use std::io::{BufRead, Read, Result, Seek, SeekFrom, Error, ErrorKind};
const BIN_READER_CAPACITY: usize = 1024 * 1024;
const BIN_MIN_BUFFER_SPACE: usize = 10 * 1024;
pub struct CancellableBufReader<R> {
buffer: BufReader<R>,
- cancel: CancellationToken
+ cancel: CancellationToken,
}
impl<R> CancellableBufReader<R> {
pub fn new(reader: R, cancel: CancellationToken) -> Self {
CancellableBufReader {
- buffer: BufReader::new(
- BIN_READER_CAPACITY,
- BIN_MIN_BUFFER_SPACE,
- reader),
- cancel
+ buffer: BufReader::new(BIN_READER_CAPACITY, BIN_MIN_BUFFER_SPACE, reader),
+ cancel,
}
}
}
@@ -28,7 +24,7 @@ impl<R: Read> Read for CancellableBufReader<R> {
if self.cancel.is_cancelled() {
return Ok(0);
}
-
+
self.buffer.read(buffer)
}
}
@@ -36,9 +32,9 @@ impl<R: Read> Read for CancellableBufReader<R> {
impl<R: Read> BufRead for CancellableBufReader<R> {
fn fill_buf(&mut self) -> Result<&[u8]> {
if self.cancel.is_cancelled() {
- return Ok(&[][..]);
+ return Ok(&[]);
}
-
+
self.buffer.fill_buf()
}
@@ -56,7 +52,7 @@ impl<R: Seek> Seek for CancellableBufReader<R> {
if self.cancel.is_cancelled() {
return Err(Error::from(ErrorKind::NotFound));
}
-
+
self.buffer.seek(pos)
}
-}
\ No newline at end of file
+}
diff --git a/application/apps/indexer/processor/src/search/searchers/mod.rs b/application/apps/indexer/processor/src/search/searchers/mod.rs
index 737bb6b5c5..d747794984 100644
--- a/application/apps/indexer/processor/src/search/searchers/mod.rs
+++ b/application/apps/indexer/processor/src/search/searchers/mod.rs
@@ -1,9 +1,6 @@
use crate::{
grabber::GrabError,
- search::{
- buffer::CancellableBufReader,
- error::SearchError,
- },
+ search::{buffer::CancellableBufReader, error::SearchError},
};
use grep_regex::RegexMatcher;
use grep_searcher::{sinks::UTF8, Searcher};
diff --git a/application/apps/indexer/sources/Cargo.toml b/application/apps/indexer/sources/Cargo.toml
index bf1a74a1dd..c7e4b905a7 100644
--- a/application/apps/indexer/sources/Cargo.toml
+++ b/application/apps/indexer/sources/Cargo.toml
@@ -6,7 +6,7 @@ edition = "2021"
[dependencies]
async-stream = "0.3"
-bufread.workspace = true
+bufread = { path = "../addons/bufread" }
bytes = "1.3"
etherparse = "0.16"
futures.workspace = true
diff --git a/application/apps/indexer/sources/src/socket/tcp.rs b/application/apps/indexer/sources/src/socket/tcp.rs
index e6e381683d..e13be90402 100644
--- a/application/apps/indexer/sources/src/socket/tcp.rs
+++ b/application/apps/indexer/sources/src/socket/tcp.rs
@@ -24,7 +24,7 @@ impl TcpSource {
let binding_address: String = addr.into();
Ok(Self {
buffer: DeqBuffer::new(8192),
- socket: TcpStream::connect(binding_address).await?,
+ socket: TcpStream::connect(&binding_address).await?,
tmp_buffer: vec![0u8; MAX_DATAGRAM_SIZE],
binding_address,
reconnect_info,
diff --git a/application/apps/indexer/stypes/Cargo.toml b/application/apps/indexer/stypes/Cargo.toml
index e61036c895..05c759e31d 100644
--- a/application/apps/indexer/stypes/Cargo.toml
+++ b/application/apps/indexer/stypes/Cargo.toml
@@ -35,7 +35,7 @@ envvars = { workspace = true, optional = true }
tokio = { workspace = true }
walkdir = { workspace = true }
node-bindgen = { git = "https://github.com/infinyon/node-bindgen.git", branch="master" }
-proptest = "1.5"
+proptest = { workspace = true }
paste = "1.0"
uuid = { workspace = true, features = ["serde", "v4"] }
remove_dir_all = "1.0"
diff --git a/application/apps/protocol/Cargo.lock b/application/apps/protocol/Cargo.lock
index 251582e50c..e28ca0e757 100644
--- a/application/apps/protocol/Cargo.lock
+++ b/application/apps/protocol/Cargo.lock
@@ -66,7 +66,8 @@ dependencies = [
[[package]]
name = "dlt-core"
version = "0.18.0"
-source = "git+https://github.com/kruss/dlt-core.git?branch=remove_buf_redux#87c7cea8be5224790ba8457ff531b4b6804c1298"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0b304e32f1164b8c2ef1dc746b32d321f25f88a32672f0f5bcba2df0f70a3b70"
dependencies = [
"byteorder",
"bytes",
diff --git a/application/apps/rustcore/rs-bindings/Cargo.lock b/application/apps/rustcore/rs-bindings/Cargo.lock
index c122744999..f8284ad2b2 100644
--- a/application/apps/rustcore/rs-bindings/Cargo.lock
+++ b/application/apps/rustcore/rs-bindings/Cargo.lock
@@ -344,10 +344,20 @@ dependencies = [
"serde",
]
+[[package]]
+name = "buf_redux"
+version = "0.8.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b953a6887648bb07a535631f2bc00fbdb2a2216f135552cb3f534ed136b9c07f"
+dependencies = [
+ "memchr",
+ "safemem",
+ "slice-deque",
+]
+
[[package]]
name = "bufread"
-version = "0.1.0"
-source = "git+https://github.com/kruss/bufread#5dec80376c0cbbbf3c4f864cf88b965cceb67c3b"
+version = "0.1.1"
[[package]]
name = "bumpalo"
@@ -631,9 +641,10 @@ dependencies = [
[[package]]
name = "dlt-core"
version = "0.18.0"
-source = "git+https://github.com/kruss/dlt-core.git?branch=remove_buf_redux#87c7cea8be5224790ba8457ff531b4b6804c1298"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0b304e32f1164b8c2ef1dc746b32d321f25f88a32672f0f5bcba2df0f70a3b70"
dependencies = [
- "bufread",
+ "buf_redux",
"byteorder",
"bytes",
"derive_more",
@@ -1264,6 +1275,15 @@ dependencies = [
"winapi",
]
+[[package]]
+name = "mach"
+version = "0.2.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "86dd2487cdfea56def77b88438a2c915fb45113c5319bfe7e14306ca4cd0b0e1"
+dependencies = [
+ "libc",
+]
+
[[package]]
name = "mach2"
version = "0.4.2"
@@ -2000,6 +2020,12 @@ version = "1.0.18"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f3cb5ba0dc43242ce17de99c180e96db90b235b8a9fdc9543c96d2209116bd9f"
+[[package]]
+name = "safemem"
+version = "0.3.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ef703b7cb59335eae2eb93ceb664c0eb7ea6bf567079d843e09420219668e072"
+
[[package]]
name = "same-file"
version = "1.0.6"
@@ -2207,6 +2233,17 @@ dependencies = [
"autocfg",
]
+[[package]]
+name = "slice-deque"
+version = "0.2.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ffddf594f5f597f63533d897427a570dbaa9feabaaa06595b74b71b7014507d7"
+dependencies = [
+ "libc",
+ "mach",
+ "winapi",
+]
+
[[package]]
name = "smallvec"
version = "1.13.2"