From 1d7993b435d09123cd18f8b82ccb4dd8e5a894c8 Mon Sep 17 00:00:00 2001 From: iphydf Date: Sat, 25 Jan 2025 11:35:14 +0000 Subject: [PATCH] feat: Add simdjson to the stack. This one is quite fast. If we ever need to do fast json parsing, this is the one for it. --- .bazelrc | 5 ++ .clang-format | 84 +++++++++++++++++++++++++- .github/workflows/ci.yml | 6 ++ WORKSPACE | 8 +++ third_party/libsodium/BUILD.bazel | 9 +++ third_party/libsodium/libsodium_test.c | 40 ++++++++++++ third_party/simdjson.BUILD | 9 +++ 7 files changed, 160 insertions(+), 1 deletion(-) create mode 100644 third_party/libsodium/libsodium_test.c create mode 100644 third_party/simdjson.BUILD diff --git a/.bazelrc b/.bazelrc index 7bb3ae76..2fb3f837 100644 --- a/.bazelrc +++ b/.bazelrc @@ -209,6 +209,10 @@ build:msan --copt='-fsanitize-memory-track-origins=2' build:msan --linkopt='-fsanitize=memory' build:msan --test_env=MSAN_OPTIONS=color=always:external_symbolizer_path=/nix/store/h6rpdk8nb097diyqk1vsglw9chwgfsjr-llvm-18.1.8/bin/llvm-symbolizer build:msan --per_file_copt='//toxic[:/]@-UAUDIO,-UVIDEO,-UPYTHON' +# msan doesn't like Go (or Go doesn't like Bazel's msan): +# gcc_traceback.c:15:10: fatal error: 'sanitizer/msan_interface.h' file not found +build:msan --build_tag_filters=-no-san,-haskell,-windows,-macos,-go +build:msan --test_tag_filters=-no-san,-haskell,-windows,-macos,-go build:tsan --config=sanitizer build:tsan --features=tsan @@ -671,6 +675,7 @@ build --per_file_copt='external/sdl2[:/]@-Wno-pedantic' build --per_file_copt='external/sdl2[:/]@-Wno-sign-compare' build --per_file_copt='external/sdl2[:/]@-Wno-tautological-value-range-compare' build --per_file_copt='external/sdl2[:/]@-Wno-unused-but-set-variable' +build --per_file_copt='external/simdjson[:/]@-Wno-unused-member-function' build --per_file_copt='external/sqlcipher[:/]@-Wno-atomic-implicit-seq-cst' build --per_file_copt='external/sqlcipher[:/]@-Wno-cast-function-type-strict' build --per_file_copt='external/sqlcipher[:/]@-Wno-empty-translation-unit' diff --git a/.clang-format b/.clang-format index ee1bee2b..9da7025a 100644 --- a/.clang-format +++ b/.clang-format @@ -1,2 +1,84 @@ -BasedOnStyle: Google +--- +# Language +Standard: Cpp11 + +# Indentation +IndentWidth: 4 +ContinuationIndentWidth: 4 +AccessModifierOffset: -4 +IndentCaseLabels: false +NamespaceIndentation: None + +# Spacing +UseTab: Never +SpaceBeforeParens: ControlStatements +SpacesBeforeTrailingComments: 1 +SpaceInEmptyParentheses: false +SpacesInAngles: false +SpacesInParentheses: false +SpacesInSquareBrackets: false +SpacesInCStyleCastParentheses: false +SpaceBeforeAssignmentOperators: true +MaxEmptyLinesToKeep: 2 + +# Alignment +AlignAfterOpenBracket: Align +AlignConsecutiveAssignments: false +AlignConsecutiveDeclarations: false +AlignEscapedNewlinesLeft: true +AlignOperands: true +AlignTrailingComments: true + +# Argument Packing +BinPackArguments: true +BinPackParameters: true + +# Break handling ColumnLimit: 100 +BreakBeforeBraces: Mozilla +BreakBeforeBinaryOperators: NonAssignment +BreakConstructorInitializersBeforeComma: true +AlwaysBreakTemplateDeclarations: true +ConstructorInitializerAllOnOneLineOrOnePerLine: false +Cpp11BracedListStyle: true + +# Break penalities +PenaltyBreakBeforeFirstCallParameter: 200 +PenaltyBreakComment: 300 +PenaltyBreakFirstLessLess: 120 +PenaltyBreakString: 1000 +PenaltyExcessCharacter: 5 +PenaltyReturnTypeOnItsOwnLine: 60 + +# Includes +SortIncludes: true +IncludeIsMainRegex: '([-_](test|fuzz_test))?$' +IncludeBlocks: Regroup +IncludeCategories: + # Match local headers + - Regex: '^"[[:alnum:]_]+\.h"$' + Priority: 1 + # Match project headers + - Regex: '^"[[:alnum:]_]+/.+\.h"$' + Priority: 2 + # Match Qt headers + - Regex: '^$' + Priority: 3 + # Match other headers + - Regex: '.*' + Priority: 4 + +# Short blocks +AllowShortBlocksOnASingleLine: false +AllowShortCaseLabelsOnASingleLine: false +AllowShortIfStatementsOnASingleLine: false +AllowShortLoopsOnASingleLine: false +AllowShortFunctionsOnASingleLine: Empty + +# Set pointer format +DerivePointerAlignment: false +PointerAlignment: Left + +# "const X" instead of "X const" +QualifierAlignment: Left +... diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 20917d5f..739e4ef8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -55,6 +55,12 @@ jobs: - name: Build toxchat/toktok-stack:latest run: docker build -t toxchat/toktok-stack:latest -f tools/built/src/Dockerfile . + - name: Build toxchat/toktok-stack:latest-msan + run: | + docker build -t toxchat/toktok-stack:latest-msan -f tools/built/src/Dockerfile.msan . + docker rmi toxchat/toktok-stack:latest-msan + docker system prune -f + - name: Build toxchat/toktok-stack:latest-asan run: | docker build -t toxchat/toktok-stack:latest-asan -f tools/built/src/Dockerfile.asan . diff --git a/WORKSPACE b/WORKSPACE index 00ce8cb4..c55fe3f4 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -597,6 +597,14 @@ http_archive( urls = ["https://github.com/nlohmann/json/releases/download/v3.11.3/include.zip"], ) +# https://github.com/simdjson/simdjson/releases +new_github_archive( + name = "simdjson", + integrity = "sha256-hl/62dOtM7ARIHS6iJ7pNNtT1hPq/GOS7A0aDJ5bAFc=", + repo = "simdjson/simdjson", + version = "v3.11.6", +) + # https://github.com/libunwind/libunwind/releases http_archive( name = "libunwind", diff --git a/third_party/libsodium/BUILD.bazel b/third_party/libsodium/BUILD.bazel index c72b065e..43ec79f1 100644 --- a/third_party/libsodium/BUILD.bazel +++ b/third_party/libsodium/BUILD.bazel @@ -1,6 +1,15 @@ +load("@rules_cc//cc:defs.bzl", "cc_test") + licenses(["notice"]) exports_files( srcs = glob(["config/*.h"]) + ["version.h"], visibility = ["@libsodium//:__pkg__"], ) + +cc_test( + name = "libsodium_test", + size = "small", + srcs = ["libsodium_test.c"], + deps = ["@libsodium"], +) diff --git a/third_party/libsodium/libsodium_test.c b/third_party/libsodium/libsodium_test.c new file mode 100644 index 00000000..4d92e58d --- /dev/null +++ b/third_party/libsodium/libsodium_test.c @@ -0,0 +1,40 @@ +#include +#include + +static const char message[] = "hello world"; +static const char expected_hash[crypto_generichash_BYTES] = + "\x25\x6c\x83\xb2\x97\x11\x4d\x20\x1b\x30\x17\x9f\x3f\x0e\xf0\xca" + "\xce\x97\x83\x62\x2d\xa5\x97\x43\x26\xb4\x36\x17\x8a\xee\xf6\x10"; + +// Check whether we can get a hash of a message. +int main(void) +{ + if (sodium_init() < 0) { + return 1; + } + + unsigned char hash[crypto_generichash_BYTES]; + crypto_generichash(hash, sizeof hash, (const unsigned char*)message, sizeof message - 1, NULL, 0); + + for (size_t i = 0; i < sizeof hash; i++) { + if (hash[i] != expected_hash[i]) { + fprintf(stderr, "expected: "); + for (size_t j = 0; j < sizeof expected_hash; j++) { + fprintf(stderr, "%02x", expected_hash[j]); + } + fputc('\n', stderr); + fprintf(stderr, "actual: "); + for (size_t j = 0; j < sizeof hash; j++) { + fprintf(stderr, "%02x", hash[j]); + } + fputs("\n ", stderr); + for (size_t j = 0; j < i; j++) { + fputs(" ", stderr); + } + fputs("^^\n", stderr); + return 1; + } + } + + return 0; +} diff --git a/third_party/simdjson.BUILD b/third_party/simdjson.BUILD new file mode 100644 index 00000000..5d2b642e --- /dev/null +++ b/third_party/simdjson.BUILD @@ -0,0 +1,9 @@ +load("@rules_cc//cc:defs.bzl", "cc_library") + +cc_library( + name = "simdjson", + srcs = ["singleheader/simdjson.cpp"], + hdrs = ["singleheader/simdjson.h"], + includes = ["singleheader"], + visibility = ["//visibility:public"], +)