refactor: Rework opus/vpx builds to support arm64.

We can now run toxav_basic_test on qemu-aarch64.
TokTok · Dec 22, 2023 · 10eb91b · 10eb91b
1 parent df0b30d
commit 10eb91b
Show file tree

Hide file tree

Showing 25 changed files with 702 additions and 192 deletions.
diff --git a/.bazelrc b/.bazelrc
@@ -26,6 +26,7 @@ build --enable_platform_specific_config
 build --watchfs
 build --local_cpu_resources=HOST_CPUS
 build --features=layering_check
+build --distinct_host_configuration=false
 
 build --experimental_inmemory_dotd_files
 build --experimental_inmemory_jdeps_files
@@ -229,7 +230,7 @@ build:gnulike --per_file_copt='external/.*[-_]avx\\.c$@-mavx,-w'
 build:gnulike --per_file_copt='external/.*[-_]avx2\\.c$@-mavx2,-w'
 build:gnulike --per_file_copt='external/.*[-_]avx512f\\.c$@-mavx512f,-w'
 build:gnulike --per_file_copt='external/.*[-_]sse\\.c$@-mssse3,-msse4,-w'
-build:gnulike --per_file_copt='external/.*[-_]sse4\\.c$@-msse4,-w'
+build:gnulike --per_file_copt='external/.*[-_]sse4\\.c$@-msse4,-msse4.1,-w'
 build:gnulike --per_file_copt='external/.*[-_]sse41\\[email protected],-w'
 build:gnulike --per_file_copt='external/.*[-_]sse4_1\\[email protected],-w'
 build:gnulike --per_file_copt='external/.*[-_]ssse3\\.c$@-mssse3,-w'
@@ -275,10 +276,6 @@ build:gnulike --per_file_copt='//tools/debug[:/]@-g'
 test --test_output=errors
 test --test_verbose_timeout_warnings=true
 
-# This breaks Kythe.
-build --distinct_host_configuration=false
-build:kythe --distinct_host_configuration=true
-
 ##############################################################################
 #
 # :: Warning flags (some compiler-specific).
@@ -321,6 +318,9 @@ build:gnulike --per_file_copt='//(c-|go-|hs-|jvm-|py_|qtox|toxic|toxins)@-pedant
 # Source software.
 build:gnulike --per_file_copt='external/com_google_absl[:/]@-UNO_FRAME_POINTER,-fno-omit-frame-pointer'
 
+# Hack to get absl to compile for arm64.
+build:linux-arm64-musl --per_file_copt='external/com_google_absl/absl/base/internal/low_level_alloc.cc@-Doff64_t=long'
+
 # Compile toxcore as C++.
 build:clang --per_file_copt='//c-toxcore/@-xc++,-std=c++17'
 build:clang --per_file_copt='//c-toxcore/@-Wno-alloca'
@@ -381,7 +381,7 @@ build:clang --copt='-Wno-unsafe-buffer-usage'
 build:clang --copt='-Wno-deprecated-builtins'
 
 # LPTSTR cast for WSAStringToAddress is necessary.
-build:windows-amd64 --per_file_copt='//c-toxcore/toxcore:network.c@-Wno-cast-qual'
+build:windows-x86_64 --per_file_copt='//c-toxcore/toxcore:network.c@-Wno-cast-qual'
 
 build:clang --per_file_copt='//c-toxcore/toxav:video.c@-Wno-used-but-marked-unused'
 build:clang --per_file_copt='//c-toxcore/super_donators@-w'
@@ -528,14 +528,16 @@ build:gnulike --host_per_file_copt='external/yasm[:/]@-Wno-unused-but-set-variab
 #
 ##############################################################################
 
+build --config=clang
+
 # Flags for systems with GNU linker or compatible.
-build:gnuld --linkopt='-Wl,-z,noexecstack'
-#build:gnuld --linkopt='-Wl,--fatal-warnings'
-build:gnuld --linkopt='-Wl,--warn-common'
-build:gnuld --linkopt='-fPIE'
+build --host_linkopt='-Wl,-z,noexecstack'
+#build --host_linkopt='-Wl,--fatal-warnings'
+#build --host_linkopt='-Wl,--warn-common'
+build --host_linkopt='-fPIE'
 # Not enabled, because (at least - I gave up when trying to fix libvpx)
 # libsodium and libvpx assembly code doesn't define noexecstack.
-#build:gnuld --linkopt='-Wl,--warn-execstack'
+#build --host_linkopt='-Wl,--warn-execstack'
 
 # Continuous Integration platforms tend to not have IPv6.
 build:ci --per_file_copt='//c-toxcore/auto_tests@-DUSE_IPV6=0'
@@ -554,36 +556,34 @@ build:docker --config=ci
 #
 ##############################################################################
 
-build:nix --config=clang
-build:nix --config=gnuld
 build:nix --build_tag_filters=-windows,-macos
 
-build:zig --config=clang
 build:zig --copt=-fPIC
 build:zig --dynamic_mode=off
+build:zig --features=fully_static_link
 # See: https://github.com/uber/hermetic_cc_toolchain/issues/134
 build:zig --sandbox_add_mount_pair=/tmp
 
+build:linux-arm64-musl --config=zig
 build:linux-arm64-musl --platforms=@zig_sdk//platform:linux_arm64
 build:linux-arm64-musl --extra_toolchains=@zig_sdk//toolchain:linux_arm64_musl
-build:linux-arm64-musl --config=zig
 build:linux-arm64-musl --build_tag_filters=-windows,-macos
 
-build:linux-amd64-musl --platforms=@zig_sdk//platform:linux_amd64
-build:linux-amd64-musl --extra_toolchains=@zig_sdk//toolchain:linux_amd64_musl
-build:linux-amd64-musl --config=zig
-build:linux-amd64-musl --build_tag_filters=-windows,-macos
+build:linux-x86_64-musl --config=zig
+build:linux-x86_64-musl --platforms=@zig_sdk//platform:linux_amd64
+build:linux-x86_64-musl --extra_toolchains=@zig_sdk//toolchain:linux_amd64_musl
+build:linux-x86_64-musl --build_tag_filters=-windows,-macos
 
-build:windows-amd64 --platforms=@zig_sdk//platform:windows_amd64
-build:windows-amd64 --extra_toolchains=@zig_sdk//toolchain:windows_amd64
-build:windows-amd64 --config=zig
-build:windows-amd64 --build_tag_filters=-macos
+build:windows-x86_64 --config=zig
+build:windows-x86_64 --platforms=@zig_sdk//platform:windows_amd64
+build:windows-x86_64 --extra_toolchains=@zig_sdk//toolchain:windows_amd64
+build:windows-x86_64 --build_tag_filters=-macos
 
 # Pick *one* of these for the target platform. "nix" is the default Linux target.
 # TODO(iphydf): Move to .bazelrc.local.
 #build --config=linux-arm64-musl
-#build --config=linux-amd64-musl
-#build --config=windows-amd64
+#build --config=linux-x86_64-musl
+#build --config=windows-x86_64
 build --config=nix
 
 try-import %workspace%/.bazelrc.local

diff --git a/.restyled.yaml b/.restyled.yaml
@@ -6,3 +6,6 @@ exclude:
   - "third_party/haskell/happy/templates/*"
   - "third_party/libsodium/version.h"
   - "third_party/libzmq/**/*.hpp"
+  - "third_party/libvpx/**/*.h"
+  - "third_party/libvpx/**/*.mk"
+  - "third_party/opus/**/*.h"
diff --git a/WORKSPACE b/WORKSPACE
@@ -415,8 +415,8 @@ http_archive(
 github_archive(
     name = "com_google_googletest",
     repo = "google/googletest",
-    sha256 = "263000d4afc2a7a26dd75fe23fc86535c73785984ab053b4dfb231caa76b03d6",
-    version = "89b25572dbd7668499d2cfd01dea905f8c44e019",
+    sha256 = "93b52882cff4e8fb344070106facee0df5e61be91b1948a69c8581b3bcbe0c61",
+    version = "b3a9ba2b8e975550799838332803d468797ae2e1",
 )
 
 new_github_archive(

diff --git a/c-toxcore b/c-toxcore
diff --git a/third_party/BUILD.autoconf b/third_party/BUILD.autoconf
@@ -9,3 +9,9 @@ filegroup(
     srcs = ["bin/autoreconf"],
     visibility = ["//visibility:public"],
 )
+
+filegroup(
+    name = "autoupdate",
+    srcs = ["bin/autoupdate"],
+    visibility = ["//visibility:public"],
+)
diff --git a/third_party/BUILD.libtool b/third_party/BUILD.libtool
@@ -9,3 +9,9 @@ filegroup(
     srcs = ["bin/libtoolize"],
     visibility = ["//visibility:public"],
 )
+
+filegroup(
+    name = "libtool.m4",
+    srcs = ["share/aclocal/libtool.m4"],
+    visibility = ["//visibility:public"],
+)
diff --git a/third_party/BUILD.libvpx b/third_party/BUILD.libvpx
@@ -3,20 +3,24 @@ load("@toktok//third_party/yasm:build_defs.bzl", "asm_library")
 
 asm_library(
     name = "assemblies",
-    srcs = glob([
-        "vp*/**/x86/*.asm",
-    ]) + [
-        "vpx_ports/float_control_word.asm",
-    ] + select({
+    srcs = select({
+        "@toktok//tools/config:x86_64": glob([
+            "vp*/**/x86/*.asm",
+        ]) + [
+            "vpx_ports/float_control_word.asm",
+        ],
+    }) + select({
+        "@toktok//tools/config:linux": [],
         "@toktok//tools/config:windows": ["vpx_ports/emms_mmx.asm"],
-        "//conditions:default": [],
     }),
-    hdrs = [
-        "third_party/x86inc/x86inc.asm",
-        "vpx_dsp/x86/bitdepth_conversion_sse2.asm",
-        "vpx_ports/x86_abi_support.asm",
-        "@toktok//third_party/libvpx:vpx_config.asm",
-    ],
+    hdrs = select({
+        "@toktok//tools/config:x86_64": [
+            "third_party/x86inc/x86inc.asm",
+            "vpx_dsp/x86/bitdepth_conversion_sse2.asm",
+            "vpx_ports/x86_abi_support.asm",
+            "@toktok//third_party/libvpx:vpx_config.asm",
+        ],
+    }),
     archive = False,
     asmopts = [
         "-I$(GENDIR)/external/libvpx",  # Generated headers.
@@ -25,61 +29,129 @@ asm_library(
     ],
 )
 
+CONFIGURE = """
+export PATH="$$PATH:$$PWD/$$(dirname $(location @diffutils//:diff))"
+export PATH="$$PATH:$$PWD/$$(dirname $(location @gnumake))"
+export PATH="$$PATH:$$PWD/$$(dirname $(location @perl))"
+export PATH="$$PATH:$$PWD/$$(dirname $(location @yasm))"
+if [[ "$(CC)" == /* ]]; then
+  export CC="$(CC)"
+  export CXX="$(CC)"
+else
+  export CC="$$PWD/$(CC)"
+  export CXX="$$PWD/$(CC)"
+fi
+
+cd `dirname $(location configure)`
+mkdir -p gcc-wrapper
+echo '"$$CC" "$$@"' > gcc-wrapper/gcc
+chmod +x gcc-wrapper/gcc
+export PATH="$$PATH:$$PWD/gcc-wrapper"
+
+./configure \
+    --disable-avx512 \
+    --disable-webm-io \
+    --enable-pic \
+    --enable-shared \
+    --enable-vp9-temporal-denoising \
+    --enable-vp9-highbitdepth \
+    --enable-error-concealment \
+    --enable-internal-stats || (cat config.log && false)
+make vpx_version.h
+DIR=`pwd`
+cd -
+for i in $(OUTS); do
+  cp $$DIR/`basename $$i` $$i
+done
+"""
+
 genrule(
-    name = "regen_config",
+    name = "config-linux-arm64",
     srcs = glob(["**/*"]),
     outs = [
-        "vpx_config.asm",
-        "vpx_config.h",
-        "vpx_version.h",
-        "libs-x86_64-linux-gcc.mk",
+        "linux-arm64/libs-arm64-linux-gcc.mk",
+        "linux-arm64/vpx_config.h",
+        "linux-arm64/vpx_version.h",
     ],
-    cmd = """
-        export PATH=$$PWD/$$(dirname $(location @yasm)):$$PATH
-        cd `dirname $(location configure)`
-        ./configure --disable-avx512 --disable-webm-io --enable-pic --enable-shared --enable-vp9-temporal-denoising --enable-vp9-highbitdepth --enable-error-concealment --enable-internal-stats
-        make vpx_version.h
-        DIR=`pwd`
-        cd -
-        for i in $(OUTS); do
-          cp $$DIR/`basename $$i` $$i
-        done
-    """,
-    exec_tools = ["@yasm"],
+    cmd = CONFIGURE,
+    exec_tools = [
+        "@diffutils//:diff",
+        "@gnumake",
+        "@perl",
+        "@yasm",
+    ],
+    toolchains = ["@rules_cc//cc:current_cc_toolchain"],
 )
 
+genrule(
+    name = "config-linux-x86_64",
+    srcs = glob(["**/*"]),
+    outs = [
+        "linux-x86_64/libs-x86_64-linux-gcc.mk",
+        "linux-x86_64/vpx_config.asm",
+        "linux-x86_64/vpx_config.h",
+        "linux-x86_64/vpx_version.h",
+    ],
+    cmd = CONFIGURE,
+    exec_tools = [
+        "@diffutils//:diff",
+        "@gnumake",
+        "@perl",
+        "@yasm",
+    ],
+    toolchains = ["@rules_cc//cc:current_cc_toolchain"],
+)
+
+alias(
+    name = "config",
+    actual = select({
+        "@toktok//tools/config:linux-arm64": ":config-linux-arm64",
+        "@toktok//tools/config:linux-x86_64": ":config-linux-x86_64",
+    }),
+)
+
+RTCDS = [
+    ("vp8", "vp8/common/rtcd_defs.pl"),
+    ("vp9", "vp9/common/vp9_rtcd_defs.pl"),
+    ("vpx_dsp", "vpx_dsp/vpx_dsp_rtcd_defs.pl"),
+    ("vpx_scale", "vpx_scale/vpx_scale_rtcd.pl"),
+]
+
 [genrule(
-    name = "%s_rtcd" % mod,
+    name = "%s_%s_rtcd" % (cpu, mod),
     srcs = [
         "build/make/rtcd.pl",
-        "@toktok//third_party/libvpx:libs-x86_64-linux-gcc.mk",
+        "@toktok//third_party/libvpx:libs.mk",
         defs,
     ],
-    outs = ["%s_rtcd.h" % mod],
+    outs = ["%s/%s_rtcd.h" % (cpu, mod)],
     cmd = " ".join([
-        "$(location @perl//:perl)",
+        "$(location @perl)",
         "$(location build/make/rtcd.pl)",
-        "--arch=x86_64",
+        "--arch=%s" % cpu,
         "--sym=%s_rtcd" % mod,
         "--disable-avx512",
-        "--config=$(location @toktok//third_party/libvpx:libs-x86_64-linux-gcc.mk)",
+        "--config=$(location @toktok//third_party/libvpx:libs.mk)",
         "$(location %s)" % defs,
         "> $@",
     ]),
     tools = ["@perl"],
-) for mod, defs in [
-    ("vp8", "vp8/common/rtcd_defs.pl"),
-    ("vp9", "vp9/common/vp9_rtcd_defs.pl"),
-    ("vpx_dsp", "vpx_dsp/vpx_dsp_rtcd_defs.pl"),
-    ("vpx_scale", "vpx_scale/vpx_scale_rtcd.pl"),
+) for mod, defs in RTCDS for cpu in [
+    "arm64",
+    "x86_64",
 ]]
 
-HDRS = glob(["**/*.h"]) + [
-    "vp8_rtcd.h",
-    "vp9_rtcd.h",
-    "vpx_dsp_rtcd.h",
-    "vpx_scale_rtcd.h",
-]
+[genrule(
+    name = "%s_rtcd" % mod,
+    srcs = select({
+        "@toktok//tools/config:linux-arm64": ["arm64_%s_rtcd" % mod],
+        "@toktok//tools/config:linux-x86_64": ["x86_64_%s_rtcd" % mod],
+    }),
+    outs = ["%s_rtcd.h" % mod],
+    cmd = "cp $< $@",
+) for mod, _ in RTCDS]
+
+HDRS = glob(["**/*.h"]) + ["%s_rtcd.h" % mod for mod, _ in RTCDS]
 
 cc_library(
     name = "headers_private",
@@ -95,14 +167,14 @@ filegroup(
 
 cc_library(
     name = "libvpx",
-    srcs = [":assemblies"] + glob(
+    srcs = glob(
         ["vp*/**/*.c"],
         exclude = [
             "**/arm/**",
             "**/loongarch/**",
             "**/mips/**",
             "**/ppc/**",
-            "**/x86/*_avx512.c",
+            "**/x86/**",
             "vp8/encoder/mr_dissim.c",
             "vpx_ports/arm_cpudetect.c",
             "vpx_ports/loongarch_cpudetect.c",
@@ -111,8 +183,19 @@ cc_library(
             "vpx_ports/emms_mmx.c",
         ],
     ) + select({
-        "@toktok//tools/config:windows": [],
-        "//conditions:default": ["vpx_ports/emms_mmx.c"],
+        "@toktok//tools/config:arm64": glob(
+            ["vp*/**/arm/**/*.c"],
+            exclude = [
+                "vpx_dsp/arm/vpx_convolve8_neon.c",
+            ],
+        ) + ["vpx_ports/arm_cpudetect.c"],
+        "@toktok//tools/config:x86_64": glob(
+            ["vp*/**/x86/**/*.c"],
+            exclude = ["**/*_avx512.c"],
+        ) + [
+            "vpx_ports/emms_mmx.c",
+            ":assemblies",
+        ],
     }),
     hdrs = [
         "vpx/vp8cx.h",
+29 −2		.github/workflows/post-submit.yml
+3 −2		.github/workflows/sonar-scan.yml
+1 −1		other/bootstrap_daemon/docker/tox-bootstrapd.sha256
+5 −9		toxcore/network.c