From 470d642b6d33892cadd9284a4f447d0c7f55a73a Mon Sep 17 00:00:00 2001 From: Adam Kewley Date: Tue, 21 Jan 2025 14:44:36 +0100 Subject: [PATCH] Refactor scripts directory --- .../continuous-integration-workflow.yml | 16 ++++-- docs/source/building-from-source.rst | 25 ++++++---- docs/source/setup-development-environment.rst | 3 ++ scripts/build_emscripten.sh | 16 ++---- ...debian-buster.sh => build_ubuntu-20.04.sh} | 50 ++----------------- scripts/build_ubuntu-20.04_debugging.sh | 16 +----- scripts/build_ubuntu-24.04_debugging.sh | 6 +++ scripts/build_windows-rapid.bat | 2 +- scripts/build_windows.py | 23 +-------- scripts/generate_icon_files.py | 0 scripts/get_osc_github_download_counts.py | 4 +- scripts/macos_check-dependencies.py | 7 +++ scripts/macos_check-sdk.py | 4 ++ scripts/setup_emscripten.sh | 17 +++++++ scripts/setup_ubuntu-20.04.sh | 14 ++++++ ...SL2_debugging.sh => setup_ubuntu-24.04.sh} | 3 -- 16 files changed, 90 insertions(+), 116 deletions(-) rename scripts/{build_debian-buster.sh => build_ubuntu-20.04.sh} (62%) mode change 100755 => 100644 mode change 100644 => 100755 scripts/build_ubuntu-20.04_debugging.sh create mode 100755 scripts/build_ubuntu-24.04_debugging.sh mode change 100644 => 100755 scripts/build_windows-rapid.bat mode change 100644 => 100755 scripts/build_windows.py mode change 100644 => 100755 scripts/generate_icon_files.py mode change 100644 => 100755 scripts/get_osc_github_download_counts.py create mode 100644 scripts/setup_emscripten.sh create mode 100755 scripts/setup_ubuntu-20.04.sh rename scripts/{build_ubuntu-24-WSL2_debugging.sh => setup_ubuntu-24.04.sh} (57%) mode change 100644 => 100755 diff --git a/.github/workflows/continuous-integration-workflow.yml b/.github/workflows/continuous-integration-workflow.yml index 69bf6bb3ff..00b33e2107 100644 --- a/.github/workflows/continuous-integration-workflow.yml +++ b/.github/workflows/continuous-integration-workflow.yml @@ -35,14 +35,18 @@ jobs: path: osc submodules: 'recursive' + - name: setup system + run: | + sudo apt-get install -y build-essential cmake libgtk-3-dev liblapack-dev + sudo apt-get install -y xvfb # virtual desktop, for tests + - name: run build script run: | cd $GITHUB_WORKSPACE/osc - sudo apt-get install xvfb # virtual desktop, for tests export OSC_BUILD_CONCURRENCY=$(nproc) export CC=clang export CXX=clang++ - xvfb-run ./scripts/build_debian-buster.sh + xvfb-run ./scripts/build_ubuntu-20.04.sh - uses: actions/upload-artifact@v4 with: @@ -84,12 +88,16 @@ jobs: path: osc submodules: 'recursive' + - name: setup system + run: | + sudo apt-get install -y build-essential cmake libgtk-3-dev liblapack-dev + sudo apt-get install -y xvfb # virtual desktop, for tests + - name: run build script run: | cd $GITHUB_WORKSPACE/osc - sudo apt-get install xvfb # virtual desktop, for tests export OSC_BUILD_CONCURRENCY=$(nproc) export CC=gcc-12 export CXX=g++-12 - xvfb-run ./scripts/build_debian-buster.sh + xvfb-run ./scripts/build_ubuntu-20.04.sh diff --git a/docs/source/building-from-source.rst b/docs/source/building-from-source.rst index 06f293f408..f81244c250 100644 --- a/docs/source/building-from-source.rst +++ b/docs/source/building-from-source.rst @@ -111,19 +111,18 @@ Building on MacOS (Sonoma or newer) 1. The ``osc-build`` directory should contain the built installer -Building on Ubuntu (20 or newer) --------------------------------- +Building on Ubuntu (20.04 or newer) +----------------------------------- 1. Get ``git``: 1. Install ``git`` via your package manager (e.g. ``apt-get install git``) - 2. Get a C++20-compatible compiler: 1. Install ``g++`` / ``clang++``` via your package manager (e.g. ``apt-get install g++``) 2. They must be new enough to compile C++20 (e.g. clang >= clang-11) 3. If they aren't new enough, most OSes provide a way to install a newer compiler toolchain (e.g. ``apt-get install clang-11``). You can configure which compiler is used to build OpenSim Creator by setting the ``CC`` and ``CXX`` environment variables. E.g. - ``CC=clang-11 CXX=clang++-11 ./scripts/build_debian-buster.sh`` + ``CC=clang-11 CXX=clang++-11 ./scripts/build_ubuntu-20.04.sh`` 3. Get C++20-compatible standard library headers (usually required on Ubuntu 20): 1. ``sudo apt-get install libstdc++-10-dev`` 4. Get ``cmake``: @@ -131,11 +130,19 @@ Building on Ubuntu (20 or newer) 2. If your cmake is too old, build one from source, see: https://askubuntu.com/a/865294 5. Get ``python`` and ``pip`` (*optional*: you only need this if you want to build documentation): 1. Install ``python3`` and ``pip3`` via your package manager (e.g. ``apt-get install python3 pip3``) -6. Build OpenSim Creator in a terminal: +6. Use ``git`` to get OpenSim Creator's (+ dependencies') source code: 1. Clone ``opensim-creator``: ``git clone https://github.com/ComputationalBiomechanicsLab/opensim-creator --recursive`` 2. ``cd`` into the source dir: ``cd opensim-creator`` - 3. If you have multiple C++ compilers, make sure that the ``CC`` and ``CXX`` environment variables point to + 3. The remaining build steps are performed in the source directory +7. Get python libraries (*optional*: you only need this if you want to build documentation): + 1. ``cd`` into the ``opensim-creator`` source directory (if you haven't already) + 2. Install all necessary python libraries into your current python environment with ``pip install -r docs/requirements.txt && pip install -r docs/requirements-dev.txt``` +8. Build OpenSim Creator from source: + 1. ``cd`` into the ``opensim-creator`` source directory (if you haven't already) + 1. If you have multiple C++ compilers, make sure that the ``CC`` and ``CXX`` environment variables point to compilers that are compatible with C++20. E.g. ``export CC=clang-12``, ``export CXX=clang++-12`` - 4. Run the build script: ``scripts/build_debian-buster.sh`` -7. Done: - 1. The ``osc-build`` directory should contain the built installer + 3. Run the build script: ``scripts/build_ubuntu-20.04.sh`` + 4. You can also accelerate it by setting the number of threads: ``OSC_BUILD_CONCURRENCY=20 ./scripts/build_ubuntu-20.04.sh`` +9. Done: + 1. After the build is complete, the ``osc-build`` directory should contain the built installer + diff --git a/docs/source/setup-development-environment.rst b/docs/source/setup-development-environment.rst index 4eb3aa2f04..68f7649a25 100644 --- a/docs/source/setup-development-environment.rst +++ b/docs/source/setup-development-environment.rst @@ -10,6 +10,9 @@ These are some generic tips that might be handy when setting up your own develop Visual Studio 2022 ------------------ +- Open a terminal/powershell window +- Run ``git clone --recursive https://github.com/ComputationalBiomechanicsLab/opensim-creator`` to get a complete copy of OpenSim Creator's sourcecode +- ``cd opensim-creator`` - Run ``python scripts/build_windows.py --skip-osc`` (described above) to get a complete build of OSC's dependencies. - In Visual Studio 2020, open ``opensim-creator`` as a folder project diff --git a/scripts/build_emscripten.sh b/scripts/build_emscripten.sh index 2f1ce98205..e1ac421982 100755 --- a/scripts/build_emscripten.sh +++ b/scripts/build_emscripten.sh @@ -4,7 +4,9 @@ # of OpenSim Creator using the Emscripten SDK (`emmake`, `emcc`, etc.) toolchain # to build a wasm version of the binary # -# usage (must be ran in repository root): `bash build_emscripten.sh` +# usage (must be ran in repository root): `bash scripts/build_emscripten.sh` +# note: assumes `emsdk` is already active (e.g. `./emsdk/emsdk activate latest`) +# note: assumes `emsdk` is sourced (e.g. `./emsdk/emsdk_env.sh`) set -xeuo pipefail @@ -17,18 +19,6 @@ OSC_DEPS_BUILD_TYPE=${OSC_DEPS_BUILD_TYPE:-`echo ${OSC_BASE_BUILD_TYPE}`} # build type for OSC OSC_BUILD_TYPE=${OSC_BUILD_TYPE-`echo ${OSC_BASE_BUILD_TYPE}`} -# install emsdk -if [ ! -d emsdk ]; then - git clone https://github.com/emscripten-core/emsdk.git - cd emsdk/ - ./emsdk install latest - cd - -fi - -# activate emsdk -./emsdk/emsdk activate latest -source ./emsdk/emsdk_env.sh - # build oscar (only) with emsdk # # - this is a custom build for now because I can't build all dependencies diff --git a/scripts/build_debian-buster.sh b/scripts/build_ubuntu-20.04.sh old mode 100755 new mode 100644 similarity index 62% rename from scripts/build_debian-buster.sh rename to scripts/build_ubuntu-20.04.sh index 7585614b92..9313d9dfd2 --- a/scripts/build_debian-buster.sh +++ b/scripts/build_ubuntu-20.04.sh @@ -1,9 +1,9 @@ #!/usr/bin/env bash # -# `build_debian-buster.sh`: performs an end-to-end build of OpenSim Creator +# `build_ubuntu-20.04.sh`: performs an end-to-end build of OpenSim Creator # on Debian platforms # -# usage (must be ran in repository root): `bash build_debian-buster.sh` +# usage (must be ran in repository root): `bash build_ubuntu-20.04.sh` # error out of this script if it fails for any reason set -xeuo pipefail @@ -35,14 +35,6 @@ OSC_BUILD_CONCURRENCY=${OSC_BUILD_CONCURRENCY:-1} # package package everything into a .deb installer OSC_BUILD_TARGET=${OSC_BUILD_TARGET:-package} -# set this if you want to skip installing system-level deps -# -# OSC_SKIP_APT - -# set this if you want to build the docs -# -# OSC_BUILD_DOCS - set +x echo "----- starting build -----" echo "" @@ -53,49 +45,13 @@ echo " OSC_DEPS_BUILD_TYPE = ${OSC_DEPS_BUILD_TYPE}" echo " OSC_BUILD_TYPE = ${OSC_BUILD_TYPE}" echo " OSC_BUILD_CONCURRENCY = ${OSC_BUILD_CONCURRENCY}" echo " OSC_BUILD_TARGET = ${OSC_BUILD_TARGET}" -echo " OSC_SKIP_APT = ${OSC_SKIP_APT:-OFF}" -echo " OSC_BUILD_DOCS = ${OSC_BUILD_DOCS:-OFF}" echo "" set -x - -echo "----- printing system (pre-dependency install) info -----" +echo "----- printing system info -----" df # print disk usage ls -la . # print build dir contents uname -a # print distro details - - -echo "----- ensuring all submodules are up-to-date -----" -git submodule update --init --recursive - - -if [[ -z ${OSC_SKIP_APT:+x} ]]; then - echo "----- getting system-level dependencies -----" - - # if root is running this script then do not use `sudo` (some distros - # do not have 'sudo' available) - if [[ "${UID}" == 0 ]]; then - sudo='' - else - sudo='sudo' - fi - - ${sudo} apt-get update - - # osc: main dependencies - ${sudo} apt-get install -y build-essential cmake libgtk-3-dev liblapack-dev - - # osc: docs dependencies (if OSC_BUILD_DOCS is set) - [[ ! -z "${OSC_BUILD_DOCS:+z}" ]] && ${sudo} apt-get install python3 python3-pip - [[ ! -z "${OSC_BUILD_DOCS:+z}" ]] && ${sudo} pip3 install -r docs/requirements.txt - - echo "----- finished getting system-level dependencies -----" -else - echo "----- skipping getting system-level dependencies (OSC_SKIP_APT is set) -----" -fi - - -echo "----- printing system (post-dependency install) info -----" cc --version c++ --version cmake --version diff --git a/scripts/build_ubuntu-20.04_debugging.sh b/scripts/build_ubuntu-20.04_debugging.sh old mode 100644 new mode 100755 index ea9e8f02f7..a6cfc1635f --- a/scripts/build_ubuntu-20.04_debugging.sh +++ b/scripts/build_ubuntu-20.04_debugging.sh @@ -1,22 +1,8 @@ #!/usr/bin/env bash -# setup system dependencies -sudo apt update -sudo apt install -y cmake pkg-config libgtk-3-dev libblas-dev liblapack-dev clang-11 clang-tidy-11 libstdc++-10-dev export CC=clang-11 export CXX=clang++-11 - -# install a newer cmake than the one supplied by apt -sudo apt install -y libssl-dev # OpenSSL headers, for cmake -wget https://github.com/Kitware/CMake/releases/download/v3.31.2/cmake-3.31.2.tar.gz -tar xvf cmake-3.31.2.tar.gz -cd cmake-3.31.2 -./bootstrap -make -j$(nproc) && sudo make install -cd - - export CLANG_TIDY=echo # clang-tidy-11 is screwed on ubuntu20.04 -git clone --recurse-submodules https://github.com/ComputationalBiomechanicsLab/opensim-creator -cd opensim-creator + ./scripts/build_linux_debugging.sh diff --git a/scripts/build_ubuntu-24.04_debugging.sh b/scripts/build_ubuntu-24.04_debugging.sh new file mode 100755 index 0000000000..459d0f17a6 --- /dev/null +++ b/scripts/build_ubuntu-24.04_debugging.sh @@ -0,0 +1,6 @@ +#!/usr/bin/env bash + +# WSL2 on Ubuntu24: ln -sf /mnt/wslg/runtime-dir/wayland-* $XDG_RUNTIME_DIR/ + +./scripts/build_linux_debugging.sh + diff --git a/scripts/build_windows-rapid.bat b/scripts/build_windows-rapid.bat old mode 100644 new mode 100755 index 16a2bc2da6..c84624f4a8 --- a/scripts/build_windows-rapid.bat +++ b/scripts/build_windows-rapid.bat @@ -8,4 +8,4 @@ cmake -S third_party -B osc-dependencies-build -G Ninja -DCMAKE_BUILD_TYPE=%OSC_ cmake --build osc-dependencies-build --config %OSC_BUILD_TYPE% -v cmake -S . -B osc-build -G Ninja -DCMAKE_BUILD_TYPE=%OSC_BUILD_TYPE% -DCMAKE_PREFIX_PATH=%cd%/osc-dependencies-install cmake --build osc-build --config %OSC_BUILD_TYPE% -v -ctest --test-dir osc-build +ctest --test-dir osc-build -j%NUMBER_OF_PROCESSORS% diff --git a/scripts/build_windows.py b/scripts/build_windows.py old mode 100644 new mode 100755 index 077f339d94..b370cb60e6 --- a/scripts/build_windows.py +++ b/scripts/build_windows.py @@ -3,7 +3,7 @@ # `build_windows`: performs an end-to-end build of OpenSim Creator on # the Windows platform # -# usage (must be ran in repository root): `python3 build_windows.py` +# usage (must be ran in repository root): `python3 scripts/build_windows.py` import argparse import logging @@ -35,7 +35,6 @@ def __init__( build_concurrency=multiprocessing.cpu_count(), build_target="package", build_docs="OFF", - build_skip_submodules=False, build_skip_osc=False): self.base_build_type = os.getenv("OSC_BASE_BUILD_TYPE", base_build_type) @@ -46,7 +45,6 @@ def __init__( self.build_docs = _is_truthy_envvar(os.getenv("OSC_BUILD_DOCS", build_docs)) self.generator_flags = f'-G"Visual Studio 17 2022" -A x64' self.build_dir = build_dir - self.skip_submodules = build_skip_submodules self.skip_osc = build_skip_osc def __repr__(self): @@ -79,21 +77,6 @@ def log_build_params(conf: BuildConfiguration): with Section("logging build params"): logging.info(conf) -def ensure_submodules_are_up_to_date(conf: BuildConfiguration): - if (conf.skip_submodules): - logging.info('--skip-submodules was provided: skipping git submodule update') - return - - with Section("update git submodules"): - _run("git submodule update --init --recursive") - -def install_system_dependencies(conf: BuildConfiguration): - with Section("install system dependencies"): - if conf.build_docs: - _run("pip install -r docs/requirements.txt") - else: - logging.info("skipping pip install: this build isn't building docs") - def build_osc_dependencies(conf: BuildConfiguration): with Section("build osc dependencies"): _run( @@ -142,7 +125,6 @@ def main(): # parse CLI args parser = argparse.ArgumentParser() parser.add_argument('--jobs', '-j', type=int, default=conf.concurrency) - parser.add_argument('--skip-submodules', help='skip running git submodule update --init --recursive', default=conf.skip_submodules, action='store_true') parser.add_argument('--skip-osc', help='skip building OSC (handy if you plan on building OSC via Visual Studio)', default=conf.skip_osc, action='store_true') parser.add_argument('--build-dir', '-B', help='build binaries in the specified directory', type=str, default=conf.build_dir) parser.add_argument('--build-type', help='the type of build to produce (CMake string: Debug, Release, RelWithDebInfo, etc.)', type=str, default=conf.base_build_type) @@ -150,14 +132,11 @@ def main(): # overwrite build configuration with any CLI args args = parser.parse_args() conf.concurrency = args.jobs - conf.skip_submodules = args.skip_submodules conf.skip_osc = args.skip_osc conf.build_dir = args.build_dir conf.base_build_type = args.build_type log_build_params(conf) - ensure_submodules_are_up_to_date(conf) - install_system_dependencies(conf) build_osc_dependencies(conf) build_osc(conf) diff --git a/scripts/generate_icon_files.py b/scripts/generate_icon_files.py old mode 100644 new mode 100755 diff --git a/scripts/get_osc_github_download_counts.py b/scripts/get_osc_github_download_counts.py old mode 100644 new mode 100755 index dd9f5b5809..f82bd6c8cd --- a/scripts/get_osc_github_download_counts.py +++ b/scripts/get_osc_github_download_counts.py @@ -1,12 +1,12 @@ #!/usr/bin/env python3 -# `get_osc_github_download_counts`: print the GitHub download counters for +# `get_osc_github_download_counts.py`: print the GitHub download counters for # all releases of OpenSim Creator to the standard output import datetime import itertools import functools -import requests +import requests # must be installed via pip: `pip install requests` import re class Row: diff --git a/scripts/macos_check-dependencies.py b/scripts/macos_check-dependencies.py index b6c4e1e7ed..6779c62cd2 100755 --- a/scripts/macos_check-dependencies.py +++ b/scripts/macos_check-dependencies.py @@ -1,5 +1,12 @@ #!/usr/bin/env python3 +# `macos_check_dependencies.py`: checks that the given binary only (natively) +# depends on common, or system-level, binaries. +# +# This is useful for double-checking that the native buildsystem isn't sneakliy +# linking to something installed somewhere else on the build machine, which might +# prevent end-users from being able to run the binary. + import argparse import subprocess diff --git a/scripts/macos_check-sdk.py b/scripts/macos_check-sdk.py index 14141f5da3..343309c9ed 100755 --- a/scripts/macos_check-sdk.py +++ b/scripts/macos_check-sdk.py @@ -1,5 +1,9 @@ #!/usr/bin/env python3 +# `macos_check-sdk.py`: checks that the specified binary uses the +# specified version of the MacOS SDK. This is handy for double-checking +# that the build system is using the correct SDK, which can impact +# which versions of MacOS can run the binary. import argparse import subprocess diff --git a/scripts/setup_emscripten.sh b/scripts/setup_emscripten.sh new file mode 100644 index 0000000000..70b477e858 --- /dev/null +++ b/scripts/setup_emscripten.sh @@ -0,0 +1,17 @@ +#!/usr/bin/env bash +# +# `setup_emscripten.sh`: installs any necessary dependencies for +# building OpenSim Creator via emscripten. + +set -xeuo pipefail +# install emsdk +if [ ! -d emsdk ]; then + git clone https://github.com/emscripten-core/emsdk.git + cd emsdk/ + ./emsdk install latest + cd - +fi + +# note: the caller should activate/source emsdk like this +# ./emsdk/emsdk activate latest +# source ./emsdk/emsdk_env.sh diff --git a/scripts/setup_ubuntu-20.04.sh b/scripts/setup_ubuntu-20.04.sh new file mode 100755 index 0000000000..0141c66fda --- /dev/null +++ b/scripts/setup_ubuntu-20.04.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash + +# setup system dependencies +sudo apt install -y cmake pkg-config libgtk-3-dev libblas-dev liblapack-dev clang-11 clang-tidy-11 libstdc++-10-dev + +# install a newer cmake than the one supplied by apt +sudo apt install -y libssl-dev # OpenSSL headers, for cmake +wget https://github.com/Kitware/CMake/releases/download/v3.31.2/cmake-3.31.2.tar.gz +tar xvf cmake-3.31.2.tar.gz +cd cmake-3.31.2 +./bootstrap +make -j$(nproc) && sudo make install +cd - + diff --git a/scripts/build_ubuntu-24-WSL2_debugging.sh b/scripts/setup_ubuntu-24.04.sh old mode 100644 new mode 100755 similarity index 57% rename from scripts/build_ubuntu-24-WSL2_debugging.sh rename to scripts/setup_ubuntu-24.04.sh index 76d22dcc95..ed19ba25e3 --- a/scripts/build_ubuntu-24-WSL2_debugging.sh +++ b/scripts/setup_ubuntu-24.04.sh @@ -3,7 +3,4 @@ # WSL2 on Ubuntu24: ln -sf /mnt/wslg/runtime-dir/wayland-* $XDG_RUNTIME_DIR/ sudo apt install clang clang++ clang-tidy cmake pkg-config libgtk-3-dev libblas-dev liblapack-dev -git clone --recurse-submodules https://github.com/ComputationalBiomechanicsLab/opensim-creator -cd opensim-creator -./scripts/build_linux_debugging.sh