Merge pull request #335 from pmienk/master

CI cpu flag detection and optional usage, dev script reduction of make clean, docker refinements.

docker/README.md

@@ -65,5 +65,9 @@ docker compose --env-file bs-linux.env --file bs-linux.yml up -t 3600 -d
 
 ##### Terminating container
 ```
-docker compose --env-file bs-linux.env --file bs-linux.yml down -t 3600 -v
+docker compose --env-file bs-linux.env --file bs-linux.yml down -t 3600
 ```
+
+Note that `-v` would remove listed volumes.
+In the case of `bs`, this includes blockchain storage.
+

docker/bs-linux.Dockerfile

@@ -1,7 +1,7 @@
 FROM alpine:latest AS build
 
 ENV OPTIMIZATION="-O3"
-ENV BUILD_DEPS="build-base linux-headers gcc make autoconf automake libtool pkgconf git wget bash"
+ENV BUILD_DEPS="build-base linux-headers g++ make autoconf automake libtool pkgconf git wget bash"
 
 ENV CFLAGS="${OPTIMIZATION}"
 ENV CXXFLAGS="${OPTIMIZATION}"
@@ -14,22 +14,24 @@ RUN mkdir -p /build/src /build/obj /build/prefix
 COPY developer_setup.sh /build
 COPY src/ /build/src
 
-RUN /build/developer_setup.sh \
+WORKDIR /build
+
+RUN ./developer_setup.sh \
     --build-target=dependencies \
     --build-src-dir=/build/src \
     --build-obj-dir=/build/obj \
     --prefix=/build/prefix \
     --build-mode=configure \
     --disable-shared \
     --enable-static \
-    --enable-isystem \
-    --without-consensus \
     --with-icu \
     --build-icu \
     --build-boost \
     --build-zmq
 
-RUN /build/developer_setup.sh \
+ENV BOOST_ROOT="/build/prefix"
+
+RUN ./developer_setup.sh \
     --build-target=libbitcoin \
     --build-src-dir=/build/src \
     --build-obj-dir=/build/obj \
@@ -40,11 +42,9 @@ RUN /build/developer_setup.sh \
     --enable-isystem \
     --without-consensus \
     --with-icu \
-    --build-icu \
-    --build-boost \
-    --build-zmq
+    --with-boost=/build/prefix
 
-RUN /build/developer_setup.sh \
+RUN ./developer_setup.sh \
     --build-target=project \
     --build-src-dir=/build/src \
     --build-obj-dir=/build/obj \
@@ -55,17 +55,15 @@ RUN /build/developer_setup.sh \
     --enable-isystem \
     --without-consensus \
     --with-icu \
-    --build-icu \
-    --build-boost \
-    --build-zmq
+    --with-boost=/build/prefix
 
 RUN rm -rf /build/src /build/obj
 
 
 
 FROM alpine:latest AS runtime
 
-ENV RUNTIME_DEPS="bash gcc"
+ENV RUNTIME_DEPS="bash libstdc++"
 
 RUN apk update && \
     apk add --update ${RUNTIME_DEPS}

docker/bs-linux.yml

@@ -1,3 +1,7 @@
+volumes:
+  blockchain:
+    name: "bitcoin-blockchain"
+
 networks:
   net:
 
@@ -19,7 +23,7 @@ services:
       - 9093:9093/tcp
       - 9094:9094/tcp
     volumes:
-      - ${STORAGE_BITCOIN_DATA}:/bitcoin/blockchain
+      - blockchain:/bitcoin/blockchain
       - ${STORAGE_BITCOIN_CONF}:/bitcoin/conf
     networks:
       - net

docker/bx-linux.Dockerfile

@@ -1,7 +1,7 @@
 FROM alpine:latest AS build
 
 ENV OPTIMIZATION="-O3"
-ENV BUILD_DEPS="build-base linux-headers gcc make autoconf automake libtool pkgconf git wget bash"
+ENV BUILD_DEPS="build-base linux-headers g++ make autoconf automake libtool pkgconf git wget bash"
 
 ENV CFLAGS="${OPTIMIZATION}"
 ENV CXXFLAGS="${OPTIMIZATION}"
@@ -14,21 +14,24 @@ RUN mkdir -p /build/src /build/obj /build/prefix
 COPY developer_setup.sh /build
 COPY src/ /build/src
 
-RUN /build/developer_setup.sh \
+WORKDIR /build
+
+RUN ./developer_setup.sh \
     --build-target=dependencies \
     --build-src-dir=/build/src \
     --build-obj-dir=/build/obj \
     --prefix=/build/prefix \
     --build-mode=configure \
     --disable-shared \
     --enable-static \
-    --enable-isystem \
     --with-icu \
     --build-icu \
     --build-boost \
     --build-zmq
 
-RUN /build/developer_setup.sh \
+ENV BOOST_ROOT="/build/prefix"
+
+RUN ./developer_setup.sh \
     --build-target=libbitcoin \
     --build-src-dir=/build/src \
     --build-obj-dir=/build/obj \
@@ -38,11 +41,9 @@ RUN /build/developer_setup.sh \
     --enable-static \
     --enable-isystem \
     --with-icu \
-    --build-icu \
-    --build-boost \
-    --build-zmq
+    --with-boost=/build/prefix
 
-RUN /build/developer_setup.sh \
+RUN ./developer_setup.sh \
     --build-target=project \
     --build-src-dir=/build/src \
     --build-obj-dir=/build/obj \
@@ -52,9 +53,7 @@ RUN /build/developer_setup.sh \
     --enable-static \
     --enable-isystem \
     --with-icu \
-    --build-icu \
-    --build-boost \
-    --build-zmq \
+    --with-boost=/build/prefix \
     --without-tests
 
 RUN rm -rf /build/src /build/obj
@@ -63,12 +62,11 @@ RUN rm -rf /build/src /build/obj
 
 FROM alpine:latest AS runtime
 
-ENV RUNTIME_DEPS="gcc"
+ENV RUNTIME_DEPS="libstdc++"
 
 RUN apk update && \
     apk add --update ${RUNTIME_DEPS}
 
 COPY --from=build /build/prefix/bin/bx /bitcoin/bx
 
-WORKDIR /bitcoin
 ENTRYPOINT [ "/bitcoin/bx" ]

generate4.xml

@@ -291,7 +291,7 @@
 	<axis name="secp256k1" value="build" />
         <option autotools="--enable-isystem" />
         <option autotools="--enable-avx2" cmake="-Denable-avx2=on" />
-        <option autotools="--enable-sse4" cmake="-Denable-sse4=on" />
+        <option autotools="--enable-sse41" cmake="-Denable-sse41=on" />
       </job>
 
       <job build="all" 
@@ -315,14 +315,14 @@
         <!-- <axis name="icu" /> -->
         <axis name="secp256k1" value="build" />
         <option autotools="--enable-isystem" />
-        <option autotools="--enable-sse4" cmake="-Denable-sse4=on" />
+        <option autotools="--enable-sse41" cmake="-Denable-sse41=on" />
       </job>
 
       <job build="autotools" 
            system="linux" image="ubuntu-22.04" 
            compiler="gcc" version="11" 
            assert="ndebug" link="static" optimization="size" 
-           coverage="true">
+           coverage="true" detectcpuflags="true">
         <axis name="boost" value="build" />
         <axis name="icu" value="build" />
         <axis name="secp256k1" value="build" />

templates/gsl.ci.yml

@@ -497,9 +497,21 @@ function get_job_coverage(job)
         my.coverage = "cov"
     endif
 
-return my.coverage
+    return my.coverage
 endfunction # get_job_coverage
 
+function get_job_cpuflag_detection(job)
+    define my.job = get_job_cpuflag_detection.job
+    define my.value = "ignore"
+
+    if (defined(my.job.detectcpuflags) & !is_empty(my.job.detectcpuflags) &\
+      (my.job.detectcpuflags = "true"))
+        my.value = "detect"
+    endif
+
+    return my.value
+endfunction # get_job_cpuflag_detection
+
 function get_coverage_exclusions(ci)
     define my.ci = get_coverage_exclusions.ci
 
@@ -578,6 +590,7 @@ endfunction # get_sln_tests
             optimization: "$(get_job_optimization(my.job))"
             assert: "$(get_job_assert(my.job))"
             coverage: "$(get_job_coverage(my.job))"
+            detectcpuflags: "$(get_job_cpuflag_detection(my.job))"
 .       for my.ci.axis by "$(name)" as _axis
             $(_axis.name): "$( get_job_shell_axis_value(my.config, my.integration, my.job, _axis))"
 .       endfor
@@ -650,6 +663,39 @@ endfunction # get_sln_tests
 .
 .endmacro # emit_step_prepare_toolchain
 .
+.macro emit_step_cpuflag_determination(configuration, repository)
+.   define my.config = emit_step_cpuflag_determination.configuration
+.   define my.repository = emit_step_cpuflag_determination.repository
+.
+.   define my.sse41 = defined(my.config.cmake) ?? "-Denable-sse41=on" ? "--enable-sse41"
+.   define my.avx2 = defined(my.config.cmake) ?? "-Denable-avx2=on" ? "--enable-avx2"
+.   define my.avx512 = defined(my.config.cmake) ?? "-Denable-avx512=on" ? "--enable-avx512"
+.   define my.shani = defined(my.config.cmake) ?? "-Denable-shani=on" ? "--enable-shani"
+.
+      - name: Determine CPU flags
+        shell: bash
+        run: |
+          if [[ -n \$(cat /proc/cpuinfo | grep flags | grep " sse4_1 ") ]]; then
+              echo "CPU_SUPPORT_SSE41=$(my.sse41)" >> $GITHUB_ENV
+          fi
+
+          if [[ -n \$(cat /proc/cpuinfo | grep flags | grep " avx " | grep " avx2 ") ]]; then
+              echo "CPU_SUPPORT_AVX2=$(my.avx2)" >> $GITHUB_ENV
+          fi
+
+          if [[ -n \$(cat /proc/cpuinfo | grep flags | grep " avx512bw ") ]]; then
+              echo "CPU_SUPPORT_AVX512=$(my.avx512)" >> $GITHUB_ENV
+          fi
+
+          if [[ -n \$(cat /proc/cpuinfo | grep flags | grep " sha_ni ") ]]; then
+              echo "CPU_SUPPORT_SHANI=$(my.shani)" >> $GITHUB_ENV
+          fi
+
+          if [[ ${{ matrix.detectcpuflags }} == 'detect' ]]; then
+              echo "CPU_SUPPORTED_FLAGS='$CPU_SUPPORT_SSE41 $CPU_SUPPORT_AVX2 $CPU_SUPPORT_AVX512 $CPU_SUPPORT_SHANI'" >> $GITHUB_ENV
+          fi
+.endmacro # emit_step_cpuflag_determination
+.
 .macro emit_step_display_environment(configuration, repository)
 .   define my.config = emit_step_display_environment.configuration
 .   define my.repository = emit_step_display_environment.repository
@@ -739,6 +785,7 @@ endfunction # get_sln_tests
 .   endif
           ${{ env.LINKAGE }}
           ${{ env.ASSERT_NDEBUG }}
+          ${{ env.CPU_SUPPORTED_FLAGS }}
 .       for my.ci.axis as _axis
           ${{ matrix.$(_axis.name) }}
 .       endfor
@@ -861,6 +908,8 @@ endfunction # get_sln_tests
 
 .   emit_step_prepare_toolchain(my.config)
 
+.   emit_step_cpuflag_determination(my.config, my.repository)
+
 .   emit_step_parameter_denormalization(my.config, my.repository)
 
 .   emit_step_display_environment(my.config, my.repository)

templates/gsl.developer_setup.sh

@@ -128,6 +128,10 @@ handle_custom_options()
             display_error ""
             display_help
             exit 1
+        elif [[ ($BUILD_MODE != "build") ]]; then
+            BUILD_PRECLEAN=true
+        else
+            BUILD_PRECLEAN=false
         fi
 
         if [[ ! ($BUILD_OBJ_DIR) ]]; then
@@ -231,7 +235,7 @@ make_project_directory()
     PROJ_CONFIG_DIR=\$(pwd)
 
     if [[ -f "$PROJ_NAME/configure" ]]; then
-        if [[ $BUILD_MODE != "reuse" ]]; then
+        if [[ $BUILD_MODE != "build" ]]; then
             # reconfigure using autoreconf
             autoreconf -i
         fi
@@ -242,7 +246,7 @@ make_project_directory()
 
     push_obj_directory "$PROJ_NAME"
     configure_options "$PROJ_CONFIG_DIR" "$@"
-    make_jobs true "$JOBS"
+    make_jobs $BUILD_PRECLEAN "$JOBS"
 
     if [[ $TEST == true ]]; then
         make_tests "$JOBS"

templates/shared/common_install_shell_artifacts.gsl

@@ -708,7 +708,7 @@ configure_options()
 .
 .macro define_make_tests(has_preclean_option)
 .   define my.make_jobs_preclean = is_true(my.has_preclean_option) \
-        ?? "make_jobs true" ? "make_jobs"
+        ?? "make_jobs $BUILD_PRECLEAN" ? "make_jobs"
 .   define my.make_jobs_noclean = is_true(my.has_preclean_option) \
         ?? "make_jobs false" ? "make_jobs"
 .