diff --git a/docker/README.md b/docker/README.md index 315dd9c..3c1c5a4 100644 --- a/docker/README.md +++ b/docker/README.md @@ -1,8 +1,6 @@ Dockerfiles to generate the boost_superproject_build images. -While there should preferably be only one official docker image to build the boost superproject, that image is currently in transition, going from python2 to python3. And, as time passes, the best choice of a base docker image also changes. Therefore, let's keep a few copies of the Dockerfile available. After things stabilize, older copies can be removed. - | file | description | hub.docker.com image | comments | | ---- | ----------- | -------------------- | -------- | | docker/python2/xenial/Dockerfile | Python2 on Xenial | cppalliance/boost_superproject_build:build-deps-5 | minor asciidoctor syntax problems, attributed to ruby version. Don't use Xenial. | @@ -12,3 +10,9 @@ While there should preferably be only one official docker image to build the boo | docker/python3/focal/Dockerfile | Python3 on Focal | cppalliance/boost_superproject_build:20.04-v2 | 2023-09 Install rclone and aws cli | | docker/python3/focal/Dockerfile | Python3 on Focal | cppalliance/boost_superproject_build:20.04-v3 | 2023-10 installs nodejs, npm for antora builds | | docker/python3/focal/Dockerfile | Python3 on Focal | cppalliance/boost_superproject_build:20.04-v4 | 2023-10 asciidoctor-diagram, asciidoctor-multipage, @mermaid-js/mermaid-cli | +| docker/python3/jammy/Dockerfile | Python3 on Jammy | cppalliance/boost_superproject_build:22.04-v1 | 2024-05, with autocancel workflows | +| docker/python3/noble/Dockerfile | Python3 on Noble | cppalliance/boost_superproject_build:24.04-v1 | 2024-10 | +| docker/python3/noble/Dockerfile | Python3 on Noble | cppalliance/boost_superproject_build:24.04-v2 | 2024-11 Include few more package updates. used in Jenkins | +| docker/python3/noble/Dockerfile | Python3 on Noble | cppalliance/boost_superproject_build:24.04-v3 | 2024-12 Update many pip/gem packages | + +See [technical-notes.md](technical-notes.md) for development details. diff --git a/docker/python3/noble/date.txt b/docker/python3/noble/date.txt deleted file mode 100644 index 00f8c71..0000000 --- a/docker/python3/noble/date.txt +++ /dev/null @@ -1 +0,0 @@ -Fri Aug 30 14:14:32 UTC 2024 diff --git a/docker/python3/noble/v1/Dockerfile b/docker/python3/noble/v1/Dockerfile new file mode 100644 index 0000000..399f653 --- /dev/null +++ b/docker/python3/noble/v1/Dockerfile @@ -0,0 +1,115 @@ +FROM ubuntu:noble +LABEL maintainer="samuel.d.darwin@gmail.com" + +WORKDIR /root + +RUN set -xe \ + && apt-get update \ + && DEBIAN_FRONTEND="noninteractive" apt-get install -y tzdata \ + && DEBIAN_FRONTEND=noninteractive apt-get install -y locales \ + && apt-get install -y \ + apt-transport-https \ + bison \ + ca-certificates \ + clang \ + cmake \ + curl \ + default-jre-headless \ + docbook \ + docbook-xml \ + docbook-xsl \ + docutils-common \ + docutils-doc \ + dos2unix \ + flex \ + ghostscript \ + git \ + graphviz \ + jq \ + libsaxonhe-java \ + openssh-client \ + p7zip-full \ + python-is-python3 \ + python3-pip \ + python3-venv \ + rsync \ + ruby \ + software-properties-common \ + sshpass \ + sudo \ + texlive \ + texlive-latex-extra \ + unzip \ + vim \ + wget \ + xsltproc \ + && sed -i -e 's/# en_US.UTF-8 UTF-8/en_US.UTF-8 UTF-8/' /etc/locale.gen \ + && dpkg-reconfigure --frontend=noninteractive locales \ + && update-locale LANG=en_US.UTF-8 \ + && python3 -m venv /opt/venvboostdocs \ + && export PATH=/opt/venvboostdocs/bin:$PATH \ + && mkdir /tmp/aws_cli_install && cd /tmp/aws_cli_install && curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" && unzip awscliv2.zip && ./aws/install && cd /root && rm -rf /tmp/aws_cli_install \ + && mkdir /tmp/rclone_install && cd /tmp/rclone_install && wget https://downloads.rclone.org/v1.63.1/rclone-v1.63.1-linux-amd64.deb && dpkg -i rclone-v1.63.1-linux-amd64.deb && cd /root && rm -rf /tmp/rclone_install \ + && curl -s -S --retry 10 -L -o gh_2.40.1_linux_amd64.deb https://github.com/cli/cli/releases/download/v2.40.1/gh_2.40.1_linux_amd64.deb \ + && dpkg -i gh_2.40.1_linux_amd64.deb \ + && gem install public_suffix --version 5.0.3 \ + && gem install css_parser --version 1.16.0 \ + && git clone -b 'Release_1_9_5' --depth 1 https://github.com/doxygen/doxygen.git \ + && cd doxygen \ + && cmake -H. -Bbuild -DCMAKE_BUILD_TYPE=Release \ + && cd build \ + && make install \ + && cd && rm -rf doxygen* \ + && curl -s -S --retry 10 -L -o saxonhe.zip https://sourceforge.net/projects/saxon/files/Saxon-HE/9.9/SaxonHE9-9-1-4J.zip/download \ + && unzip saxonhe.zip \ + && rm /usr/share/java/Saxon-HE.jar \ + && cp saxon9he.jar /usr/share/java/Saxon-HE.jar \ + && rm -r * \ + && mkdir build && cd build \ + && curl -s -S --retry 10 -L -o rapidxml.zip http://sourceforge.net/projects/rapidxml/files/latest/download \ + && unzip -n -d rapidxml rapidxml.zip \ + && pip3 install setuptools \ + && pip3 install docutils==0.19 \ + && pip3 install Jinja2==3.1.2 \ + && pip3 install MarkupSafe==2.1.1 \ + && pip3 install Sphinx==5.2.1 \ + && pip3 install git+https://github.com/pfultz2/sphinx-boost@8ad7d424c6b613864976546d801439c34a27e3f6 \ + && pip3 install myst-parser==0.18.1 \ + && curl -s -S --retry 10 -L -o docbook-xml.zip http://www.docbook.org/xml/4.5/docbook-xml-4.5.zip \ + && unzip -n -d docbook-xml docbook-xml.zip \ + && curl -s -S --retry 10 -L -o docbook-xsl.zip https://sourceforge.net/projects/docbook/files/docbook-xsl/1.79.1/docbook-xsl-1.79.1.zip/download \ + && unzip -n -d docbook-xsl docbook-xsl.zip \ + && gem install asciidoctor --version 2.0.17 \ + && gem install asciidoctor-diagram --version 2.2.14 \ + && gem install asciidoctor-multipage --version 0.0.18 \ + && gem install rouge --version 4.0.0 \ + && gem install pygments.rb --version 2.3.0 \ + && pip3 install Pygments==2.13.0 \ + && pip3 install https://github.com/bfgroup/jam_pygments/archive/master.zip \ + && pip3 install future==1.0.0 \ + && pip3 install six==1.14.0 \ + && chmod -R 777 /opt/venvboostdocs \ + && rm -rf /var/lib/apt/lists/* \ + && apt-get clean -y + +ENV PATH /opt/venvboostdocs/bin:$PATH +ENV NODE_VERSION=18.18.1 +RUN curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.39.5/install.sh | bash +ENV NVM_DIR=/root/.nvm +RUN . "$NVM_DIR/nvm.sh" && nvm install ${NODE_VERSION} +RUN . "$NVM_DIR/nvm.sh" && nvm use v${NODE_VERSION} +RUN . "$NVM_DIR/nvm.sh" && nvm alias default v${NODE_VERSION} +ENV PATH="/root/.nvm/versions/node/v${NODE_VERSION}/bin/:${PATH}" +RUN node --version +RUN npm --version +RUN npm install -g gulp-cli@2.3.0 +RUN npm install -g @mermaid-js/mermaid-cli@10.5.1 +# Temporary zip fix 1. Not working. +# RUN wget http://launchpadlibrarian.net/740831076/zip_3.0-14_amd64.deb && sudo dpkg -i zip_3.0-14_amd64.deb && rm zip_3.0-14_amd64.deb +# Temporary zip fix 2: +COPY zip /usr/bin/zip +ENV LC_ALL en_US.UTF-8 +ENV LANG en_US.UTF-8 +ENV LANGUAGE en_US.UTF-8 + +# public_suffix and css_parser using versions that support ruby 2.5 diff --git a/docker/python3/noble/README.md b/docker/python3/noble/v1/README.md similarity index 100% rename from docker/python3/noble/README.md rename to docker/python3/noble/v1/README.md diff --git a/docker/python3/noble/buildimage.sh b/docker/python3/noble/v1/buildimage.sh similarity index 100% rename from docker/python3/noble/buildimage.sh rename to docker/python3/noble/v1/buildimage.sh diff --git a/docker/python3/noble/buildrelease.sh b/docker/python3/noble/v1/buildrelease.sh similarity index 100% rename from docker/python3/noble/buildrelease.sh rename to docker/python3/noble/v1/buildrelease.sh diff --git a/docker/python3/noble/zip b/docker/python3/noble/v1/zip similarity index 100% rename from docker/python3/noble/zip rename to docker/python3/noble/v1/zip diff --git a/docker/python3/noble/Dockerfile b/docker/python3/noble/v2/Dockerfile similarity index 100% rename from docker/python3/noble/Dockerfile rename to docker/python3/noble/v2/Dockerfile diff --git a/docker/python3/noble/v2/README.md b/docker/python3/noble/v2/README.md new file mode 100644 index 0000000..5440218 --- /dev/null +++ b/docker/python3/noble/v2/README.md @@ -0,0 +1,41 @@ + +Earlier notes: + +In the Dockerfile, docbook is being installed twice. As apt packages: + +``` + docbook \ + docbook-xml \ + docbook-xsl \ +``` + +and zip files: +``` + && curl -s -S --retry 10 -L -o docbook-xml.zip http://www.docbook.org/xml/4.5/docbook-xml-4.5.zip \ + && unzip -n -d docbook-xml docbook-xml.zip \ + && curl -s -S --retry 10 -L -o docbook-xsl.zip https://sourceforge.net/projects/docbook/files/docbook-xsl/1.79.1/docbook-xsl-1.79.1.zip/download \ + && unzip -n -d docbook-xsl docbook-xsl.zip +``` + +There are boostbook warnings if DOCBOOK_XSL_DIR and DOCBOOK_DTD_DIR aren't specified. build_docs is setting those variables, pointing to the unzipped location. +The main release script does not set those variables. It is using the default. Which might be the apt packages. +Versions are approximately the same currently between packages and downloads. + +--- + +2023 note. Focal had pinned these gems to support Ruby 2.5 + +``` + && gem install public_suffix --version 4.0.7 \ + && gem install css_parser --version 1.12.0 \ +``` + +In Jammy, it may be unnecessary. But, continuing to track these gems. + +``` + && gem install public_suffix --version 5.0.3 \ + && gem install css_parser --version 1.16.0 \ +``` + +In the near future they can return to being unpinned. + diff --git a/docker/python3/noble/v2/buildimage.sh b/docker/python3/noble/v2/buildimage.sh new file mode 100755 index 0000000..f5b0bd3 --- /dev/null +++ b/docker/python3/noble/v2/buildimage.sh @@ -0,0 +1,5 @@ +#!/bin/bash + +# update the image name as necessary. +imagename="cppalliance/boost_superproject_build:24.04-v2" +docker build --progress=plain -t $imagename . 2>&1 | tee /tmp/output.txt diff --git a/docker/python3/noble/v2/buildrelease.sh b/docker/python3/noble/v2/buildrelease.sh new file mode 100755 index 0000000..a648711 --- /dev/null +++ b/docker/python3/noble/v2/buildrelease.sh @@ -0,0 +1,26 @@ +#!/bin/bash + +# Run this inside the container + +set -ex + +cd $HOME + +boostbranch=develop +if [ ! -d project ]; then + git clone https://github.com/boostorg/boost project + cd project + # "git checkout" can also checkout commits + git checkout $boostbranch + cd .. +fi + +export CIRCLECI=true +export CIRCLE_BRANCH=develop +export CIRCLE_WORKING_DIRECTORY=~/project + +wget "https://raw.githubusercontent.com/boostorg/release-tools/master/ci_boost_common.py" -P ${HOME} +wget "https://raw.githubusercontent.com/boostorg/release-tools/master/ci_boost_release.py" -P ${HOME} + +python3 ${HOME}/ci_boost_release.py checkout_post +EOL=LF python3 ${HOME}/ci_boost_release.py test_override diff --git a/docker/python3/noble/v3/Dockerfile b/docker/python3/noble/v3/Dockerfile new file mode 100644 index 0000000..c5ea81d --- /dev/null +++ b/docker/python3/noble/v3/Dockerfile @@ -0,0 +1,105 @@ +FROM ubuntu:noble +LABEL maintainer="samuel.d.darwin@gmail.com" + +WORKDIR /root +RUN mkdir -p /tmp/build +COPY requirements.txt /tmp/build/requirements.txt +COPY Gemfile /tmp/build/Gemfile +ARG GH_CLI_VERSION=2.63.2 +ARG RCLONE_VERSION=1.68.2 +ARG DOXYGEN_VERSION=1_12_0 + +RUN set -xe \ + && apt-get update \ + && DEBIAN_FRONTEND="noninteractive" apt-get install -y tzdata \ + && DEBIAN_FRONTEND=noninteractive apt-get install -y locales \ + && apt-get install -y \ + apt-transport-https \ + bison \ + ca-certificates \ + clang \ + cmake \ + curl \ + default-jre-headless \ + docbook \ + docbook-xml \ + docbook-xsl \ + docutils-common \ + docutils-doc \ + dos2unix \ + flex \ + ghostscript \ + git \ + graphviz \ + jq \ + libsaxonhe-java \ + openssh-client \ + openssl libssl-dev \ + p7zip-full \ + python-is-python3 \ + python3-pip \ + python3-venv \ + rsync \ + ruby \ + software-properties-common \ + sshpass \ + sudo \ + texlive \ + texlive-latex-extra \ + unzip \ + vim \ + wget \ + xsltproc \ + && sed -i -e 's/# en_US.UTF-8 UTF-8/en_US.UTF-8 UTF-8/' /etc/locale.gen \ + && dpkg-reconfigure --frontend=noninteractive locales \ + && update-locale LANG=en_US.UTF-8 \ + && python3 -m venv /opt/venvboostdocs \ + && export PATH=/opt/venvboostdocs/bin:$PATH \ + && mkdir /tmp/aws_cli_install && cd /tmp/aws_cli_install && curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" && unzip awscliv2.zip && ./aws/install && cd /root && rm -rf /tmp/aws_cli_install \ + && mkdir /tmp/rclone_install && cd /tmp/rclone_install && wget https://downloads.rclone.org/v${RCLONE_VERSION}/rclone-v${RCLONE_VERSION}-linux-amd64.deb && dpkg -i rclone-v${RCLONE_VERSION}-linux-amd64.deb && cd /root && rm -rf /tmp/rclone_install \ + && curl -s -S --retry 10 -L -o gh_${GH_CLI_VERSION}_linux_amd64.deb https://github.com/cli/cli/releases/download/v${GH_CLI_VERSION}/gh_${GH_CLI_VERSION}_linux_amd64.deb \ + && dpkg -i gh_${GH_CLI_VERSION}_linux_amd64.deb \ + && git clone -b "Release_${DOXYGEN_VERSION}" --depth 1 https://github.com/doxygen/doxygen.git \ + && cd doxygen \ + && cmake -H. -Bbuild -DCMAKE_BUILD_TYPE=Release \ + && cd build \ + && make install \ + && cd && rm -rf doxygen* \ + && curl -s -S --retry 10 -L -o saxonhe.zip https://sourceforge.net/projects/saxon/files/Saxon-HE/9.9/SaxonHE9-9-1-4J.zip/download \ + && unzip saxonhe.zip \ + && rm /usr/share/java/Saxon-HE.jar \ + && cp saxon9he.jar /usr/share/java/Saxon-HE.jar \ + && rm -r * \ + && mkdir -p build && cd build \ + && gem install bundler \ + && pip3 install -r /tmp/build/requirements.txt \ + && bundle install --gemfile=/tmp/build/Gemfile \ + && curl -s -S --retry 10 -L -o rapidxml.zip http://sourceforge.net/projects/rapidxml/files/latest/download \ + && unzip -n -d rapidxml rapidxml.zip \ + && curl -s -S --retry 10 -L -o docbook-xml.zip http://www.docbook.org/xml/4.5/docbook-xml-4.5.zip \ + && unzip -n -d docbook-xml docbook-xml.zip \ + && curl -s -S --retry 10 -L -o docbook-xsl.zip https://sourceforge.net/projects/docbook/files/docbook-xsl/1.79.1/docbook-xsl-1.79.1.zip/download \ + && unzip -n -d docbook-xsl docbook-xsl.zip \ + && chmod -R 777 /opt/venvboostdocs \ + && rm -rf /var/lib/apt/lists/* \ + && apt-get clean -y + +ENV PATH /opt/venvboostdocs/bin:$PATH +ENV NODE_VERSION=18.18.1 +RUN curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.39.5/install.sh | bash +ENV NVM_DIR=/root/.nvm +RUN . "$NVM_DIR/nvm.sh" && nvm install ${NODE_VERSION} +RUN . "$NVM_DIR/nvm.sh" && nvm use v${NODE_VERSION} +RUN . "$NVM_DIR/nvm.sh" && nvm alias default v${NODE_VERSION} +ENV PATH="/root/.nvm/versions/node/v${NODE_VERSION}/bin/:${PATH}" +RUN node --version +RUN npm --version +RUN npm install -g gulp-cli@2.3.0 +RUN npm install -g @mermaid-js/mermaid-cli@10.5.1 +# For other CI jobs: +RUN groupadd -g 150 jenkins +RUN useradd jenkins -u 150 -g 150 -m -s /bin/bash +RUN echo "ALL ALL=(ALL) NOPASSWD:ALL" > /etc/sudoers.d/all +ENV LC_ALL en_US.UTF-8 +ENV LANG en_US.UTF-8 +ENV LANGUAGE en_US.UTF-8 diff --git a/docker/python3/noble/v3/Gemfile b/docker/python3/noble/v3/Gemfile new file mode 100644 index 0000000..5d49ef3 --- /dev/null +++ b/docker/python3/noble/v3/Gemfile @@ -0,0 +1,7 @@ +source 'http://rubygems.org' + +gem 'asciidoctor',' 2.0.23' +gem 'asciidoctor-diagram', '2.3.1' +gem 'asciidoctor-multipage', '0.0.19' +gem 'rouge', '4.5.1' +gem 'pygments.rb', '3.0.0' diff --git a/docker/python3/noble/v3/README.md b/docker/python3/noble/v3/README.md new file mode 100644 index 0000000..b9df178 --- /dev/null +++ b/docker/python3/noble/v3/README.md @@ -0,0 +1,2 @@ + +Refer to the README.md in the docker/ folder. diff --git a/docker/python3/noble/v3/buildimage.sh b/docker/python3/noble/v3/buildimage.sh new file mode 100755 index 0000000..31f8618 --- /dev/null +++ b/docker/python3/noble/v3/buildimage.sh @@ -0,0 +1,6 @@ +#!/bin/bash + +# update the image name as necessary. +imagename="cppalliance/boost_superproject_build:24.04-v3" +# docker build --progress=plain -t $imagename . 2>&1 | tee /tmp/output.txt +docker build -t $imagename . 2>&1 | tee /tmp/output2.txt diff --git a/docker/python3/noble/v3/buildrelease.sh b/docker/python3/noble/v3/buildrelease.sh new file mode 100755 index 0000000..a648711 --- /dev/null +++ b/docker/python3/noble/v3/buildrelease.sh @@ -0,0 +1,26 @@ +#!/bin/bash + +# Run this inside the container + +set -ex + +cd $HOME + +boostbranch=develop +if [ ! -d project ]; then + git clone https://github.com/boostorg/boost project + cd project + # "git checkout" can also checkout commits + git checkout $boostbranch + cd .. +fi + +export CIRCLECI=true +export CIRCLE_BRANCH=develop +export CIRCLE_WORKING_DIRECTORY=~/project + +wget "https://raw.githubusercontent.com/boostorg/release-tools/master/ci_boost_common.py" -P ${HOME} +wget "https://raw.githubusercontent.com/boostorg/release-tools/master/ci_boost_release.py" -P ${HOME} + +python3 ${HOME}/ci_boost_release.py checkout_post +EOL=LF python3 ${HOME}/ci_boost_release.py test_override diff --git a/docker/python3/noble/v3/requirements.txt b/docker/python3/noble/v3/requirements.txt new file mode 100644 index 0000000..bce5cb0 --- /dev/null +++ b/docker/python3/noble/v3/requirements.txt @@ -0,0 +1,14 @@ +Jinja2==3.1.4 +MarkupSafe==3.0.2 +Pygments==2.18.0 +Sphinx==5.2.1 +# Sphinx==8.1.3 # when boostorg/gil and boostorg/python are ready +docutils==0.19 +# docutils==0.21.2 # when boostorg/gil and boostorg/python are ready +future==1.0.0 +git+https://github.com/pfultz2/sphinx-boost@8ad7d424c6b613864976546d801439c34a27e3f6 +https://github.com/bfgroup/jam_pygments/archive/master.zip +myst-parser==0.18.1 +# myst-parser==4.0.0 # # when boostorg/gil and boostorg/python are ready +setuptools==75.6.0 +six==1.17.0 diff --git a/docker/technical-notes.md b/docker/technical-notes.md new file mode 100644 index 0000000..0244c2a --- /dev/null +++ b/docker/technical-notes.md @@ -0,0 +1,26 @@ + +2024-12-10 + +In the Dockerfile, docbook is being installed twice. As apt packages: + +``` + docbook \ + docbook-xml \ + docbook-xsl \ +``` + +and zip files: +``` + && curl -s -S --retry 10 -L -o docbook-xml.zip http://www.docbook.org/xml/4.5/docbook-xml-4.5.zip \ + && unzip -n -d docbook-xml docbook-xml.zip \ + && curl -s -S --retry 10 -L -o docbook-xsl.zip https://sourceforge.net/projects/docbook/files/docbook-xsl/1.79.1/docbook-xsl-1.79.1.zip/download \ + && unzip -n -d docbook-xsl docbook-xsl.zip +``` + +Versions are approximately the same currently between packages and downloads. + +When release-tools executes, /root/build/site-config.jam points to the copies in /root/build so the apt packages are ignored. + +When build_docs scripts run, there are boostbook warnings if DOCBOOK_XSL_DIR and DOCBOOK_DTD_DIR aren't specified. build_docs is setting those variables, pointing to the unzipped location in BOOST_ROOT/build. + +So in both cases, the zip files are used instead of the apt packages. At the moment this seems acceptable, without a reason to modify it.