build: use zopfli compression to reduce fw size by over 550 bytes

Update scripts to make smaller firmware/delta OTA images (~5%)
Update docker image/debian

.gitlab-ci.yml

@@ -7,7 +7,7 @@ stages:
   - test
   - deploy
 
-image: blockstream/verde@sha256:acd212ebeed9563cc0fdf7f94d03159adfb201342b03f5c0dbf3cc68f986e5a1
+image: blockstream/verde@sha256:391a4337c174f2eb47dc9f31d7e25d19a5f101bbc07fa8a296866c1ee52b8473
 
 test_bip85_rsa_key_gen:
   tags:
@@ -57,6 +57,7 @@ test_configs:
   script:
     - . $HOME/esp/esp-idf/export.sh
     - idf.py all size-components size
+    - source /venv/bin/activate
     - ./tools/fwprep.py build/jade.bin build
     - ./tools/mkindex.py build $(basename $(ls build/*_fw.bin) | cut -d'_' -f1)
     - cp sdkconfig build/

Dockerfile

@@ -1,5 +1,5 @@
-FROM debian:bookworm@sha256:b877a1a3fdf02469440f1768cf69c9771338a875b7add5e80c45b756c92ac20a AS base
-RUN dpkg --add-architecture i386 && apt-get update -qq && apt-get upgrade --no-install-recommends --no-install-suggests -yqq && apt-get install --no-install-recommends --no-install-suggests -yqq git wget libncurses-dev flex bison gperf libffi-dev libssl-dev dfu-util cmake ninja-build build-essential ca-certificates ccache curl make pkg-config python3 python3-dev python3-pip python3-setuptools python3-serial python3-click python3-cryptography python3-future python3-pyparsing python3-pyelftools python3-pkg-resources python3-wheel python3-venv python3-sphinx unzip bluez-tools bluez libusb-1.0-0 clang lld clang-format libglib2.0-dev libpixman-1-dev libgcrypt20-dev virtualenv libslirp-dev gcc-multilib libc6-dev-i386 libsdl2-dev:i386 && apt-get -yqq autoremove && apt-get -yqq clean && rm -rf /var/lib/apt/lists/* /var/cache/* /tmp/*
+FROM debian:bookworm@sha256:321341744acb788e251ebd374aecc1a42d60ce65da7bd4ee9207ff6be6686a62 AS base
+RUN dpkg --add-architecture i386 && apt-get update -qq && apt-get upgrade --no-install-recommends --no-install-suggests -yqq && apt-get install --no-install-recommends --no-install-suggests -yqq git wget libncurses-dev flex bison gperf libffi-dev libssl-dev dfu-util cmake ninja-build build-essential ca-certificates ccache curl make pkg-config python3 python3-dev python3-pip python3-setuptools python3-serial python3-click python3-cryptography python3-future python3-pyparsing python3-pyelftools python3-pkg-resources python3-wheel python3-venv python3-sphinx unzip bluez-tools bluez libusb-1.0-0 clang lld clang-format libglib2.0-dev libpixman-1-dev libgcrypt20-dev virtualenv libslirp-dev gcc-multilib libc6-dev-i386 libsdl2-dev:i386 libcurl4-openssl-dev:i386 && apt-get -yqq autoremove && apt-get -yqq clean && rm -rf /var/lib/apt/lists/* /var/cache/* /tmp/*
 RUN update-alternatives --install /usr/bin/python python /usr/bin/python3 10
 RUN python3 -m pip install --break-system-packages --user pycodestyle
 

Dockerfile.dev

@@ -1,4 +1,4 @@
-FROM blockstream/verde@sha256:acd212ebeed9563cc0fdf7f94d03159adfb201342b03f5c0dbf3cc68f986e5a1
+FROM blockstream/verde@sha256:391a4337c174f2eb47dc9f31d7e25d19a5f101bbc07fa8a296866c1ee52b8473
 
 RUN echo "source /root/esp/esp-idf/export.sh" >> /etc/bash.bashrc
 

Dockerfile.qemu

@@ -1,4 +1,4 @@
-FROM blockstream/verde@sha256:acd212ebeed9563cc0fdf7f94d03159adfb201342b03f5c0dbf3cc68f986e5a1 as builder
+FROM blockstream/verde@sha256:391a4337c174f2eb47dc9f31d7e25d19a5f101bbc07fa8a296866c1ee52b8473 as builder
 
 ARG SDK_CONFIG=configs/sdkconfig_qemu_psram.defaults
 
@@ -12,7 +12,7 @@ RUN ./tools/fwprep.py build/jade.bin build
 ENV PATH=$PATH:/jade/main/qemu
 RUN make-flash-img.sh
 
-FROM debian:bookworm@sha256:b877a1a3fdf02469440f1768cf69c9771338a875b7add5e80c45b756c92ac20a
+FROM debian:bookworm@sha256:321341744acb788e251ebd374aecc1a42d60ce65da7bd4ee9207ff6be6686a62
 
 RUN apt-get update -yqq && apt-get install -yqq libpixman-1-dev libglib2.0-dev libslirp-dev libncurses-dev
 

logo/README.md

@@ -1,7 +1,7 @@
 # Files in this directory are compressed using python
 
 ```
-python -c "import zlib; import sys; open(sys.argv[2], 'wb').write(zlib.compress(open(sys.argv[1], 'rb').read(), 9))" uncompressed_file.bin compressed_file.bin.gz
+python -c "from zopfli import zlib; import sys; open(sys.argv[2], 'wb').write(zlib.compress(open(sys.argv[1], 'rb').read()))" uncompressed_file.bin compressed_file.bin.gz
 ```
 
 # The input file used is the output of GIMP RGB C-Source image dump (file.c) which contains a stucture Picture.

logo/bmp_to_compressed_bgr565_byteswapped.py

@@ -1,4 +1,5 @@
-import zlib
+from zopfli import zlib
+
 import struct
 from PIL import Image
 
@@ -28,7 +29,7 @@ def compress_to_bgr565(bmp_file):
 
     data_len = data_len.to_bytes(2, 'little')
     compressed_data = bytes([width]) + data_len + swapped_data
-    compressed_data = zlib.compress(compressed_data, level=9)
+    compressed_data = zlib.compress(compressed_data)
     compressed_file = bmp_file.replace('.bmp', '.bin.gz')
     with open(compressed_file, "wb") as file:
         file.write(compressed_data)

requirements.txt

@@ -96,3 +96,76 @@ requests==2.26.0 \
     --hash=sha256:6c1246513ecd5ecd4528a0906f910e8f0f9c6b8ec72030dc9fd154dc1a6efd24 \
     --hash=sha256:b8aa58f8cf793ffd8782d3d8cb19e66ef36f7aba4353eec859e74678b01b07a7
 
+# logo/firmware compression
+zopfli==0.2.3.post1 \
+    --hash=sha256:e63d558847166543c2c9789e6f985400a520b7eacc4b99181668b2c3aeadd352 \
+    --hash=sha256:0aa5f90d6298bda02a95bc8dc8c3c19004d5a4e44bda00b67ca7431d857b4b54 \
+    --hash=sha256:0cc20b02a9531559945324c38302fd4ba763311632d0ec8a1a0aa9c10ea363e6 \
+    --hash=sha256:1d8cc06605519e82b16df090e17cb3990d1158861b2872c3117f1168777b81e4 \
+    --hash=sha256:1f990634fd5c5c8ced8edddd8bd45fab565123b4194d6841e01811292650acae \
+    --hash=sha256:2345e713260a350bea0b01a816a469ea356bc2d63d009a0d777691ecbbcf7493 \
+    --hash=sha256:2768c877f76c8a0e7519b1c86c93757f3c01492ddde55751e9988afb7eff64e1 \
+    --hash=sha256:29ea74e72ffa6e291b8c6f2504ce6c146b4fe990c724c1450eb8e4c27fd31431 \
+    --hash=sha256:34a99592f3d9eb6f737616b5bd74b48a589fdb3cb59a01a50d636ea81d6af272 \
+    --hash=sha256:3654bfc927bc478b1c3f3ff5056ed7b20a1a37fa108ca503256d0a699c03bbb1 \
+    --hash=sha256:3657e416ffb8f31d9d3424af12122bb251befae109f2e271d87d825c92fc5b7b \
+    --hash=sha256:37d011e92f7b9622742c905fdbed9920a1d0361df84142807ea2a528419dea7f \
+    --hash=sha256:3827170de28faf144992d3d4dcf8f3998fe3c8a6a6f4a08f1d42c2ec6119d2bb \
+    --hash=sha256:39e576f93576c5c223b41d9c780bbb91fd6db4babf3223d2a4fe7bf568e2b5a8 \
+    --hash=sha256:3a89277ed5f8c0fb2d0b46d669aa0633123aa7381f1f6118c12f15e0fb48f8ca \
+    --hash=sha256:3c163911f8bad94b3e1db0a572e7c28ba681a0c91d0002ea1e4fa9264c21ef17 \
+    --hash=sha256:3f0197b6aa6eb3086ae9e66d6dd86c4d502b6c68b0ec490496348ae8c05ecaef \
+    --hash=sha256:48dba9251060289101343110ab47c0756f66f809bb4d1ddbb6d5c7e7752115c5 \
+    --hash=sha256:4915a41375bdee4db749ecd07d985a0486eb688a6619f713b7bf6fbfd145e960 \
+    --hash=sha256:4c1226a7e2c7105ac31503a9bb97454743f55d88164d6d46bc138051b77f609b \
+    --hash=sha256:4e50ffac74842c1c1018b9b73875a0d0a877c066ab06bf7cccbaa84af97e754f \
+    --hash=sha256:518f1f4ed35dd69ce06b552f84e6d081f07c552b4c661c5312d950a0b764a58a \
+    --hash=sha256:5aad740b4d4fcbaaae4887823925166ffd062db3b248b3f432198fc287381d1a \
+    --hash=sha256:5f272186e03ad55e7af09ab78055535c201b1a0bcc2944edb1768298d9c483a4 \
+    --hash=sha256:5fcfc0dc2761e4fcc15ad5d273b4d58c2e8e059d3214a7390d4d3c8e2aee644e \
+    --hash=sha256:60db20f06c3d4c5934b16cfa62a2cc5c3f0686bffe0071ed7804d3c31ab1a04e \
+    --hash=sha256:615a8ac9dda265e9cc38b2a76c3142e4a9f30fea4a79c85f670850783bc6feb4 \
+    --hash=sha256:6482db9876c68faac2d20a96b566ffbf65ddaadd97b222e4e73641f4f8722fc4 \
+    --hash=sha256:6617fb10f9e4393b331941861d73afb119cd847e88e4974bdbe8068ceef3f73f \
+    --hash=sha256:676919fba7311125244eb0c4393679ac5fe856e5864a15d122bd815205369fa0 \
+    --hash=sha256:6c2d2bc8129707e34c51f9352c4636ca313b52350bbb7e04637c46c1818a2a70 \
+    --hash=sha256:71390dbd3fbf6ebea9a5d85ffed8c26ee1453ee09248e9b88486e30e0397b775 \
+    --hash=sha256:716cdbfc57bfd3d3e31a58e6246e8190e6849b7dbb7c4ce39ef8bbf0edb8f6d5 \
+    --hash=sha256:75a26a2307b10745a83b660c404416e984ee6fca515ec7f0765f69af3ce08072 \
+    --hash=sha256:7be5cc6732eb7b4df17305d8a7b293223f934a31783a874a01164703bc1be6cd \
+    --hash=sha256:7cce242b5df12b2b172489daf19c32e5577dd2fac659eb4b17f6a6efb446fd5c \
+    --hash=sha256:81c341d9bb87a6dbbb0d45d6e272aca80c7c97b4b210f9b6e233bf8b87242f29 \
+    --hash=sha256:89899641d4de97dbad8e0cde690040d078b6aea04066dacaab98e0b5a23573f2 \
+    --hash=sha256:8d5ab297d660b75c159190ce6d73035502310e40fd35170aed7d1a1aea7ddd65 \
+    --hash=sha256:8fbe5bcf10d01aab3513550f284c09fef32f342b36f56bfae2120a9c4d12c130 \
+    --hash=sha256:91a2327a4d7e77471fa4fbb26991c6de4a738c6fc6a33e09bb25f56a870a4b7b \
+    --hash=sha256:95a260cafd56b8fffa679918937401c80bb38e1681c448b988022e4c3610965d \
+    --hash=sha256:96484dc0f48be1c5d7ae9f38ed1ce41e3675fd506b27c11a6607f14b49101e99 \
+    --hash=sha256:9a6aec38a989bad7ddd1ef53f1265699e49e294d08231b5313d61293f3cd6237 \
+    --hash=sha256:9ba214f4f45bec195ee8559651154d3ac2932470b9d91c5715fc29c013349f8c \
+    --hash=sha256:9f4a7ec2770e6af05f5a02733fd3900f30a9cd58e5d6d3727e14c5bcd6e7d587 \
+    --hash=sha256:a1cf720896d2ce998bc8e051d4b4ce0d8bec007aab6243102e8e1d22a0b2fb3f \
+    --hash=sha256:a241a68581d34d67b40c425cce3d1fd211c092f99d9250947824ccba9f491949 \
+    --hash=sha256:a53b18797cdef27e019db595d66c4b077325afe2fd62145953275f53d84ce40c \
+    --hash=sha256:a82fc2dbebe6eb908b9c665e71496f8525c1bc4d2e3a7a7722ef2b128b6227c8 \
+    --hash=sha256:a86eb88e06bd87e1fff31dac878965c26b0c26db59ddcf78bb0379a954b120de \
+    --hash=sha256:aa588b21044f8a74e423d8c8a4c7fc9988501878aacced793467010039c50734 \
+    --hash=sha256:b05296e8bc88c92e2b21e0a9bae4740c1551ee613c1d93a51fd28a7a0b2b6fbb \
+    --hash=sha256:b0ec13f352ea5ae0fc91f98a48540512eed0767d0ec4f7f3cb92d92797983d18 \
+    --hash=sha256:b3df42f52502438ee973042cc551877d24619fa1cd38ef7b7e9ac74200daca8b \
+    --hash=sha256:b78008a69300d929ca2efeffec951b64a312e9a811e265ea4a907ab546d79fa6 \
+    --hash=sha256:b9026a21b6d41eb0e2e63f5bc1242c3fcc43ecb770963cda99a4307863dac12e \
+    --hash=sha256:bbe429fc50686bb2a2608a30843e36fbaa123462a5284f136c7d9e0145220bfd \
+    --hash=sha256:bfa1eb759e07d8b7aa7a310a2bc535e127ee70addf90dc8d4b946b593c3e51a8 \
+    --hash=sha256:c1e0ed5d84ffa2d677cc9582fc01e61dab2e7ef8b8996e055f0a76167b1b94df \
+    --hash=sha256:c4278d1873ce6e803e5d4f8d702fd3026bd67fca744aa98881324d1157ddf748 \
+    --hash=sha256:cac2b37ab21c2b36a10b685b1893ebd6b0f83ae26004838ac817680881576567 \
+    --hash=sha256:cbe6df25807227519debd1a57ab236f5f6bad441500e85b13903e51f93a43214 \
+    --hash=sha256:cd2c002f160502608dcc822ed2441a0f4509c52e86fcfd1a09e937278ed1ca14 \
+    --hash=sha256:e0137dd64a493ba6a4be37405cfd6febe650a98cc1e9dca8f6b8c63b1db11b41 \
+    --hash=sha256:eb45a34f23da4f8bc712b6376ca5396914b0b7c09adbb001dad964eb7f3132f8 \
+    --hash=sha256:ecb7572df5372abce8073df078207d9d1749f20b8b136089916a4a0868d56051 \
+    --hash=sha256:f12000a6accdd4bf0a3fa6eaa1b1c7a7bc80af0a2edf3f89d770d3dcce1d0e22 \
+    --hash=sha256:f7d69c1a7168ad0e9cb864e8663acb232986a0c9c9cb9801f56bf6214f53a54d \
+    --hash=sha256:f815fcc2b2a457977724bad97fb4854022980f51ce7b136925e336b530545ae1 \
+    --hash=sha256:fc39f5c27f962ec8660d8d20c24762431131b5d8c672b44b0a54cf2b5bcde9b9

tools/fwtools.py

@@ -1,3 +1,4 @@
+from zopfli.zlib import compress as zopfli_compress
 import zlib
 import logging
 import re
@@ -89,7 +90,7 @@ def parse_compressed_filename(filepath):
 # Returns compressed data
 def compress(uncompressed):
     logger.info(f'Compressing {len(uncompressed)} bytes')
-    compressed = zlib.compress(uncompressed, 9)
+    compressed = zopfli_compress(uncompressed)
     logger.info(f'Compressed to {len(compressed)} bytes')
     return compressed