From d6015b13c4b9cff60c796c15374767f998db7842 Mon Sep 17 00:00:00 2001 From: Douglas Thor Date: Sat, 13 Jan 2024 14:51:53 -0800 Subject: [PATCH 01/13] Update .gitignore --- .gitignore | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/.gitignore b/.gitignore index 441b083..9f16ff8 100644 --- a/.gitignore +++ b/.gitignore @@ -140,3 +140,25 @@ tags # Keep it secret. Keep it safe. secrets.sh + +### End of Custom Things ### + +# Created by https://www.toptal.com/developers/gitignore/api/bazel +# Edit at https://www.toptal.com/developers/gitignore?templates=bazel + +### Bazel ### +# gitignore template for Bazel build system +# website: https://bazel.build/ + +# Ignore all bazel-* symlinks. There is no full list since this can change +# based on the name of the directory bazel is cloned into. +/bazel-* + +# Directories for the Bazel IntelliJ plugin containing the generated +# IntelliJ project files and plugin configuration. Seperate directories are +# for the IntelliJ, Android Studio and CLion versions of the plugin. +/.ijwb/ +/.aswb/ +/.clwb/ + +# End of https://www.toptal.com/developers/gitignore/api/bazel From 989d77edd4b43a2ccf9f4065ec3354113277ba7e Mon Sep 17 00:00:00 2001 From: Douglas Thor Date: Sat, 13 Jan 2024 13:50:30 -0800 Subject: [PATCH 02/13] Add shfmt to pre-commit --- .pre-commit-config.yaml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 5a5dba8..8978b9b 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -72,3 +72,8 @@ repos: rev: v0.971 hooks: - id: mypy + + - repo: https://github.com/jumanjihouse/pre-commit-hooks + rev: 2.1.6 + hooks: + - id: shfmt From f63b4220343e4564b430fd14c55a8f693d6e51be Mon Sep 17 00:00:00 2001 From: Douglas Thor Date: Sun, 14 Jan 2024 12:52:20 -0800 Subject: [PATCH 03/13] Add requirements_lock.txt for bazel --- requirements_lock.txt | 38 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) create mode 100644 requirements_lock.txt diff --git a/requirements_lock.txt b/requirements_lock.txt new file mode 100644 index 0000000..c47c176 --- /dev/null +++ b/requirements_lock.txt @@ -0,0 +1,38 @@ +appdirs==1.4.3 +argon2-cffi==23.1.0 +argon2-cffi-bindings==21.2.0 +attrs==19.3.0 +cffi==1.16.0 +cfgv==3.1.0 +click==8.1.7 +construct==2.10.68 +coverage==7.3.2 +distlib==0.3.6 +exceptiongroup==1.1.3 +filelock==3.8.0 +future==0.18.3 +identify==2.5.6 +importlib-metadata==1.5.0 +importlib-resources==1.0.2 +iniconfig==2.0.0 +lxml==4.9.3 +more-itertools==8.2.0 +nodeenv==1.3.5 +packaging==20.1 +platformdirs==2.5.2 +pluggy==0.13.1 +pre-commit==3.4.0 +pycparser==2.21 +pycryptodomex==3.19.0 +pykeepass==4.0.3 +pyparsing==2.4.6 +pytest==7.4.2 +pytest-cov==4.1.0 +python-dateutil==2.8.2 +PyYAML==6.0.1 +six==1.14.0 +toml==0.10.0 +tomli==2.0.1 +virtualenv==20.16.5 +wcwidth==0.1.8 +zipp==3.0.0 From 95ee4065e797e303e5c93fe145976e3d683826c9 Mon Sep 17 00:00:00 2001 From: Douglas Thor Date: Sat, 13 Jan 2024 13:47:30 -0800 Subject: [PATCH 04/13] Add a script to setup bazel --- setup_bazel.sh | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 setup_bazel.sh diff --git a/setup_bazel.sh b/setup_bazel.sh new file mode 100644 index 0000000..a795452 --- /dev/null +++ b/setup_bazel.sh @@ -0,0 +1,18 @@ +#!/bin/bash + +# Install bazel (https://bazel.build) via bazelisk (https://github.com/bazelbuild/bazelisk). +# +# Bazelisk is a wrapper around bazel and can be invoked the same way as the OG `bazel`. +# +# Puts the bazelisk binary at /usr/local/bin/bazelisk and .../bazel + +BAZEL_VERSION="v1.19.0" +URL="https://github.com/bazelbuild/bazelisk/releases/download/$BAZEL_VERSION/bazelisk-linux-amd64" + +TMP_PATH="/tmp/bazelisk" +BINARY_PATH="/usr/local/bin/bazelisk" + +wget $URL -O $TMP_PATH +sudo cp $TMP_PATH $BINARY_PATH +sudo chmod a+x $BINARY_PATH +sudo ln $BINARY_PATH /usr/local/bin/bazel From a46f6ecf4c0ad3b829dcf90bd0bc72b75f2a82b2 Mon Sep 17 00:00:00 2001 From: Douglas Thor Date: Sat, 13 Jan 2024 14:13:24 -0800 Subject: [PATCH 05/13] Add WORKSPACE --- WORKSPACE | 60 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 60 insertions(+) create mode 100644 WORKSPACE diff --git a/WORKSPACE b/WORKSPACE new file mode 100644 index 0000000..90667d4 --- /dev/null +++ b/WORKSPACE @@ -0,0 +1,60 @@ +# Name the workspace +workspace(name = "bitwarden-to-keepass") + +# Install rules_python, which allows us to define how bazel should work with python files. +# See https://rules-python.readthedocs.io/en/latest/getting-started.html#using-a-workspace-file +# Note: These "##### START ... #####" comments are just my own - they do not have +# any meaning in bazel. +##### START install rules_python snippet (https://github.com/bazelbuild/rules_python/releases/tag/0.28.0) ##### +load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") + +http_archive( + name = "rules_python", + sha256 = "d70cd72a7a4880f0000a6346253414825c19cdd40a28289bdf67b8e6480edff8", + strip_prefix = "rules_python-0.28.0", + url = "https://github.com/bazelbuild/rules_python/releases/download/0.28.0/rules_python-0.28.0.tar.gz", +) + +load("@rules_python//python:repositories.bzl", "py_repositories") + +py_repositories() +##### END install rules_python snippet ##### + + +# Use a hermetic python rather than relying on a system-installed interpreter: +# See https://rules-python.readthedocs.io/en/stable/getting-started.html#toolchain-registration +##### START hermetic python snippet ##### +load("@rules_python//python:repositories.bzl", "py_repositories", "python_register_toolchains") + +py_repositories() + +python_register_toolchains( + name = "python3_8", + # Available versions are listed in @rules_python//python:versions.bzl. + # We recommend using the same version your team is already standardized on. + python_version = "3.8.18", + # Support coverage. See https://rules-python.readthedocs.io/en/stable/coverage.html + register_coverage_tool = True, +) + +load("@python3_8//:defs.bzl", interpreter = "interpreter") +##### END hermetic python snippet ##### + + +# Install dependencies from PyPI. +# See https://rules-python.readthedocs.io/en/stable/pypi-dependencies.html#using-a-workspace-file +# and https://rules-python.readthedocs.io/en/stable/pip.html +##### START install python dependencies snippet ##### +load("@rules_python//python:pip.bzl", "pip_parse") + +pip_parse( + name = "pypi", + python_interpreter_target = interpreter, # From hermetic python snippet + # TODO: Can requirements come from pyproject.toml? + requirements_lock = "//:requirements_lock.txt", +) + +load("@pypi//:requirements.bzl", "install_deps") + +install_deps() +##### END install python dependencies snippet ##### From 1d032476e365c226d04e660aab77944b78a6aa36 Mon Sep 17 00:00:00 2001 From: Douglas Thor Date: Sun, 14 Jan 2024 12:48:35 -0800 Subject: [PATCH 06/13] Add stuff that helps bazel work with pytest --- tools/bazel/BUILD | 4 ++++ tools/bazel/README.md | 26 +++++++++++++++++++++ tools/bazel/defs.bzl | 47 ++++++++++++++++++++++++++++++++++++++ tools/bazel/pytest_shim.py | 28 +++++++++++++++++++++++ 4 files changed, 105 insertions(+) create mode 100644 tools/bazel/BUILD create mode 100644 tools/bazel/README.md create mode 100644 tools/bazel/defs.bzl create mode 100644 tools/bazel/pytest_shim.py diff --git a/tools/bazel/BUILD b/tools/bazel/BUILD new file mode 100644 index 0000000..e2adf59 --- /dev/null +++ b/tools/bazel/BUILD @@ -0,0 +1,4 @@ +# We have to tell bazel to make our pytest shim available to everyone. +exports_files([ + "pytest_shim.py", +]) diff --git a/tools/bazel/README.md b/tools/bazel/README.md new file mode 100644 index 0000000..db883cb --- /dev/null +++ b/tools/bazel/README.md @@ -0,0 +1,26 @@ +# Bazel Tools + +This dir contains helpers and shims used by bazel. + +It's a central location for storing things that can be used by all bazel BUILD +files. + +For this project, we use it to define the pytest shim that executes `pytest` +on the project and the `pytest_test` bazel rule. + +Other uses may include: + ++ A shim for setting environment variables ++ Special build rules ++ Uhh... Other things + + +## What's In Here? + ++ `BUILD`: The bazel build file that exports `pytest_shim.py` so that every + other BUILD file can use it. ++ `defs.bzl`: Kinda like a bazel config file. Defines the `pytest_test` rule + as a wrapper around the `py_test` rule. ++ `pytest_shim.py`: A python module that gets set as the main entry point + when running `pytest_test` rules. ++ `README.md`: This file. diff --git a/tools/bazel/defs.bzl b/tools/bazel/defs.bzl new file mode 100644 index 0000000..6107ab8 --- /dev/null +++ b/tools/bazel/defs.bzl @@ -0,0 +1,47 @@ +load("@rules_python//python:defs.bzl", _py_test = "py_test") +load("@pypi//:requirements.bzl", "requirement") + +def pytest_test(name, srcs, deps, **kwargs): + """ + A bazel rule for running python tests via pytest. + + Usage: + ```bazel + pytest_test( + name = "test_main", + srcs = ["test_main.py"], + deps = [ + # The module to test + "//src/my_package:main", + # Still need to include pytest. See TODO below. + requirement("pytest"), + ], + ) + ``` + """ + if "main" in kwargs: + fail("Can't set 'main' - we set it in pytest_test rule definition.") + + shim = "//tools/bazel:pytest_shim.py" + + # TODO: We can automagically update the deps if we want. For now, to make + # sure I understand bazel, I won't be doing that. + # deps = + + # Include the shim as a source. + all_srcs = [shim] + srcs + + _py_test( + name = name, + srcs = all_srcs, + main = shim, + # TODO: What's this 'location'? + args = ["$(location {})".format(src) for src in srcs], + # Within bazel, `$HOME` is typically set to $TEST_TMPDIR`, but it looks + # like there's a bug? https://github.com/bazelbuild/bazel/issues/10652 + # So we inject it manually. + # TODO: Don't point to root, point to $TEST_TMPDIR instead. + env = {"HOME": "/"}, + deps = deps, + **kwargs, + ) diff --git a/tools/bazel/pytest_shim.py b/tools/bazel/pytest_shim.py new file mode 100644 index 0000000..85d217e --- /dev/null +++ b/tools/bazel/pytest_shim.py @@ -0,0 +1,28 @@ +""" +A Bazel shim for executing pytest. + +By default, the `py_test` bazel rule (https://bazel.build/reference/be/python#py_test) +will simply run the test module. In the `unittest` world, typically every module +will have: + +```python +if __name__ == "__main__": + unittest.main() +``` + +so tests actually get run during `bazel test`. + +However, this project uses `pytest` as a test runner and test modules don't +have `if __name__ == "__main__":` code in them, so when `bazel test` runs, +nothing actually happens. + +This shim gets added to all `pytest_test` rules and set to bazel's `main`, so +it's what gets executed when a pytest file is "run" with `bazel test`. +""" +import sys + +import pytest + +if __name__ == "__main__": + exit_code = pytest.main(sys.argv[1:]) + sys.exit(exit_code) From 835b7a33b80495c12e4094adf3190e2ae4ff665f Mon Sep 17 00:00:00 2001 From: Douglas Thor Date: Sat, 13 Jan 2024 14:32:30 -0800 Subject: [PATCH 07/13] Add Bazel BUILD files. --- BUILD | 7 ++++++ src/bitwarden_to_keepass/BUILD | 27 ++++++++++++++++++++++ tests/BUILD | 41 ++++++++++++++++++++++++++++++++++ 3 files changed, 75 insertions(+) create mode 100644 BUILD create mode 100644 src/bitwarden_to_keepass/BUILD create mode 100644 tests/BUILD diff --git a/BUILD b/BUILD new file mode 100644 index 0000000..a359fc6 --- /dev/null +++ b/BUILD @@ -0,0 +1,7 @@ +# Tell bazel to include some files +filegroup( + name = "mygroup", + srcs = [ + "requirements_lock.txt", + ], +) diff --git a/src/bitwarden_to_keepass/BUILD b/src/bitwarden_to_keepass/BUILD new file mode 100644 index 0000000..56f0f37 --- /dev/null +++ b/src/bitwarden_to_keepass/BUILD @@ -0,0 +1,27 @@ +load("@rules_python//python:defs.bzl", "py_binary") +load("@pypi//:requirements.bzl", "requirement") + +py_library( + name = "__init__", + srcs = ["__init__.py"], + deps = [], +) + +py_library( + name = "main", + srcs = ["main.py"], + deps = [ + ":__init__", + requirement("pykeepass"), + ], + visibility = ["//visibility:public"], +) + +py_binary( + name = "cli", + srcs = ["cli.py"], + deps = [ + ":main", + requirement("click"), + ], +) diff --git a/tests/BUILD b/tests/BUILD new file mode 100644 index 0000000..a193995 --- /dev/null +++ b/tests/BUILD @@ -0,0 +1,41 @@ +load("//tools/bazel:defs.bzl", "pytest_test") +load("@rules_python//python:defs.bzl", "py_binary", "py_test") +load("@pypi//:requirements.bzl", "requirement") + +filegroup( + name = "test_data", + srcs = glob([ + "data/*", + ]), +) + +py_library( + name = "__init__", + srcs = ["__init__.py"], + deps = [], +) + +py_library( + name = "conftest", + srcs = ["conftest.py"], + deps = [ + requirement("pytest"), + ], +) + +pytest_test( + name = "test_main", + srcs = ["test_main.py"], + # imports = ["src/bitwarden_to_keepass"], + deps = [ + # To reference something defined in another BUILD file, prefex with + # "//" + "//src/bitwarden_to_keepass:main", + # Reference to current BUILD file + ":__init__", + ":conftest", + requirement("pykeepass"), + requirement("pytest"), + ], + data = [":test_data"], +) From 16e0161e9b8c5969334d0b333318f5bcde7b5749 Mon Sep 17 00:00:00 2001 From: Douglas Thor Date: Sun, 14 Jan 2024 13:17:27 -0800 Subject: [PATCH 08/13] Update README --- README.md | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/README.md b/README.md index 6bbfb26..7d98da5 100644 --- a/README.md +++ b/README.md @@ -147,9 +147,32 @@ Then push tags to github. CI will build the source distribution and wheel and upload them to PyPI. +### Bazel + +I'm experimenting with [`bazel`][bazel] for running tests (and perhaps also compiling +a binary in the future). + +To run tests: + +```console +$ bazel test //tests:test_main +``` + +There are a couple other CLI args that might be useful: + ++ `--test_output=streamed`: Run tests serially and show the output of pytest. + +To build (though note that this doesn't fully work yet): + +```console +$ bazel build //src/bitwarden_to_keepass:cli +``` + + ## Changelog See [CHANGELOG.md](./CHANGELOG.md). [bw-cli]: https://bitwarden.com/help/cli/ +[bazel]: https://bazel.build/ From 86c07a0b47f5a9f863df9752c238118dbed9bd12 Mon Sep 17 00:00:00 2001 From: Douglas Thor Date: Sun, 14 Jan 2024 13:26:26 -0800 Subject: [PATCH 09/13] Update readme --- README.md | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/README.md b/README.md index 7d98da5..e425b60 100644 --- a/README.md +++ b/README.md @@ -158,6 +158,22 @@ To run tests: $ bazel test //tests:test_main ``` +If a test fails, you'll see something like: + +``` +INFO: Build completed, 1 test FAILED, 2 total actions +//tests:test_main FAILED in 1.7s + /home/dthor/.cache/bazel/_bazel_dthor/7076d176777da645a0c7cf0359126a31/execroot/_main/bazel-out/k8-fastbuild/testlogs/tests/test_main/test.log + + Executed 1 out of 1 test: 1 fails locally. +``` + +To see the logs of that test, open that file in `less` or whatever you prefer: + +```console +$ less bazel-out/k8-fastbuild/testlogs/tests/test_main/test.log +``` + There are a couple other CLI args that might be useful: + `--test_output=streamed`: Run tests serially and show the output of pytest. From 052a64ec8e4e49f8decca76386ed9d2812855cb8 Mon Sep 17 00:00:00 2001 From: Douglas Thor Date: Sun, 14 Jan 2024 13:29:33 -0800 Subject: [PATCH 10/13] Color pytest output --- tools/bazel/defs.bzl | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/tools/bazel/defs.bzl b/tools/bazel/defs.bzl index 6107ab8..029c8fe 100644 --- a/tools/bazel/defs.bzl +++ b/tools/bazel/defs.bzl @@ -37,11 +37,15 @@ def pytest_test(name, srcs, deps, **kwargs): main = shim, # TODO: What's this 'location'? args = ["$(location {})".format(src) for src in srcs], - # Within bazel, `$HOME` is typically set to $TEST_TMPDIR`, but it looks - # like there's a bug? https://github.com/bazelbuild/bazel/issues/10652 - # So we inject it manually. - # TODO: Don't point to root, point to $TEST_TMPDIR instead. - env = {"HOME": "/"}, + env = { + # Within bazel, `$HOME` is typically set to $TEST_TMPDIR`, but it looks + # like there's a bug? https://github.com/bazelbuild/bazel/issues/10652 + # So we inject it manually. + # TODO: Don't point to root, point to $TEST_TMPDIR instead. + "HOME": "/", + # Force pytest to always color output. + "PYTEST_ADDOPTS": "--color=yes", + }, deps = deps, **kwargs, ) From 18ae61d23d8ec26ef53135879bb907ea7b8908be Mon Sep 17 00:00:00 2001 From: Douglas Thor Date: Sun, 14 Jan 2024 13:33:06 -0800 Subject: [PATCH 11/13] Add missing 'how to install bazel' to README --- README.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/README.md b/README.md index e425b60..b825073 100644 --- a/README.md +++ b/README.md @@ -152,6 +152,12 @@ upload them to PyPI. I'm experimenting with [`bazel`][bazel] for running tests (and perhaps also compiling a binary in the future). +First, install `bazel`: + +```console +$ source setup_bazel.sh +``` + To run tests: ```console From 012e8c5e05df8af985d94174bd546bdccfbc835d Mon Sep 17 00:00:00 2001 From: Douglas Thor Date: Sun, 14 Jan 2024 13:35:08 -0800 Subject: [PATCH 12/13] Remove shfmt from pre-commit --- .pre-commit-config.yaml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 8978b9b..5a5dba8 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -72,8 +72,3 @@ repos: rev: v0.971 hooks: - id: mypy - - - repo: https://github.com/jumanjihouse/pre-commit-hooks - rev: 2.1.6 - hooks: - - id: shfmt From 9277059ea97871ba45b94ce81c6e863f0d8739e0 Mon Sep 17 00:00:00 2001 From: Douglas Thor Date: Sun, 14 Jan 2024 13:38:57 -0800 Subject: [PATCH 13/13] Update readme --- README.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/README.md b/README.md index b825073..aec6d95 100644 --- a/README.md +++ b/README.md @@ -158,6 +158,12 @@ First, install `bazel`: $ source setup_bazel.sh ``` +Then activate your venv: + +```console +$ . .venv/bin/activate +``` + To run tests: ```console