Use pip to avoid kaniko error

.envrc

@@ -1,5 +1,3 @@
-use flake
 dotenv
-
-# https://python-poetry.org/docs/basic-usage/#activating-the-virtual-environment
-source $(poetry env info --path)/bin/activate
+use flake
+layout python

.github/workflows/ci.yaml

@@ -17,32 +17,29 @@ jobs:
         with:
           fetch-depth: 2
 
-      # https://github.com/actions/setup-python/blob/main/docs/advanced-usage.md#caching-packages
-      - name: Install poetry
-        run: pipx install poetry
       - name: Setup Python
         uses: actions/setup-python@v5
         with:
-          python-version: "3.11"
-          cache: poetry
+          cache: pip
+          python-version-file: 'pyproject.toml'
 
       - name: Install dependencies
-        run: poetry install
+        run: pip install -e ".[dev,test]"
 
       # https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-python#using-ruff-to-lint-code
       - name: Ruff
-        run: poetry run ruff check --output-format=github .
+        run: ruff check --output-format=github .
         continue-on-error: true
 
       - name: Mypy
-        run: poetry run mypy .
+        run: mypy .
 
       # https://github.com/codecov/example-python
       - name: Pytest
-        run: poetry run pytest --cov=src --cov-report=xml
+        run: pytest --cov=src --cov-report=xml
 
       - name: Compile Vertex Pipeline
-        run: poetry run python -m src.ainu_lm_pipeline.compile
+        run: python -m src.ainu_lm_pipeline.compile
 
       - name: Codecov
         uses: codecov/codecov-action@v3

flake.nix

@@ -10,9 +10,8 @@
   in
   {
     devShell.aarch64-darwin = pkgs.mkShell {
-      buildInputs = with pkgs; [
+      packages = with pkgs; [
         python311
-        poetry
         google-cloud-sdk
       ];
     };

pyproject.toml

@@ -1,44 +1,52 @@
-[tool.poetry]
+[build-system]
+requires = ["setuptools"]
+build-backend = "setuptools.build_meta"
+
+[project]
 name = "ainu-lm"
 version = "0.1.0"
 description = ""
-authors = ["Ryo Igarashi <[email protected]>"]
-license = "MIT"
+authors = [
+	{ name = "Ryo Igarashi <[email protected]>", email = "[email protected]" },
+]
 readme = "README.md"
-package-mode = false
+requires-python = ">=3.10"
 
-[tool.poetry.dependencies]
-python = ">=3.10,<3.12.0"
-datasets = "^2.18.0"
-transformers = "^4.38.2"
-sentencepiece = "^0.2.0"
-tokenizers = "^0.15.2"
-google-cloud-storage = "^2.16.0"
-cloudml-hypertune = "^0.1.0.dev6"
-accelerate = "^0.28.0"
-google-cloud-aiplatform = "^1.44.0"
-google-cloud-pipeline-components = "^2.11.0"
-tensorboard = "^2.16.2"
+dependencies = [
+	"datasets==2.18.0",
+	"transformers==4.38.2",
+	"sentencepiece==0.2.0",
+	"tokenizers==0.15.2",
+	"google-cloud-storage==2.16.0",
+	"cloudml-hypertune==0.1.0.dev6",
+	"accelerate==0.28.0",
+	"google-cloud-aiplatform==1.44.0",
+	"google-cloud-pipeline-components==2.11.0",
+	"tensorboard==2.16.2"
+]
 
+[project.optional-dependencies]
+test = [
+	"pytest==8.1.1",
+	"pytest-cov==4.1.0"
+]
+dev = [
+	"ruff==0.3.3",
+	"mypy==1.9.0",
+	"torch==2.2.1"
+]
 
-[tool.poetry.group.dev.dependencies]
-pytest = "^8.1.1"
-pytest-cov = "^4.1.0"
-ruff = "^0.3.3"
-mypy = "^1.9.0"
-torch = "^2.2.1"
-
-[build-system]
-requires = ["poetry-core"]
-build-backend = "poetry.core.masonry.api"
+[tool.setuptools]
+packages = ["src"]
 
 [tool.mypy]
 ignore_missing_imports = true
 exclude = ["ainu_lm_pipeline"]
 
 [tool.ruff.lint]
 select = ["E4", "E7", "E9", "F", "I", "ANN"]
+# selfに型つけろって言ってくるやつ
 ignore = ["ANN101", "ANN102"]
 
-[tool.ruff.per-file-ignores]
-"__init__.py" = ["F401"]
+[tool.ruff.lint.per-file-ignores]
+"__init__.py" = ["F401"]

src/ainu_lm_pipeline/submit.py

@@ -1,11 +1,17 @@
 import argparse
 import os
+from datetime import datetime
 
 from google.cloud import aiplatform
 from google.cloud.aiplatform.pipeline_jobs import PipelineJob
 
 from . import config as cfg
 
+
+def get_timestamp() -> str:
+    return datetime.now().strftime("%Y%m%d%H%M%S")
+
+
 parser = argparse.ArgumentParser()
 parser.add_argument("--commit-sha", type=str, required=True)
 
@@ -14,7 +20,7 @@
     aiplatform.init(project=cfg.PROJECT_ID, location=cfg.REGION)
 
     args = parser.parse_args()
-    job_id = f"pipeline-ainu-lm-{args.commit_sha}"
+    job_id = f"pipeline-ainu-lm-{get_timestamp()}"
 
     pipeline_params = {
         "pipeline_job_id": job_id,
@@ -32,7 +38,7 @@
         job_id=job_id,
         pipeline_root=cfg.PIPELINE_ROOT,
         parameter_values=pipeline_params,
-        enable_caching=True,
+        enable_caching=False,
     )
 
     pipeline_job.run(sync=True)

src/ainu_lm_trainer/Dockerfile

@@ -6,14 +6,8 @@ ENV TRANSFORMERS_NO_ADVISORY_WARNINGS=1
 
 RUN apt update
 
-COPY pyproject.toml poetry.lock /workspace/
-
-RUN pip install poetry \
- && poetry config virtualenvs.create false \
- && poetry export -f requirements.txt --output requirements.txt --without-hashes \
- && pip uninstall -y poetry
-
-RUN pip install -r requirements.txt
+COPY pyproject.toml /workspace/
+RUN pip install -e .
 
 COPY . /workspace