Python: new pre-commit actions and pre-commit as GHA (microsoft#6376)

### Motivation and Context

<!-- Thank you for your contribution to the semantic-kernel repo!
Please help reviewers and future users, providing the following
information:
  1. Why is this change required?
  2. What problem does it solve?
  3. What scenario does it contribute to?
  4. If it fixes an open issue, please link to the issue here.
-->

- Added static code checking to pre-commit (check-ast and
nbqa-check-ast)
- Added Bandit security checking to pre-commit
- Added pre-commit step to python-lint workflow, if it works, can delete
seperate mypy, ruff and black tests

### Description

<!-- Describe your changes, the overall approach, the underlying design.
These notes will help understanding how your code works. Thanks! -->

### Contribution Checklist

<!-- Before submitting this PR, please make sure: -->

- [x] The code builds clean without any errors or warnings
- [x] The PR follows the [SK Contribution
Guidelines](https://github.com/microsoft/semantic-kernel/blob/main/CONTRIBUTING.md)
and the [pre-submission formatting
script](https://github.com/microsoft/semantic-kernel/blob/main/CONTRIBUTING.md#development-scripts)
raises no violations
- [x] All unit tests pass, and I have added new tests where possible
- [x] I didn't break anyone 😄

.github/workflows/python-lint.yml

@@ -7,67 +7,22 @@ on:
       - 'python/**'
 
 jobs:
-  ruff:
+  pre-commit:
     if: '!cancelled()'
     strategy:
       fail-fast: false
       matrix:
         python-version: ["3.10"]
     runs-on: ubuntu-latest
-    timeout-minutes: 5
+    continue-on-error: true
     steps:
-      - run: echo "/root/.local/bin" >> $GITHUB_PATH
       - uses: actions/checkout@v4
       - name: Install poetry
         run: pipx install poetry
       - uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python-version }}
           cache: "poetry"
-      - name: Install Semantic Kernel
-        run: cd python && poetry install --no-ansi
-      - name: Run ruff
-        run: cd python && poetry run ruff check .
-  black:
-    if: '!cancelled()'
-    strategy:
-      fail-fast: false
-      matrix:
-        python-version: ["3.10"]
-    runs-on: ubuntu-latest
-    timeout-minutes: 5
-    steps:
-      - run: echo "/root/.local/bin" >> $GITHUB_PATH
-      - uses: actions/checkout@v4
-      - name: Install poetry
-        run: pipx install poetry
-      - uses: actions/setup-python@v5
-        with:
-          python-version: ${{ matrix.python-version }}
-          cache: "poetry"
-      - name: Install Semantic Kernel
-        run: cd python && poetry install --no-ansi
-      - name: Run black
-        run: cd python && poetry run black --check .
-  mypy:
-    if: '!cancelled()'
-    strategy:
-      fail-fast: false
-      matrix:
-        python-version: ["3.10"]
-    runs-on: ubuntu-latest
-    timeout-minutes: 5
-    steps:
-      - run: echo "/root/.local/bin" >> $GITHUB_PATH
-      - uses: actions/checkout@v4
-      - name: Install poetry
-        run: pipx install poetry
-      - uses: actions/setup-python@v5
-        with:
-          python-version: ${{ matrix.python-version }}
-          cache: "poetry"
-      - name: Install Semantic Kernel
-        run: cd python && poetry install --no-ansi
-      - name: Run mypy
-        run: cd python && poetry run mypy -p semantic_kernel --config-file=mypy.ini
-
+      - name: Install dependencies
+        run: cd python && poetry install
+      - uses: pre-commit/[email protected]

.github/workflows/python-test-coverage.yml

@@ -10,7 +10,6 @@ jobs:
   python-tests-coverage:
     name: Create Test Coverage Messages
     runs-on: ${{ matrix.os }}
-    continue-on-error: true
     permissions:
       pull-requests: write
       contents: read
@@ -21,14 +20,17 @@ jobs:
         os: [ubuntu-latest]
     steps:
       - name: Wait for unit tests to succeed
+        continue-on-error: true
         uses: lewagon/[email protected]
         with:
           ref: ${{ github.event.pull_request.head.sha }}
           check-name: 'Python Unit Tests (${{ matrix.python-version}}, ${{ matrix.os }})'
           repo-token: ${{ secrets.GH_ACTIONS_PR_WRITE }}
           wait-interval: 10
+          allowed-conclusions: success
       - uses: actions/checkout@v4
       - name: Download coverage
+        continue-on-error: true
         uses: dawidd6/action-download-artifact@v3
         with:
           name: python-coverage-${{ matrix.os }}-${{ matrix.python-version }}.txt
@@ -37,6 +39,7 @@ jobs:
           search_artifacts: true
           if_no_artifact_found: warn
       - name: Download pytest
+        continue-on-error: true
         uses: dawidd6/action-download-artifact@v3
         with:
           name: pytest-${{ matrix.os }}-${{ matrix.python-version }}.xml
@@ -45,6 +48,7 @@ jobs:
           search_artifacts: true
           if_no_artifact_found: warn
       - name: Pytest coverage comment
+        continue-on-error: true
         id: coverageComment
         uses: MishaKav/pytest-coverage-comment@main
         with:

.pre-commit-config.yaml

@@ -7,23 +7,37 @@ repos:
       - id: sync_with_poetry
         args: [--config=.pre-commit-config.yaml, --db=python/.conf/packages_list.json, python/poetry.lock]
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.0.1
+    rev: v4.6.0
     hooks:
       - id: check-toml
         files: \.toml$
       - id: check-yaml
         files: \.yaml$
+      - id: check-json
+        files: \.json$
+        exclude: ^python\/\.vscode\/.*
       - id: end-of-file-fixer
         files: \.py$
       - id: mixed-line-ending
         files: \.py$
-  - repo: https://github.com/psf/black
-    rev: 24.4.2
+      - id: debug-statements
+        files: ^python\/semantic_kernel\/.*\.py$
+      - id: check-ast
+        name: Check Valid Python Samples
+        types: ["python"]
+  - repo: https://github.com/nbQA-dev/nbQA
+    rev: 1.8.5
     hooks:
-      - id: black
-        files: \.py$
+      - id: nbqa-check-ast
+        name: Check Valid Python Notebooks
+        types: ["jupyter"]
+  - repo: https://github.com/asottile/pyupgrade
+    rev: v3.15.2
+    hooks:
+    - id: pyupgrade
+      args: [--py310-plus]
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.4.4
+    rev: v0.4.5
     hooks:
       - id: ruff
         args: [ --fix, --exit-non-zero-on-fix ]
@@ -36,3 +50,9 @@ repos:
       language: system
       types: [python]
       pass_filenames: false
+  - repo: https://github.com/PyCQA/bandit
+    rev: 1.7.8
+    hooks:
+      - id: bandit
+        args: ["-c", "python/pyproject.toml"]
+        additional_dependencies: [ "bandit[toml]" ]

python/.vscode/extensions.json

@@ -2,8 +2,9 @@
   // See https://go.microsoft.com/fwlink/?LinkId=827846
   // for the documentation about the extensions.json format
   "recommendations": [
-    "littlefoxteam.vscode-python-test-adapter",
     "streetsidesoftware.code-spell-checker",
     "ms-python.python",
+    "charliermarsh.ruff",
+    "rodolphebarbanneau.python-docstring-highlighter"
   ]
 }

python/.vscode/settings.json

@@ -18,7 +18,6 @@
     ],
     "python.testing.unittestEnabled": false,
     "python.testing.pytestEnabled": true,
-    "pythonTestExplorer.testFramework": "pytest",
     "[python]": {
         "editor.codeActionsOnSave": {
             "source.organizeImports": "explicit",

python/.vscode/tasks.json

@@ -93,8 +93,7 @@
             "command": "poetry",
             "args": [
                 "install",
-                "--extras",
-                "all"
+                "--all-extras"
             ],
             "presentation": {
                 "reveal": "silent",