10083 bitbucket collector (#10522)

Add support for measuring inactive branches using Bitbucket as source.
---------

Co-authored-by: tguler <[email protected]>

components/collector/.vulture_ignore_list.py

@@ -21,6 +21,8 @@
 AzureDevopsUserStoryPoints  # unused class (src/source_collectors/azure_devops/user_story_points.py:12)
 BanditSecurityWarnings  # unused class (src/source_collectors/bandit/security_warnings.py:10)
 BanditSourceUpToDateness  # unused class (src/source_collectors/bandit/source_up_to_dateness.py:10)
+last_commit  # unused variable (src/source_collectors/bitbucket/inactive_branches.py:32)
+BitbucketInactiveBranches  # unused class (src/source_collectors/bitbucket/inactive_branches.py:35)
 CalendarSourceUpToDateness  # unused class (src/source_collectors/calendar/source_up_to_dateness.py:13)
 CalendarTimeRemaining  # unused class (src/source_collectors/calendar/time_remaining.py:13)
 kind  # unused variable (src/source_collectors/cargo_audit/security_warnings.py:42)

components/collector/src/source_collectors/__init__.py

@@ -22,6 +22,7 @@
 from .azure_devops.user_story_points import AzureDevopsUserStoryPoints
 from .bandit.security_warnings import BanditSecurityWarnings
 from .bandit.source_up_to_dateness import BanditSourceUpToDateness
+from .bitbucket.inactive_branches import BitbucketInactiveBranches
 from .calendar.source_up_to_dateness import CalendarSourceUpToDateness
 from .calendar.time_remaining import CalendarTimeRemaining
 from .cargo_audit.security_warnings import CargoAuditSecurityWarnings

components/collector/src/source_collectors/bitbucket/base.py

@@ -0,0 +1,33 @@
+"""Bitbucket collector base classes."""
+
+from abc import ABC
+
+from base_collectors import SourceCollector
+from collector_utilities.functions import add_query
+from collector_utilities.type import URL
+
+
+class BitbucketBase(SourceCollector, ABC):
+    """Base class for Bitbucket collectors."""
+
+    def _basic_auth_credentials(self) -> tuple[str, str] | None:
+        """Override to return None, as the private token is passed as header."""
+        return None
+
+    def _headers(self) -> dict[str, str]:
+        """Extend to add the private token, if any, to the headers."""
+        headers = super()._headers()
+        if private_token := self._parameter("private_token"):
+            headers["Authorization"] = "Bearer " + str(private_token)
+        return headers
+
+
+class BitbucketProjectBase(BitbucketBase, ABC):
+    """Base class for Bitbucket collectors for a specific project."""
+
+    async def _bitbucket_api_url(self, api: str) -> URL:
+        """Return a Bitbucket API url for a project, if present in the parameters."""
+        url = await super()._api_url()
+        project = f"{self._parameter('owner')}/repos/{self._parameter('repository')}"
+        api_url = URL(f"{url}/rest/api/1.0/projects/{project}" + (f"/{api}" if api else ""))
+        return add_query(api_url, "limit=100&details=true")

components/collector/src/source_collectors/bitbucket/inactive_branches.py

@@ -0,0 +1,85 @@
+"""Bitbucket inactive branches collector."""
+
+from datetime import datetime
+from typing import cast
+
+from base_collectors import BranchType, InactiveBranchesSourceCollector
+from collector_utilities.date_time import datetime_from_timestamp
+from collector_utilities.exceptions import NotFoundError
+from collector_utilities.type import URL
+from model import SourceResponses
+
+from .base import BitbucketProjectBase
+
+
+class BitbucketBranchInfoError(NotFoundError):
+    """Bitbucket branch info is missing."""
+
+    def __init__(self, project: str) -> None:
+        tip = (
+            "Please check if the repository (name with owner) and access token (with repo scope) are "
+            "configured correctly."
+        )
+        super().__init__("Branch info for repository", project, extra=tip)
+
+
+class BitbucketBranchType(BranchType):
+    """Bitbucket branch information as returned by the API."""
+
+    id: str
+    default: bool
+    last_commit: datetime
+
+
+class BitbucketInactiveBranches[Branch: BitbucketBranchType](BitbucketProjectBase, InactiveBranchesSourceCollector):
+    """Collector for inactive branches."""
+
+    async def _api_url(self) -> URL:
+        """Override to return the branches API."""
+        return await self._bitbucket_api_url("branches")
+
+    async def _landing_url(self, responses: SourceResponses) -> URL:
+        """Extend to add the project branches."""
+        return URL(f"{await super()._landing_url(responses)}/{self._parameter('project')}/browse")
+
+    async def _branches(self, responses: SourceResponses) -> list[BitbucketBranchType]:
+        """Return a list of branches from the responses."""
+        branches = []
+        metadata = "com.atlassian.bitbucket.server.bitbucket-branch:latest-commit-metadata"
+        for response in responses:
+            json = await response.json()
+            branches.extend(
+                [
+                    BitbucketBranchType(
+                        name=branch["displayId"],
+                        default=branch["isDefault"],
+                        last_commit=datetime_from_timestamp(branch["metadata"][metadata]["committerTimestamp"]),
+                        id=branch["id"],
+                    )
+                    for branch in json["values"]
+                ]
+            )
+        if len(branches) == 0:
+            project = f"projects/{self._parameter('owner')}/repos/{self._parameter('repository')}"
+            raise BitbucketBranchInfoError(project)
+        return branches
+
+    def _is_default_branch(self, branch: Branch) -> bool:
+        """Return whether the branch is the default branch."""
+        return branch["default"]
+
+    def _is_branch_merged(self, branch: Branch) -> bool:
+        """Return whether the branch has been merged with the default branch."""
+        """The merged value is always set to false because the Bitbucket API does not include a merged field."""
+        return False
+
+    def _commit_datetime(self, branch: Branch) -> datetime:
+        """Override to parse the commit date from the branch."""
+        return branch["last_commit"]
+
+    def _branch_landing_url(self, branch: Branch) -> URL:
+        """Override to get the landing URL from the branch."""
+        instance_url = super()._parameter("url")
+        project = f"projects/{self._parameter('owner')}/repos/{self._parameter('repository')}/browse?at="
+        branch_id = str(branch.get("id")).lstrip("/")
+        return cast(URL, f"{instance_url}/{project}{branch_id or ''}")

components/collector/tests/source_collectors/bitbucket/base.py

@@ -0,0 +1,16 @@
+"""Bitbucket unit test base classes."""
+
+from tests.source_collectors.source_collector_test_case import SourceCollectorTestCase
+
+
+class BitbucketTestCase(SourceCollectorTestCase):
+    """Base class for testing Bitbucket collectors."""
+
+    SOURCE_TYPE = "bitbucket"
+
+    def setUp(self):
+        """Extend to add generic test fixtures."""
+        super().setUp()
+        self.set_source_parameter("branch", "branch")
+        self.set_source_parameter("owner", "owner")
+        self.set_source_parameter("repository", "repository")

components/collector/tests/source_collectors/bitbucket/test_inactive_branches.py

@@ -0,0 +1,90 @@
+"""Unit tests for the Bitbucket inactive branches collector."""
+
+from datetime import datetime
+from unittest.mock import AsyncMock
+
+from dateutil.tz import tzutc
+
+from .base import BitbucketTestCase
+
+
+class BitbucketInactiveBranchesTest(BitbucketTestCase):
+    """Unit tests for the inactive branches metric."""
+
+    METRIC_TYPE = "inactive_branches"
+    WEB_URL = "https://bitbucket/projects/owner/repos/repository/browse?at="
+
+    def setUp(self):
+        """Extend to setup fixtures."""
+        super().setUp()
+        self.set_source_parameter("branches_to_ignore", ["ignored_.*"])
+        main = self.create_branch("main", default=True)
+        unmerged = self.create_branch("unmerged_branch")
+        ignored = self.create_branch("ignored_branch")
+        active_unmerged = self.create_branch("active_unmerged_branch", active=True)
+        self.branches = self.create_branches_json([main, unmerged, ignored, active_unmerged])
+        self.unmerged_branch_entity = self.create_entity("unmerged_branch")
+        self.entities = [self.unmerged_branch_entity]
+        self.landing_url = (
+            "https://bitbucket/rest/api/1.0/projects/owner/repos/repository/branches?limit=100&details=true"
+        )
+
+    def create_branch(
+        self, name: str, *, default: bool = False, active: bool = False
+    ) -> dict[str, str | bool | dict[str, dict[str, float | int]]]:
+        """Create a Bitbucket branch."""
+        commit_date = (datetime.now(tz=tzutc()).timestamp() if active else 1554197584) * 1000
+        return {
+            "id": "refs/heads/" + name,
+            "displayId": name,
+            "type": "BRANCH",
+            "latestCommit": "ef6a9d214d509461f62f5f79b6444db55aaecc78",
+            "latestChangeset": "ef6a9d214d509461f62f5f79b6444db55aaecc78",
+            "isDefault": default,
+            "metadata": {
+                "com.atlassian.bitbucket.server.bitbucket-branch:latest-commit-metadata": {
+                    "committerTimestamp": commit_date
+                }
+            },
+        }
+
+    def create_branches_json(self, branches):
+        """Create an entity."""
+        return {"size": len(branches), "limit": 25, "isLastPage": True, "start": 0, "values": branches}
+
+    def create_entity(self, name: str) -> dict[str, str]:
+        """Create an entity."""
+        return {
+            "key": name,
+            "name": name,
+            "commit_date": "2019-04-02",
+            "merge_status": "unmerged",
+            "url": self.WEB_URL + "refs/heads/" + name,
+        }
+
+    async def test_inactive_branches(self):
+        """Test that the number of inactive branches can be measured."""
+        response = await self.collect(get_request_json_return_value=self.branches)
+        self.assert_measurement(response, value="1", entities=self.entities, landing_url=self.landing_url)
+
+    async def test_unmerged_inactive_branches(self):
+        """Test that the number of unmerged inactive branches can be measured."""
+        self.set_source_parameter("branch_merge_status", ["unmerged"])
+        response = await self.collect(get_request_json_return_value=self.branches)
+        self.assert_measurement(
+            response, value="1", entities=[self.unmerged_branch_entity], landing_url=self.landing_url
+        )
+
+    async def test_private_token(self):
+        """Test that the private token is used."""
+        self.set_source_parameter("private_token", "token")
+        inactive_branches_json = {"values": None}
+        inactive_branches_response = AsyncMock()
+        execute = AsyncMock(side_effect=[inactive_branches_response])
+        inactive_branches_response.json = AsyncMock(return_value=inactive_branches_json)
+        response = await self.collect(get_request_json_return_value=execute)
+        self.assert_measurement(
+            response,
+            landing_url=self.landing_url,
+            parse_error="Branch info for repository",
+        )

components/shared_code/src/shared_data_model/metrics.py

@@ -135,7 +135,7 @@
     change-your-default-branch).""",
         unit=Unit.BRANCHES,
         near_target="5",
-        sources=["azure_devops", "gitlab", "manual_number"],
+        sources=["azure_devops", "bitbucket", "gitlab", "manual_number"],
         tags=[Tag.CI],
     ),
     "issues": Metric(

components/shared_code/src/shared_data_model/sources/__init__.py

@@ -4,6 +4,7 @@
 from .axe import AXE_CORE, AXE_CSV, AXE_HTML_REPORTER
 from .azure_devops import AZURE_DEVOPS
 from .bandit import BANDIT
+from .bitbucket import BITBUCKET
 from .calendar_date import CALENDAR
 from .cargo_audit import CARGO_AUDIT
 from .cloc import CLOC
@@ -49,6 +50,7 @@
     "axecsv": AXE_CSV,
     "azure_devops": AZURE_DEVOPS,
     "bandit": BANDIT,
+    "bitbucket": BITBUCKET,
     "calendar": CALENDAR,
     "cargo_audit": CARGO_AUDIT,
     "cloc": CLOC,

components/shared_code/src/shared_data_model/sources/bitbucket.py

@@ -0,0 +1,82 @@
+"""Bitbucket source."""
+
+from pydantic import HttpUrl
+
+from shared_data_model.meta.entity import Entity, EntityAttribute, EntityAttributeType
+from shared_data_model.meta.source import Source
+from shared_data_model.parameters import (
+    URL,
+    BranchesToIgnore,
+    BranchMergeStatus,
+    Days,
+    PrivateToken,
+    StringParameter,
+)
+
+BITBUCKET_BRANCH_HELP_URL = HttpUrl("https://confluence.atlassian.com/bitbucketserver/branches-776639968.html")
+
+BITBUCKET = Source(
+    name="Bitbucket",
+    description="Bitbucket is a version control platform by Atlassian that supports Git, "
+    "enabling developers to collaborate on code with features like pull requests, "
+    "CI/CD, and seamless integration with tools like Jira and Trello.",
+    url=HttpUrl("https://bitbucket.org/product/guides/getting-started/overview#a-brief-overview-of-bitbucket/"),
+    documentation={
+        "generic": """```{note}
+The pagination for the Bitbucket collector has not yet been implemented,\
+and the current limit for retrieving branches for the inactive branches metric using the API is set to 100.\
+Pagination will be implemented in a future update.
+```""",
+    },
+    parameters={
+        "url": URL(
+            name="Bitbucket instance URL",
+            help="URL of the Bitbucket instance, with port if necessary, but without path. For example, "
+            "'https://bitbucket.org'.",
+            validate_on=["private_token"],
+            metrics=["inactive_branches"],
+        ),
+        "owner": StringParameter(
+            name="Owner (name of owner of the repository)",
+            short_name="owner",
+            mandatory=True,
+            help_url=HttpUrl("https://support.atlassian.com/bitbucket-cloud/docs/create-a-project/"),
+            metrics=["inactive_branches"],
+        ),
+        "repository": StringParameter(
+            name="Repository (name of the repository)",
+            short_name="repository",
+            help_url=HttpUrl("https://support.atlassian.com/bitbucket-cloud/docs/create-a-git-repository/"),
+            mandatory=True,
+            metrics=["inactive_branches"],
+        ),
+        "private_token": PrivateToken(
+            name="Private token (with read_api scope)",
+            help_url=HttpUrl("https://support.atlassian.com/bitbucket-cloud/docs/create-a-repository-access-token/"),
+            metrics=["inactive_branches"],
+        ),
+        "branches_to_ignore": BranchesToIgnore(help_url=BITBUCKET_BRANCH_HELP_URL),
+        "branch_merge_status": BranchMergeStatus(),
+        "inactive_days": Days(
+            name="Number of days since last commit after which to consider branches inactive",
+            short_name="number of days since last commit",
+            default_value="7",
+            metrics=["inactive_branches"],
+        ),
+    },
+    entities={
+        "inactive_branches": Entity(
+            name="branch",
+            name_plural="branches",
+            attributes=[
+                EntityAttribute(name="Branch name", key="name", url="url"),
+                EntityAttribute(
+                    name="Date of most recent commit",
+                    key="commit_date",
+                    type=EntityAttributeType.DATE,
+                ),
+                EntityAttribute(name="Merge status"),
+            ],
+        )
+    },
+)

components/shared_code/src/shared_data_model/sources/quality_time.py

@@ -172,6 +172,7 @@
                 "Axe-core",
                 "Azure DevOps Server",
                 "Bandit",
+                "Bitbucket",
                 "Calendar date",
                 "Cargo Audit",
                 "Checkmarx CxSAST",
@@ -223,6 +224,7 @@
                 "Axe-core": "axe_core",
                 "Azure DevOps Server": "azure_devops",
                 "Bandit": "bandit",
+                "Bitbucket": "bitbucket",
                 "Calendar date": "calendar",
                 "Cargo Audit": "cargo_audit",
                 "Checkmarx CxSAST": "cxsast",

docs/src/changelog.md

@@ -22,6 +22,7 @@ If your currently installed *Quality-time* version is not the latest version, pl
 
 ### Added
 
+- Support Bitbucket as source for the 'inactive branches' metric. Note that the amount of branches checked is limited to 100 because pagination for the Bitbucket API has not been implemented yet. Closes [#10083](https://github.com/ICTU/quality-time/issues/10083).
 - When measuring missing metrics, make the subject type and the metric type of the missing metrics link to the reference documentation. Closes [#10528](https://github.com/ICTU/quality-time/issues/10528).
 - Allow for measuring the source up-to-dateness of Trivy JSON reports. Closes [#10608](https://github.com/ICTU/quality-time/issues/10608).
 - Allow for measuring the source up-to-dateness of Harbor JSON reports. Closes [#10609](https://github.com/ICTU/quality-time/issues/10609).