diff --git a/components/collector/.vulture_ignore_list.py b/components/collector/.vulture_ignore_list.py index d264b68aeb..a14f4aa291 100644 --- a/components/collector/.vulture_ignore_list.py +++ b/components/collector/.vulture_ignore_list.py @@ -21,6 +21,8 @@ AzureDevopsUserStoryPoints # unused class (src/source_collectors/azure_devops/user_story_points.py:12) BanditSecurityWarnings # unused class (src/source_collectors/bandit/security_warnings.py:10) BanditSourceUpToDateness # unused class (src/source_collectors/bandit/source_up_to_dateness.py:10) +last_commit # unused variable (src/source_collectors/bitbucket/inactive_branches.py:32) +BitbucketInactiveBranches # unused class (src/source_collectors/bitbucket/inactive_branches.py:35) CalendarSourceUpToDateness # unused class (src/source_collectors/calendar/source_up_to_dateness.py:13) CalendarTimeRemaining # unused class (src/source_collectors/calendar/time_remaining.py:13) kind # unused variable (src/source_collectors/cargo_audit/security_warnings.py:42) diff --git a/components/collector/src/source_collectors/__init__.py b/components/collector/src/source_collectors/__init__.py index d270dfafb0..d24e1b99d8 100644 --- a/components/collector/src/source_collectors/__init__.py +++ b/components/collector/src/source_collectors/__init__.py @@ -22,6 +22,7 @@ from .azure_devops.user_story_points import AzureDevopsUserStoryPoints from .bandit.security_warnings import BanditSecurityWarnings from .bandit.source_up_to_dateness import BanditSourceUpToDateness +from .bitbucket.inactive_branches import BitbucketInactiveBranches from .calendar.source_up_to_dateness import CalendarSourceUpToDateness from .calendar.time_remaining import CalendarTimeRemaining from .cargo_audit.security_warnings import CargoAuditSecurityWarnings diff --git a/components/collector/src/source_collectors/bitbucket/__init__.py b/components/collector/src/source_collectors/bitbucket/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/components/collector/src/source_collectors/bitbucket/base.py b/components/collector/src/source_collectors/bitbucket/base.py new file mode 100644 index 0000000000..4ce76284d7 --- /dev/null +++ b/components/collector/src/source_collectors/bitbucket/base.py @@ -0,0 +1,33 @@ +"""Bitbucket collector base classes.""" + +from abc import ABC + +from base_collectors import SourceCollector +from collector_utilities.functions import add_query +from collector_utilities.type import URL + + +class BitbucketBase(SourceCollector, ABC): + """Base class for Bitbucket collectors.""" + + def _basic_auth_credentials(self) -> tuple[str, str] | None: + """Override to return None, as the private token is passed as header.""" + return None + + def _headers(self) -> dict[str, str]: + """Extend to add the private token, if any, to the headers.""" + headers = super()._headers() + if private_token := self._parameter("private_token"): + headers["Authorization"] = "Bearer " + str(private_token) + return headers + + +class BitbucketProjectBase(BitbucketBase, ABC): + """Base class for Bitbucket collectors for a specific project.""" + + async def _bitbucket_api_url(self, api: str) -> URL: + """Return a Bitbucket API url for a project, if present in the parameters.""" + url = await super()._api_url() + project = f"{self._parameter('owner')}/repos/{self._parameter('repository')}" + api_url = URL(f"{url}/rest/api/1.0/projects/{project}" + (f"/{api}" if api else "")) + return add_query(api_url, "limit=100&details=true") diff --git a/components/collector/src/source_collectors/bitbucket/inactive_branches.py b/components/collector/src/source_collectors/bitbucket/inactive_branches.py new file mode 100644 index 0000000000..57fca64837 --- /dev/null +++ b/components/collector/src/source_collectors/bitbucket/inactive_branches.py @@ -0,0 +1,85 @@ +"""Bitbucket inactive branches collector.""" + +from datetime import datetime +from typing import cast + +from base_collectors import BranchType, InactiveBranchesSourceCollector +from collector_utilities.date_time import datetime_from_timestamp +from collector_utilities.exceptions import NotFoundError +from collector_utilities.type import URL +from model import SourceResponses + +from .base import BitbucketProjectBase + + +class BitbucketBranchInfoError(NotFoundError): + """Bitbucket branch info is missing.""" + + def __init__(self, project: str) -> None: + tip = ( + "Please check if the repository (name with owner) and access token (with repo scope) are " + "configured correctly." + ) + super().__init__("Branch info for repository", project, extra=tip) + + +class BitbucketBranchType(BranchType): + """Bitbucket branch information as returned by the API.""" + + id: str + default: bool + last_commit: datetime + + +class BitbucketInactiveBranches[Branch: BitbucketBranchType](BitbucketProjectBase, InactiveBranchesSourceCollector): + """Collector for inactive branches.""" + + async def _api_url(self) -> URL: + """Override to return the branches API.""" + return await self._bitbucket_api_url("branches") + + async def _landing_url(self, responses: SourceResponses) -> URL: + """Extend to add the project branches.""" + return URL(f"{await super()._landing_url(responses)}/{self._parameter('project')}/browse") + + async def _branches(self, responses: SourceResponses) -> list[BitbucketBranchType]: + """Return a list of branches from the responses.""" + branches = [] + metadata = "com.atlassian.bitbucket.server.bitbucket-branch:latest-commit-metadata" + for response in responses: + json = await response.json() + branches.extend( + [ + BitbucketBranchType( + name=branch["displayId"], + default=branch["isDefault"], + last_commit=datetime_from_timestamp(branch["metadata"][metadata]["committerTimestamp"]), + id=branch["id"], + ) + for branch in json["values"] + ] + ) + if len(branches) == 0: + project = f"projects/{self._parameter('owner')}/repos/{self._parameter('repository')}" + raise BitbucketBranchInfoError(project) + return branches + + def _is_default_branch(self, branch: Branch) -> bool: + """Return whether the branch is the default branch.""" + return branch["default"] + + def _is_branch_merged(self, branch: Branch) -> bool: + """Return whether the branch has been merged with the default branch.""" + """The merged value is always set to false because the Bitbucket API does not include a merged field.""" + return False + + def _commit_datetime(self, branch: Branch) -> datetime: + """Override to parse the commit date from the branch.""" + return branch["last_commit"] + + def _branch_landing_url(self, branch: Branch) -> URL: + """Override to get the landing URL from the branch.""" + instance_url = super()._parameter("url") + project = f"projects/{self._parameter('owner')}/repos/{self._parameter('repository')}/browse?at=" + branch_id = str(branch.get("id")).lstrip("/") + return cast(URL, f"{instance_url}/{project}{branch_id or ''}") diff --git a/components/collector/tests/source_collectors/bitbucket/__init__.py b/components/collector/tests/source_collectors/bitbucket/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/components/collector/tests/source_collectors/bitbucket/base.py b/components/collector/tests/source_collectors/bitbucket/base.py new file mode 100644 index 0000000000..4032ddf636 --- /dev/null +++ b/components/collector/tests/source_collectors/bitbucket/base.py @@ -0,0 +1,16 @@ +"""Bitbucket unit test base classes.""" + +from tests.source_collectors.source_collector_test_case import SourceCollectorTestCase + + +class BitbucketTestCase(SourceCollectorTestCase): + """Base class for testing Bitbucket collectors.""" + + SOURCE_TYPE = "bitbucket" + + def setUp(self): + """Extend to add generic test fixtures.""" + super().setUp() + self.set_source_parameter("branch", "branch") + self.set_source_parameter("owner", "owner") + self.set_source_parameter("repository", "repository") diff --git a/components/collector/tests/source_collectors/bitbucket/test_inactive_branches.py b/components/collector/tests/source_collectors/bitbucket/test_inactive_branches.py new file mode 100644 index 0000000000..12f97d4fe0 --- /dev/null +++ b/components/collector/tests/source_collectors/bitbucket/test_inactive_branches.py @@ -0,0 +1,90 @@ +"""Unit tests for the Bitbucket inactive branches collector.""" + +from datetime import datetime +from unittest.mock import AsyncMock + +from dateutil.tz import tzutc + +from .base import BitbucketTestCase + + +class BitbucketInactiveBranchesTest(BitbucketTestCase): + """Unit tests for the inactive branches metric.""" + + METRIC_TYPE = "inactive_branches" + WEB_URL = "https://bitbucket/projects/owner/repos/repository/browse?at=" + + def setUp(self): + """Extend to setup fixtures.""" + super().setUp() + self.set_source_parameter("branches_to_ignore", ["ignored_.*"]) + main = self.create_branch("main", default=True) + unmerged = self.create_branch("unmerged_branch") + ignored = self.create_branch("ignored_branch") + active_unmerged = self.create_branch("active_unmerged_branch", active=True) + self.branches = self.create_branches_json([main, unmerged, ignored, active_unmerged]) + self.unmerged_branch_entity = self.create_entity("unmerged_branch") + self.entities = [self.unmerged_branch_entity] + self.landing_url = ( + "https://bitbucket/rest/api/1.0/projects/owner/repos/repository/branches?limit=100&details=true" + ) + + def create_branch( + self, name: str, *, default: bool = False, active: bool = False + ) -> dict[str, str | bool | dict[str, dict[str, float | int]]]: + """Create a Bitbucket branch.""" + commit_date = (datetime.now(tz=tzutc()).timestamp() if active else 1554197584) * 1000 + return { + "id": "refs/heads/" + name, + "displayId": name, + "type": "BRANCH", + "latestCommit": "ef6a9d214d509461f62f5f79b6444db55aaecc78", + "latestChangeset": "ef6a9d214d509461f62f5f79b6444db55aaecc78", + "isDefault": default, + "metadata": { + "com.atlassian.bitbucket.server.bitbucket-branch:latest-commit-metadata": { + "committerTimestamp": commit_date + } + }, + } + + def create_branches_json(self, branches): + """Create an entity.""" + return {"size": len(branches), "limit": 25, "isLastPage": True, "start": 0, "values": branches} + + def create_entity(self, name: str) -> dict[str, str]: + """Create an entity.""" + return { + "key": name, + "name": name, + "commit_date": "2019-04-02", + "merge_status": "unmerged", + "url": self.WEB_URL + "refs/heads/" + name, + } + + async def test_inactive_branches(self): + """Test that the number of inactive branches can be measured.""" + response = await self.collect(get_request_json_return_value=self.branches) + self.assert_measurement(response, value="1", entities=self.entities, landing_url=self.landing_url) + + async def test_unmerged_inactive_branches(self): + """Test that the number of unmerged inactive branches can be measured.""" + self.set_source_parameter("branch_merge_status", ["unmerged"]) + response = await self.collect(get_request_json_return_value=self.branches) + self.assert_measurement( + response, value="1", entities=[self.unmerged_branch_entity], landing_url=self.landing_url + ) + + async def test_private_token(self): + """Test that the private token is used.""" + self.set_source_parameter("private_token", "token") + inactive_branches_json = {"values": None} + inactive_branches_response = AsyncMock() + execute = AsyncMock(side_effect=[inactive_branches_response]) + inactive_branches_response.json = AsyncMock(return_value=inactive_branches_json) + response = await self.collect(get_request_json_return_value=execute) + self.assert_measurement( + response, + landing_url=self.landing_url, + parse_error="Branch info for repository", + ) diff --git a/components/shared_code/src/shared_data_model/logos/bitbucket.png b/components/shared_code/src/shared_data_model/logos/bitbucket.png new file mode 100644 index 0000000000..ef5cdb0847 Binary files /dev/null and b/components/shared_code/src/shared_data_model/logos/bitbucket.png differ diff --git a/components/shared_code/src/shared_data_model/metrics.py b/components/shared_code/src/shared_data_model/metrics.py index 8873bddc73..21523e69ef 100644 --- a/components/shared_code/src/shared_data_model/metrics.py +++ b/components/shared_code/src/shared_data_model/metrics.py @@ -135,7 +135,7 @@ change-your-default-branch).""", unit=Unit.BRANCHES, near_target="5", - sources=["azure_devops", "gitlab", "manual_number"], + sources=["azure_devops", "bitbucket", "gitlab", "manual_number"], tags=[Tag.CI], ), "issues": Metric( diff --git a/components/shared_code/src/shared_data_model/sources/__init__.py b/components/shared_code/src/shared_data_model/sources/__init__.py index cf11fdae77..ae178044c7 100644 --- a/components/shared_code/src/shared_data_model/sources/__init__.py +++ b/components/shared_code/src/shared_data_model/sources/__init__.py @@ -4,6 +4,7 @@ from .axe import AXE_CORE, AXE_CSV, AXE_HTML_REPORTER from .azure_devops import AZURE_DEVOPS from .bandit import BANDIT +from .bitbucket import BITBUCKET from .calendar_date import CALENDAR from .cargo_audit import CARGO_AUDIT from .cloc import CLOC @@ -49,6 +50,7 @@ "axecsv": AXE_CSV, "azure_devops": AZURE_DEVOPS, "bandit": BANDIT, + "bitbucket": BITBUCKET, "calendar": CALENDAR, "cargo_audit": CARGO_AUDIT, "cloc": CLOC, diff --git a/components/shared_code/src/shared_data_model/sources/bitbucket.py b/components/shared_code/src/shared_data_model/sources/bitbucket.py new file mode 100644 index 0000000000..38ec02f139 --- /dev/null +++ b/components/shared_code/src/shared_data_model/sources/bitbucket.py @@ -0,0 +1,82 @@ +"""Bitbucket source.""" + +from pydantic import HttpUrl + +from shared_data_model.meta.entity import Entity, EntityAttribute, EntityAttributeType +from shared_data_model.meta.source import Source +from shared_data_model.parameters import ( + URL, + BranchesToIgnore, + BranchMergeStatus, + Days, + PrivateToken, + StringParameter, +) + +BITBUCKET_BRANCH_HELP_URL = HttpUrl("https://confluence.atlassian.com/bitbucketserver/branches-776639968.html") + +BITBUCKET = Source( + name="Bitbucket", + description="Bitbucket is a version control platform by Atlassian that supports Git, " + "enabling developers to collaborate on code with features like pull requests, " + "CI/CD, and seamless integration with tools like Jira and Trello.", + url=HttpUrl("https://bitbucket.org/product/guides/getting-started/overview#a-brief-overview-of-bitbucket/"), + documentation={ + "generic": """```{note} +The pagination for the Bitbucket collector has not yet been implemented,\ +and the current limit for retrieving branches for the inactive branches metric using the API is set to 100.\ +Pagination will be implemented in a future update. +```""", + }, + parameters={ + "url": URL( + name="Bitbucket instance URL", + help="URL of the Bitbucket instance, with port if necessary, but without path. For example, " + "'https://bitbucket.org'.", + validate_on=["private_token"], + metrics=["inactive_branches"], + ), + "owner": StringParameter( + name="Owner (name of owner of the repository)", + short_name="owner", + mandatory=True, + help_url=HttpUrl("https://support.atlassian.com/bitbucket-cloud/docs/create-a-project/"), + metrics=["inactive_branches"], + ), + "repository": StringParameter( + name="Repository (name of the repository)", + short_name="repository", + help_url=HttpUrl("https://support.atlassian.com/bitbucket-cloud/docs/create-a-git-repository/"), + mandatory=True, + metrics=["inactive_branches"], + ), + "private_token": PrivateToken( + name="Private token (with read_api scope)", + help_url=HttpUrl("https://support.atlassian.com/bitbucket-cloud/docs/create-a-repository-access-token/"), + metrics=["inactive_branches"], + ), + "branches_to_ignore": BranchesToIgnore(help_url=BITBUCKET_BRANCH_HELP_URL), + "branch_merge_status": BranchMergeStatus(), + "inactive_days": Days( + name="Number of days since last commit after which to consider branches inactive", + short_name="number of days since last commit", + default_value="7", + metrics=["inactive_branches"], + ), + }, + entities={ + "inactive_branches": Entity( + name="branch", + name_plural="branches", + attributes=[ + EntityAttribute(name="Branch name", key="name", url="url"), + EntityAttribute( + name="Date of most recent commit", + key="commit_date", + type=EntityAttributeType.DATE, + ), + EntityAttribute(name="Merge status"), + ], + ) + }, +) diff --git a/components/shared_code/src/shared_data_model/sources/quality_time.py b/components/shared_code/src/shared_data_model/sources/quality_time.py index 6474d5c0d9..38fb681b0f 100644 --- a/components/shared_code/src/shared_data_model/sources/quality_time.py +++ b/components/shared_code/src/shared_data_model/sources/quality_time.py @@ -172,6 +172,7 @@ "Axe-core", "Azure DevOps Server", "Bandit", + "Bitbucket", "Calendar date", "Cargo Audit", "Checkmarx CxSAST", @@ -223,6 +224,7 @@ "Axe-core": "axe_core", "Azure DevOps Server": "azure_devops", "Bandit": "bandit", + "Bitbucket": "bitbucket", "Calendar date": "calendar", "Cargo Audit": "cargo_audit", "Checkmarx CxSAST": "cxsast", diff --git a/docs/src/changelog.md b/docs/src/changelog.md index e0562edbaf..b107fcabb9 100644 --- a/docs/src/changelog.md +++ b/docs/src/changelog.md @@ -22,6 +22,7 @@ If your currently installed *Quality-time* version is not the latest version, pl ### Added +- Support Bitbucket as source for the 'inactive branches' metric. Note that the amount of branches checked is limited to 100 because pagination for the Bitbucket API has not been implemented yet. Closes [#10083](https://github.com/ICTU/quality-time/issues/10083). - When measuring missing metrics, make the subject type and the metric type of the missing metrics link to the reference documentation. Closes [#10528](https://github.com/ICTU/quality-time/issues/10528). - Allow for measuring the source up-to-dateness of Trivy JSON reports. Closes [#10608](https://github.com/ICTU/quality-time/issues/10608). - Allow for measuring the source up-to-dateness of Harbor JSON reports. Closes [#10609](https://github.com/ICTU/quality-time/issues/10609).