diff --git a/idf_component_manager/cli/registry.py b/idf_component_manager/cli/registry.py index ff60787..61ecd97 100644 --- a/idf_component_manager/cli/registry.py +++ b/idf_component_manager/cli/registry.py @@ -16,10 +16,10 @@ from idf_component_manager.core import ComponentManager from idf_component_manager.utils import VersionSolverResolution from idf_component_tools import warn -from idf_component_tools.config import ConfigManager, ProfileItem +from idf_component_tools.config import ConfigManager, ProfileItem, get_profile from idf_component_tools.errors import FatalError from idf_component_tools.registry.client_errors import APIClientError -from idf_component_tools.registry.service_details import get_api_client, get_profile +from idf_component_tools.registry.service_details import get_api_client from .constants import get_profile_option, get_project_dir_option from .utils import add_options, deprecated_option diff --git a/idf_component_tools/config.py b/idf_component_tools/config.py index cfb570d..3316aeb 100644 --- a/idf_component_tools/config.py +++ b/idf_component_tools/config.py @@ -17,8 +17,9 @@ from idf_component_tools.constants import ( IDF_COMPONENT_REGISTRY_URL, + IDF_COMPONENT_STORAGE_URL, ) -from idf_component_tools.errors import FatalError +from idf_component_tools.errors import FatalError, NoSuchProfile from idf_component_tools.utils import ( Annotated, BaseModel, @@ -32,6 +33,7 @@ ) from .build_system_tools import get_idf_version +from .environment import ComponentManagerSettings RegistryUrlField = Annotated[ t.Union[ @@ -106,6 +108,10 @@ def config_dir() -> Path: return Path(os.environ.get('IDF_TOOLS_PATH') or Path.home() / '.espressif') +def config_file() -> Path: + return config_dir() / 'idf_component_manager.yml' + + def root_managed_components_dir() -> Path: return config_dir() / 'root_managed_components' / f'idf{get_idf_version()}' @@ -116,7 +122,7 @@ class ConfigError(FatalError): class ConfigManager: def __init__(self, path=None): - self.config_path = Path(path) if path else (config_dir() / 'idf_component_manager.yml') + self.config_path = Path(path) if path else config_file() self._yaml = YAML() self._raw_data: CommentedMap = None # Storage for CommentedMap from the config file @@ -191,3 +197,64 @@ def _update_data(self, config: Config) -> None: del profile_values[field_name] self._raw_data['profiles'] = profiles + + +def get_profile( + profile_name: t.Optional[str] = None, + config_path: t.Optional[str] = None, +) -> ProfileItem: + config_manager = ConfigManager(path=config_path) + config = config_manager.load() + _profile_name = ComponentManagerSettings().PROFILE or profile_name or 'default' + + if ( + _profile_name == 'default' and config.profiles.get(_profile_name) is None + ) or not _profile_name: + return ProfileItem() # empty profile + + if _profile_name in config.profiles: + return config.profiles[_profile_name] or ProfileItem() + + raise NoSuchProfile( + f'Profile "{_profile_name}" not found in config file: {config_manager.config_path}' + ) + + +def get_registry_url(profile: t.Optional[ProfileItem] = None) -> str: + """ + Env var > profile settings > default + """ + if profile is None: + profile = get_profile() + + return ( + ComponentManagerSettings().REGISTRY_URL + or (profile.registry_url if profile else IDF_COMPONENT_REGISTRY_URL) + or IDF_COMPONENT_REGISTRY_URL + ) + + +def get_storage_urls(profile: t.Optional[ProfileItem] = None) -> t.List[str]: + """ + Env var > profile settings > default + """ + if profile is None: + profile = get_profile() + + storage_url_env = ComponentManagerSettings().STORAGE_URL # fool mypy + if storage_url_env: + storage_urls = storage_url_env.split(';') + else: + storage_urls = profile.storage_urls or [] + + res = [] # sequence matters, the first url goes first + for url in storage_urls: + url = url.strip() + if url == 'default': + _url = IDF_COMPONENT_STORAGE_URL + else: + _url = url + + if _url not in res: + res.append(_url) + return res diff --git a/idf_component_tools/errors.py b/idf_component_tools/errors.py index 38df226..af37617 100644 --- a/idf_component_tools/errors.py +++ b/idf_component_tools/errors.py @@ -118,3 +118,7 @@ class RunningEnvironmentError(FatalError): class WarningAsExceptionError(FatalError): pass + + +class NoSuchProfile(FatalError): + pass diff --git a/idf_component_tools/registry/multi_storage_client.py b/idf_component_tools/registry/multi_storage_client.py index e8b27b6..35212f9 100644 --- a/idf_component_tools/registry/multi_storage_client.py +++ b/idf_component_tools/registry/multi_storage_client.py @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: 2022-2024 Espressif Systems (Shanghai) CO LTD +# SPDX-FileCopyrightText: 2022-2025 Espressif Systems (Shanghai) CO LTD # SPDX-License-Identifier: Apache-2.0 """Classes to work with ESP Component Registry""" @@ -84,11 +84,10 @@ def profile_storage_clients(self): @property @lru_cache(1) def storage_clients(self): - clients = [*self.local_storage_clients, *self.profile_storage_clients] + yield from self.local_storage_clients + yield from self.profile_storage_clients if self.registry_storage_client: - clients.append(self.registry_storage_client) - - return clients + yield self.registry_storage_client def versions(self, component_name: str, spec: str = '*') -> ComponentWithVersions: cmp_with_versions = ComponentWithVersions(component_name, []) diff --git a/idf_component_tools/registry/service_details.py b/idf_component_tools/registry/service_details.py index ff788b9..ad191f7 100644 --- a/idf_component_tools/registry/service_details.py +++ b/idf_component_tools/registry/service_details.py @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: 2022-2024 Espressif Systems (Shanghai) CO LTD +# SPDX-FileCopyrightText: 2022-2025 Espressif Systems (Shanghai) CO LTD # SPDX-License-Identifier: Apache-2.0 """Helper function to init API client""" @@ -6,76 +6,12 @@ import typing as t from idf_component_tools import ComponentManagerSettings -from idf_component_tools.config import ConfigManager, ProfileItem -from idf_component_tools.constants import ( - IDF_COMPONENT_REGISTRY_URL, - IDF_COMPONENT_STORAGE_URL, -) -from idf_component_tools.errors import FatalError +from idf_component_tools.config import ProfileItem, get_profile, get_registry_url, get_storage_urls from .api_client import APIClient from .multi_storage_client import MultiStorageClient -class NoSuchProfile(FatalError): - pass - - -def get_profile( - profile_name: t.Optional[str] = None, - config_path: t.Optional[str] = None, -) -> ProfileItem: - config_manager = ConfigManager(path=config_path) - config = config_manager.load() - _profile_name = ComponentManagerSettings().PROFILE or profile_name or 'default' - - if ( - _profile_name == 'default' and config.profiles.get(_profile_name) is None - ) or not _profile_name: - return ProfileItem() # empty profile - - if _profile_name in config.profiles: - return config.profiles[_profile_name] or ProfileItem() - - raise NoSuchProfile( - f'Profile "{profile_name}" not found in config file: {config_manager.config_path}' - ) - - -def get_registry_url(profile: t.Optional[ProfileItem] = None) -> str: - """ - Env var > profile settings > default - """ - return ( - ComponentManagerSettings().REGISTRY_URL - or (profile.registry_url if profile else IDF_COMPONENT_REGISTRY_URL) - or IDF_COMPONENT_REGISTRY_URL - ) - - -def get_storage_urls(profile: t.Optional[ProfileItem] = None) -> t.List[str]: - """ - Env var > profile settings > default - """ - storage_url_env = ComponentManagerSettings().STORAGE_URL - if storage_url_env: - _storage_urls = [url.strip() for url in storage_url_env.split(';') if url.strip()] - else: - _storage_urls = profile.storage_urls if profile else [] - - res = [] # sequence matters, the first url goes first - for url in _storage_urls: - if url == 'default': - _url = IDF_COMPONENT_STORAGE_URL - else: - _url = url - - if _url not in res: - res.append(_url) - - return res - - def get_api_client( registry_url: t.Optional[str] = None, *, diff --git a/idf_component_tools/sources/web_service.py b/idf_component_tools/sources/web_service.py index 55a32c4..0b46e95 100644 --- a/idf_component_tools/sources/web_service.py +++ b/idf_component_tools/sources/web_service.py @@ -17,6 +17,7 @@ get_format_from_path, unpack_archive, ) +from idf_component_tools.config import get_registry_url from idf_component_tools.constants import ( DEFAULT_NAMESPACE, IDF_COMPONENT_REGISTRY_URL, @@ -93,7 +94,7 @@ def download_archive(url: str, download_dir: str, save_original_filename: bool = class WebServiceSource(BaseSource): registry_url: str = Field( - default=IDF_COMPONENT_REGISTRY_URL, + default_factory=get_registry_url, validation_alias=AliasChoices( 'registry_url', 'service_url', diff --git a/idf_component_tools/utils.py b/idf_component_tools/utils.py index 17fed6b..1ca0de1 100644 --- a/idf_component_tools/utils.py +++ b/idf_component_tools/utils.py @@ -67,6 +67,7 @@ # so we need to convert them to string _http_url_adapter = TypeAdapter(HttpUrl) UrlField = Annotated[ + # return with trailing slash str, BeforeValidator(lambda value: str(_http_url_adapter.validate_python(value))) ] diff --git a/pyproject.toml b/pyproject.toml index 6d8bfc1..faa9e76 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -95,7 +95,7 @@ Changelog = "https://github.com/espressif/idf-component-manager/blob/main/CHANGE # Tools # ######### [tool.mypy] -python_version = "3.7" +python_version = "3.8" plugins = [ "pydantic.mypy" ] diff --git a/tests/conftest.py b/tests/conftest.py index 41f217f..a47818a 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -14,6 +14,7 @@ from idf_component_manager.core import ComponentManager from idf_component_tools import HINT_LEVEL, ComponentManagerSettings, get_logger +from idf_component_tools.config import config_file from idf_component_tools.hash_tools.constants import HASH_FILENAME from idf_component_tools.registry.api_client import APIClient from idf_component_tools.registry.api_models import TaskStatus @@ -463,3 +464,21 @@ def component_name(): if 'USE_REGISTRY' in os.environ: return f'test_{str(uuid4())}' return 'test' + + +@pytest.fixture +def isolate_idf_component_manager_yml(tmp_path): + config_path = config_file() + backup_path = tmp_path / 'idf_component_manager.yml' + + do_exist = config_path.is_file() + if do_exist: + shutil.move(config_path, backup_path) + yield + shutil.move(backup_path, config_path) + else: + yield + try: + config_path.unlink() + except FileNotFoundError: + pass diff --git a/tests/test_api_client.py b/tests/test_api_client.py index b5c0d04..deb7945 100644 --- a/tests/test_api_client.py +++ b/tests/test_api_client.py @@ -11,16 +11,13 @@ from idf_component_tools import LOGGING_NAMESPACE from idf_component_tools.__version__ import __version__ +from idf_component_tools.config import get_registry_url, get_storage_urls from idf_component_tools.constants import IDF_COMPONENT_REGISTRY_URL from idf_component_tools.registry.api_client import APIClient from idf_component_tools.registry.base_client import user_agent from idf_component_tools.registry.client_errors import APIClientError from idf_component_tools.registry.multi_storage_client import MultiStorageClient from idf_component_tools.registry.request_processor import join_url -from idf_component_tools.registry.service_details import ( - get_registry_url, - get_storage_urls, -) from idf_component_tools.registry.storage_client import StorageClient from idf_component_tools.semver import Version from tests.network_test_utils import use_vcr_or_real_env diff --git a/tests/test_config.py b/tests/test_config.py index 260fd58..e1244af 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: 2022-2024 Espressif Systems (Shanghai) CO LTD +# SPDX-FileCopyrightText: 2022-2025 Espressif Systems (Shanghai) CO LTD # SPDX-License-Identifier: Apache-2.0 import json @@ -10,8 +10,6 @@ ConfigError, ConfigManager, ProfileItem, -) -from idf_component_tools.registry.service_details import ( get_registry_url, get_storage_urls, ) diff --git a/tests/test_prepare_dep_dirs.py b/tests/test_prepare_dep_dirs.py index f6ffeca..2c2258c 100644 --- a/tests/test_prepare_dep_dirs.py +++ b/tests/test_prepare_dep_dirs.py @@ -168,8 +168,7 @@ def test_dependencies_with_partial_mirror(tmp_path, monkeypatch): assert lock_data['dependencies']['example/cmp'] assert lock_data['dependencies']['example/cmp']['source']['type'] == 'service' assert ( - lock_data['dependencies']['example/cmp']['source']['registry_url'] - == IDF_COMPONENT_REGISTRY_URL + lock_data['dependencies']['example/cmp']['source']['registry_url'] == 'https://notexist.me/' ) assert lock_data['dependencies']['example/cmp']['version'] == '3.0.3' diff --git a/tests/test_profile.py b/tests/test_profile.py index 6211d6a..bc15468 100644 --- a/tests/test_profile.py +++ b/tests/test_profile.py @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: 2022-2024 Espressif Systems (Shanghai) CO LTD +# SPDX-FileCopyrightText: 2022-2025 Espressif Systems (Shanghai) CO LTD # SPDX-License-Identifier: Apache-2.0 import json @@ -8,17 +8,16 @@ from jsonref import requests from pytest import fixture, raises, warns -from idf_component_tools.config import Config, ConfigError +from idf_component_tools.config import Config, ConfigError, get_profile from idf_component_tools.constants import ( DEFAULT_NAMESPACE, IDF_COMPONENT_STAGING_REGISTRY_URL, ) +from idf_component_tools.errors import NoSuchProfile from idf_component_tools.messages import UserDeprecationWarning from idf_component_tools.registry.client_errors import APIClientError from idf_component_tools.registry.service_details import ( - NoSuchProfile, get_api_client, - get_profile, get_storage_client, ) @@ -201,11 +200,12 @@ def test_storage_clients_precedence(self): local_storage_urls=['file://local1', 'file://local2'], ) - assert client.storage_clients[0].storage_url == 'file://local1' - assert client.storage_clients[1].storage_url == 'file://local2' - assert client.storage_clients[2].storage_url == 'https://something.else' + storage_clients = list(client.storage_clients) + assert storage_clients[0].storage_url == 'file://local1' + assert storage_clients[1].storage_url == 'file://local2' + assert storage_clients[2].storage_url == 'https://something.else' assert ( - client.storage_clients[3].storage_url + storage_clients[3].storage_url == requests.get(IDF_COMPONENT_STAGING_REGISTRY_URL + '/api').json()[ 'components_base_url' ] diff --git a/tests/test_registry_url.py b/tests/test_registry_url.py new file mode 100644 index 0000000..784e88f --- /dev/null +++ b/tests/test_registry_url.py @@ -0,0 +1,49 @@ +# SPDX-FileCopyrightText: 2025 Espressif Systems (Shanghai) CO LTD +# SPDX-License-Identifier: Apache-2.0 +import textwrap + +import pytest + +from idf_component_tools.config import config_file +from idf_component_tools.errors import NoSuchProfile +from idf_component_tools.sources import WebServiceSource + + +def test_set_by_env_var(monkeypatch): + monkeypatch.setenv('IDF_COMPONENT_REGISTRY_URL', 'https://foo.com') + source = WebServiceSource() + assert source.registry_url == 'https://foo.com' + + source = WebServiceSource(registry_url='https://bar.com') + assert source.registry_url == 'https://bar.com' + + +def test_set_by_profile( + monkeypatch, + isolate_idf_component_manager_yml, # noqa: ARG001 +): + monkeypatch.setenv('IDF_COMPONENT_PROFILE', 'notexist') + with pytest.raises(NoSuchProfile): + WebServiceSource() + + config_file().write_text( + textwrap.dedent(""" + profiles: + default: + registry_url: https://foo.com + + bar: + registry_url: https://bar.com + """) + ) + + monkeypatch.delenv('IDF_COMPONENT_PROFILE') + source = WebServiceSource() + assert source.registry_url == 'https://foo.com/' + + monkeypatch.setenv('IDF_COMPONENT_PROFILE', 'bar') + source = WebServiceSource() + assert source.registry_url == 'https://bar.com/' + + source = WebServiceSource(registry_url='https://baz.com') + assert source.registry_url == 'https://baz.com'