Merge branch 'fix/respect-env-vars' into 'main'

fix: use env var REGISTRY_URL and PROFILE for deps without registry_url set

Closes PACMAN-1062

See merge request espressif/idf-component-manager!488

idf_component_manager/cli/registry.py

@@ -16,10 +16,10 @@
 from idf_component_manager.core import ComponentManager
 from idf_component_manager.utils import VersionSolverResolution
 from idf_component_tools import warn
-from idf_component_tools.config import ConfigManager, ProfileItem
+from idf_component_tools.config import ConfigManager, ProfileItem, get_profile
 from idf_component_tools.errors import FatalError
 from idf_component_tools.registry.client_errors import APIClientError
-from idf_component_tools.registry.service_details import get_api_client, get_profile
+from idf_component_tools.registry.service_details import get_api_client
 
 from .constants import get_profile_option, get_project_dir_option
 from .utils import add_options, deprecated_option

idf_component_tools/config.py

@@ -17,8 +17,9 @@
 
 from idf_component_tools.constants import (
     IDF_COMPONENT_REGISTRY_URL,
+    IDF_COMPONENT_STORAGE_URL,
 )
-from idf_component_tools.errors import FatalError
+from idf_component_tools.errors import FatalError, NoSuchProfile
 from idf_component_tools.utils import (
     Annotated,
     BaseModel,
@@ -32,6 +33,7 @@
 )
 
 from .build_system_tools import get_idf_version
+from .environment import ComponentManagerSettings
 
 RegistryUrlField = Annotated[
     t.Union[
@@ -106,6 +108,10 @@ def config_dir() -> Path:
     return Path(os.environ.get('IDF_TOOLS_PATH') or Path.home() / '.espressif')
 
 
+def config_file() -> Path:
+    return config_dir() / 'idf_component_manager.yml'
+
+
 def root_managed_components_dir() -> Path:
     return config_dir() / 'root_managed_components' / f'idf{get_idf_version()}'
 
@@ -116,7 +122,7 @@ class ConfigError(FatalError):
 
 class ConfigManager:
     def __init__(self, path=None):
-        self.config_path = Path(path) if path else (config_dir() / 'idf_component_manager.yml')
+        self.config_path = Path(path) if path else config_file()
         self._yaml = YAML()
         self._raw_data: CommentedMap = None  # Storage for CommentedMap from the config file
 
@@ -191,3 +197,64 @@ def _update_data(self, config: Config) -> None:
                     del profile_values[field_name]
 
         self._raw_data['profiles'] = profiles
+
+
+def get_profile(
+    profile_name: t.Optional[str] = None,
+    config_path: t.Optional[str] = None,
+) -> ProfileItem:
+    config_manager = ConfigManager(path=config_path)
+    config = config_manager.load()
+    _profile_name = ComponentManagerSettings().PROFILE or profile_name or 'default'
+
+    if (
+        _profile_name == 'default' and config.profiles.get(_profile_name) is None
+    ) or not _profile_name:
+        return ProfileItem()  # empty profile
+
+    if _profile_name in config.profiles:
+        return config.profiles[_profile_name] or ProfileItem()
+
+    raise NoSuchProfile(
+        f'Profile "{_profile_name}" not found in config file: {config_manager.config_path}'
+    )
+
+
+def get_registry_url(profile: t.Optional[ProfileItem] = None) -> str:
+    """
+    Env var > profile settings > default
+    """
+    if profile is None:
+        profile = get_profile()
+
+    return (
+        ComponentManagerSettings().REGISTRY_URL
+        or (profile.registry_url if profile else IDF_COMPONENT_REGISTRY_URL)
+        or IDF_COMPONENT_REGISTRY_URL
+    )
+
+
+def get_storage_urls(profile: t.Optional[ProfileItem] = None) -> t.List[str]:
+    """
+    Env var > profile settings > default
+    """
+    if profile is None:
+        profile = get_profile()
+
+    storage_url_env = ComponentManagerSettings().STORAGE_URL  # fool mypy
+    if storage_url_env:
+        storage_urls = storage_url_env.split(';')
+    else:
+        storage_urls = profile.storage_urls or []
+
+    res = []  # sequence matters, the first url goes first
+    for url in storage_urls:
+        url = url.strip()
+        if url == 'default':
+            _url = IDF_COMPONENT_STORAGE_URL
+        else:
+            _url = url
+
+        if _url not in res:
+            res.append(_url)
+    return res

idf_component_tools/errors.py

@@ -118,3 +118,7 @@ class RunningEnvironmentError(FatalError):
 
 class WarningAsExceptionError(FatalError):
     pass
+
+
+class NoSuchProfile(FatalError):
+    pass

idf_component_tools/registry/multi_storage_client.py

@@ -1,4 +1,4 @@
-# SPDX-FileCopyrightText: 2022-2024 Espressif Systems (Shanghai) CO LTD
+# SPDX-FileCopyrightText: 2022-2025 Espressif Systems (Shanghai) CO LTD
 # SPDX-License-Identifier: Apache-2.0
 """Classes to work with ESP Component Registry"""
 
@@ -84,11 +84,10 @@ def profile_storage_clients(self):
     @property
     @lru_cache(1)
     def storage_clients(self):
-        clients = [*self.local_storage_clients, *self.profile_storage_clients]
+        yield from self.local_storage_clients
+        yield from self.profile_storage_clients
         if self.registry_storage_client:
-            clients.append(self.registry_storage_client)
-
-        return clients
+            yield self.registry_storage_client
 
     def versions(self, component_name: str, spec: str = '*') -> ComponentWithVersions:
         cmp_with_versions = ComponentWithVersions(component_name, [])

idf_component_tools/registry/service_details.py

@@ -1,81 +1,17 @@
-# SPDX-FileCopyrightText: 2022-2024 Espressif Systems (Shanghai) CO LTD
+# SPDX-FileCopyrightText: 2022-2025 Espressif Systems (Shanghai) CO LTD
 # SPDX-License-Identifier: Apache-2.0
 
 """Helper function to init API client"""
 
 import typing as t
 
 from idf_component_tools import ComponentManagerSettings
-from idf_component_tools.config import ConfigManager, ProfileItem
-from idf_component_tools.constants import (
-    IDF_COMPONENT_REGISTRY_URL,
-    IDF_COMPONENT_STORAGE_URL,
-)
-from idf_component_tools.errors import FatalError
+from idf_component_tools.config import ProfileItem, get_profile, get_registry_url, get_storage_urls
 
 from .api_client import APIClient
 from .multi_storage_client import MultiStorageClient
 
 
-class NoSuchProfile(FatalError):
-    pass
-
-
-def get_profile(
-    profile_name: t.Optional[str] = None,
-    config_path: t.Optional[str] = None,
-) -> ProfileItem:
-    config_manager = ConfigManager(path=config_path)
-    config = config_manager.load()
-    _profile_name = ComponentManagerSettings().PROFILE or profile_name or 'default'
-
-    if (
-        _profile_name == 'default' and config.profiles.get(_profile_name) is None
-    ) or not _profile_name:
-        return ProfileItem()  # empty profile
-
-    if _profile_name in config.profiles:
-        return config.profiles[_profile_name] or ProfileItem()
-
-    raise NoSuchProfile(
-        f'Profile "{profile_name}" not found in config file: {config_manager.config_path}'
-    )
-
-
-def get_registry_url(profile: t.Optional[ProfileItem] = None) -> str:
-    """
-    Env var > profile settings > default
-    """
-    return (
-        ComponentManagerSettings().REGISTRY_URL
-        or (profile.registry_url if profile else IDF_COMPONENT_REGISTRY_URL)
-        or IDF_COMPONENT_REGISTRY_URL
-    )
-
-
-def get_storage_urls(profile: t.Optional[ProfileItem] = None) -> t.List[str]:
-    """
-    Env var > profile settings > default
-    """
-    storage_url_env = ComponentManagerSettings().STORAGE_URL
-    if storage_url_env:
-        _storage_urls = [url.strip() for url in storage_url_env.split(';') if url.strip()]
-    else:
-        _storage_urls = profile.storage_urls if profile else []
-
-    res = []  # sequence matters, the first url goes first
-    for url in _storage_urls:
-        if url == 'default':
-            _url = IDF_COMPONENT_STORAGE_URL
-        else:
-            _url = url
-
-        if _url not in res:
-            res.append(_url)
-
-    return res
-
-
 def get_api_client(
     registry_url: t.Optional[str] = None,
     *,

idf_component_tools/sources/web_service.py

@@ -17,6 +17,7 @@
     get_format_from_path,
     unpack_archive,
 )
+from idf_component_tools.config import get_registry_url
 from idf_component_tools.constants import (
     DEFAULT_NAMESPACE,
     IDF_COMPONENT_REGISTRY_URL,
@@ -93,7 +94,7 @@ def download_archive(url: str, download_dir: str, save_original_filename: bool =
 
 class WebServiceSource(BaseSource):
     registry_url: str = Field(
-        default=IDF_COMPONENT_REGISTRY_URL,
+        default_factory=get_registry_url,
         validation_alias=AliasChoices(
             'registry_url',
             'service_url',

idf_component_tools/utils.py

@@ -67,6 +67,7 @@
 # so we need to convert them to string
 _http_url_adapter = TypeAdapter(HttpUrl)
 UrlField = Annotated[
+    # return with trailing slash
     str, BeforeValidator(lambda value: str(_http_url_adapter.validate_python(value)))
 ]
 

pyproject.toml

@@ -95,7 +95,7 @@ Changelog = "https://github.com/espressif/idf-component-manager/blob/main/CHANGE
 # Tools #
 #########
 [tool.mypy]
-python_version = "3.7"
+python_version = "3.8"
 plugins = [
     "pydantic.mypy"
 ]

tests/conftest.py

@@ -14,6 +14,7 @@
 
 from idf_component_manager.core import ComponentManager
 from idf_component_tools import HINT_LEVEL, ComponentManagerSettings, get_logger
+from idf_component_tools.config import config_file
 from idf_component_tools.hash_tools.constants import HASH_FILENAME
 from idf_component_tools.registry.api_client import APIClient
 from idf_component_tools.registry.api_models import TaskStatus
@@ -463,3 +464,21 @@ def component_name():
     if 'USE_REGISTRY' in os.environ:
         return f'test_{str(uuid4())}'
     return 'test'
+
+
+@pytest.fixture
+def isolate_idf_component_manager_yml(tmp_path):
+    config_path = config_file()
+    backup_path = tmp_path / 'idf_component_manager.yml'
+
+    do_exist = config_path.is_file()
+    if do_exist:
+        shutil.move(config_path, backup_path)
+        yield
+        shutil.move(backup_path, config_path)
+    else:
+        yield
+        try:
+            config_path.unlink()
+        except FileNotFoundError:
+            pass

tests/test_api_client.py

@@ -11,16 +11,13 @@
 
 from idf_component_tools import LOGGING_NAMESPACE
 from idf_component_tools.__version__ import __version__
+from idf_component_tools.config import get_registry_url, get_storage_urls
 from idf_component_tools.constants import IDF_COMPONENT_REGISTRY_URL
 from idf_component_tools.registry.api_client import APIClient
 from idf_component_tools.registry.base_client import user_agent
 from idf_component_tools.registry.client_errors import APIClientError
 from idf_component_tools.registry.multi_storage_client import MultiStorageClient
 from idf_component_tools.registry.request_processor import join_url
-from idf_component_tools.registry.service_details import (
-    get_registry_url,
-    get_storage_urls,
-)
 from idf_component_tools.registry.storage_client import StorageClient
 from idf_component_tools.semver import Version
 from tests.network_test_utils import use_vcr_or_real_env

tests/test_config.py

@@ -1,4 +1,4 @@
-# SPDX-FileCopyrightText: 2022-2024 Espressif Systems (Shanghai) CO LTD
+# SPDX-FileCopyrightText: 2022-2025 Espressif Systems (Shanghai) CO LTD
 # SPDX-License-Identifier: Apache-2.0
 
 import json
@@ -10,8 +10,6 @@
     ConfigError,
     ConfigManager,
     ProfileItem,
-)
-from idf_component_tools.registry.service_details import (
     get_registry_url,
     get_storage_urls,
 )

tests/test_prepare_dep_dirs.py

@@ -168,8 +168,7 @@ def test_dependencies_with_partial_mirror(tmp_path, monkeypatch):
     assert lock_data['dependencies']['example/cmp']
     assert lock_data['dependencies']['example/cmp']['source']['type'] == 'service'
     assert (
-        lock_data['dependencies']['example/cmp']['source']['registry_url']
-        == IDF_COMPONENT_REGISTRY_URL
+        lock_data['dependencies']['example/cmp']['source']['registry_url'] == 'https://notexist.me/'
     )
     assert lock_data['dependencies']['example/cmp']['version'] == '3.0.3'
 

tests/test_profile.py

@@ -1,4 +1,4 @@
-# SPDX-FileCopyrightText: 2022-2024 Espressif Systems (Shanghai) CO LTD
+# SPDX-FileCopyrightText: 2022-2025 Espressif Systems (Shanghai) CO LTD
 # SPDX-License-Identifier: Apache-2.0
 
 import json
@@ -8,17 +8,16 @@
 from jsonref import requests
 from pytest import fixture, raises, warns
 
-from idf_component_tools.config import Config, ConfigError
+from idf_component_tools.config import Config, ConfigError, get_profile
 from idf_component_tools.constants import (
     DEFAULT_NAMESPACE,
     IDF_COMPONENT_STAGING_REGISTRY_URL,
 )
+from idf_component_tools.errors import NoSuchProfile
 from idf_component_tools.messages import UserDeprecationWarning
 from idf_component_tools.registry.client_errors import APIClientError
 from idf_component_tools.registry.service_details import (
-    NoSuchProfile,
     get_api_client,
-    get_profile,
     get_storage_client,
 )
 
@@ -201,11 +200,12 @@ def test_storage_clients_precedence(self):
             local_storage_urls=['file://local1', 'file://local2'],
         )
 
-        assert client.storage_clients[0].storage_url == 'file://local1'
-        assert client.storage_clients[1].storage_url == 'file://local2'
-        assert client.storage_clients[2].storage_url == 'https://something.else'
+        storage_clients = list(client.storage_clients)
+        assert storage_clients[0].storage_url == 'file://local1'
+        assert storage_clients[1].storage_url == 'file://local2'
+        assert storage_clients[2].storage_url == 'https://something.else'
         assert (
-            client.storage_clients[3].storage_url
+            storage_clients[3].storage_url
             == requests.get(IDF_COMPONENT_STAGING_REGISTRY_URL + '/api').json()[
                 'components_base_url'
             ]