From df07bb2ec4d16c2ecbb9eea04263b3a509ebdf9b Mon Sep 17 00:00:00 2001 From: Sergei Silnov Date: Wed, 14 Aug 2024 17:47:36 +0200 Subject: [PATCH] fix: Fix filtering of files with default exclude filter patterns --- docs/en/guides/packaging_components.rst | 2 +- docs/en/reference/manifest_file.rst | 2 +- idf_component_tools/file_tools.py | 7 ++- tests/test_file_tools.py | 71 +++++++++++++++---------- 4 files changed, 49 insertions(+), 33 deletions(-) diff --git a/docs/en/guides/packaging_components.rst b/docs/en/guides/packaging_components.rst index 99dfb8e0..fc831c33 100644 --- a/docs/en/guides/packaging_components.rst +++ b/docs/en/guides/packaging_components.rst @@ -170,7 +170,7 @@ As a component developer, you may want to choose what files from the component d exclude: - "*.py" # Exclude all Python files - "**/*.list" # Exclude `.list` files in all directories - - "big_dir/**/*" # Exclude files in `big_dir` directory (but empty directory will be added to archive anyway) + - "big_dir/**/*" # Exclude `big_dir` directory and its content include: - "**/.DS_Store" # Include files excluded by default diff --git a/docs/en/reference/manifest_file.rst b/docs/en/reference/manifest_file.rst index 6be7730c..bb4c3e8a 100644 --- a/docs/en/reference/manifest_file.rst +++ b/docs/en/reference/manifest_file.rst @@ -151,7 +151,7 @@ Example: exclude: - "*.py" # Exclude all Python files - "**/*.list" # Exclude `.list` files in all directories - - "big_dir/**/*" # Exclude files in `big_dir` directory (but the empty directory will be added to the archive anyway) + - "big_dir/**/*" # Exclude `big_dir` directory and its content include: - "**/.DS_Store" # Include files excluded by default diff --git a/idf_component_tools/file_tools.py b/idf_component_tools/file_tools.py index 86484a05..f0474c9b 100644 --- a/idf_component_tools/file_tools.py +++ b/idf_component_tools/file_tools.py @@ -79,13 +79,16 @@ def exclude_paths(pattern): if exclude_default: for pattern in DEFAULT_EXCLUDE: exclude_paths(pattern) - if pattern.endswith('/**/*'): - exclude_paths(pattern[: pattern.rindex('/**/*')]) # Exclude user patterns for pattern in exclude: exclude_paths(pattern) + # Exclude all the directories + for path in list(paths): + if path.is_dir(): + paths.remove(path) + # Include everything that was explicitly added for pattern in include: include_paths(pattern) diff --git a/tests/test_file_tools.py b/tests/test_file_tools.py index 8ebc6dba..d43c3f69 100644 --- a/tests/test_file_tools.py +++ b/tests/test_file_tools.py @@ -22,46 +22,50 @@ def assets_path(tmp_path, fixtures_path): # Avoid `dirs_exist_ok=True` missing in python 2 subdir = tmp_path / 'sub' shutil.copytree(templatepath.as_posix(), subdir.as_posix()) - return subdir.as_posix() + return subdir def test_filtered_path_default(assets_path): assert filtered_paths(assets_path) == { - Path(assets_path, '1.txt'), - Path(assets_path, 'ignore.dir'), - Path(assets_path, 'ignore.dir', 'file.txt'), - Path(assets_path, 'ignore.me'), + assets_path / '1.txt', + assets_path / 'ignore.dir' / 'file.txt', + assets_path / 'ignore.me', } def test_filtered_path_no_default(assets_path): assert filtered_paths(assets_path, exclude_default=False) == { - Path(assets_path, '1.txt'), - Path(assets_path, 'ignore.dir'), - Path(assets_path, 'ignore.dir', 'file.txt'), - Path(assets_path, 'ignore.me'), - Path(assets_path, '.gitlab-ci.yml'), + assets_path / '1.txt', + assets_path / 'ignore.dir' / 'file.txt', + assets_path / 'ignore.me', + assets_path / '.gitlab-ci.yml', } -def test_filtered_path_exclude_file(assets_path): +def test_filtered_path_exclude_file_and_empty_dirs(assets_path): assert filtered_paths(assets_path, exclude=['**/file.txt']) == { - Path(assets_path, '1.txt'), - Path(assets_path, 'ignore.dir'), - Path(assets_path, 'ignore.me'), + assets_path / '1.txt', + assets_path / 'ignore.me', } -def test_filtered_path_keep_empty_dir(assets_path): +def test_filtered_path_exclude_file_and_empty_dir_kept(assets_path): + assert filtered_paths(assets_path, exclude=['**/file.txt'], include=['ignore.dir']) == { + assets_path / '1.txt', + assets_path / 'ignore.me', + assets_path / 'ignore.dir', + } + + +def test_filtered_path_removes_empty_dir(assets_path): assert filtered_paths( assets_path, exclude=[ 'ignore.dir/**/*', ], ) == { - Path(assets_path, '1.txt'), - Path(assets_path, 'ignore.me'), - Path(assets_path, 'ignore.dir'), + assets_path / '1.txt', + assets_path / 'ignore.me', } @@ -73,18 +77,18 @@ def test_filtered_path_exclude_empty_dir(assets_path): 'ignore.dir/*', ], ) == { - Path(assets_path, '1.txt'), - Path(assets_path, 'ignore.me'), + assets_path / '1.txt', + assets_path / 'ignore.me', } def test_filtered_path_exclude_dir_with_file(assets_path): - extra_path = Path(assets_path, 'ignore.dir', 'extra').as_posix() - os.mkdir(extra_path) - one_more = os.path.join(extra_path, 'one_more.txt') - shutil.copy(os.path.join(assets_path, '1.txt'), one_more) + extra_path = assets_path / 'ignore.dir' / 'extra' + extra_path.mkdir(exist_ok=True) + one_more = extra_path / 'one_more.txt' + shutil.copy(assets_path / '1.txt', one_more) - assert os.path.exists(one_more) + assert one_more.exists() assert filtered_paths( assets_path, @@ -92,10 +96,19 @@ def test_filtered_path_exclude_dir_with_file(assets_path): 'ignore.dir/*', ], ) == { - Path(assets_path, '1.txt'), - Path(assets_path, 'ignore.dir'), - Path(assets_path, 'ignore.dir', 'extra', 'one_more.txt'), - Path(assets_path, 'ignore.me'), + assets_path / '1.txt', + assets_path / 'ignore.dir' / 'extra' / 'one_more.txt', + assets_path / 'ignore.me', + } + + +def test_filtered_with_default_path(tmp_path): + (tmp_path / 'build_all.sh').touch() + (tmp_path / 'build_me').mkdir() + (tmp_path / 'build_me' / 'file').touch() + + assert filtered_paths(tmp_path, exclude_default=True) == { + tmp_path / 'build_all.sh', }