Feature keep file 1135 (#1139)

* Added gitignorefile

* Fixed gitignorefile for os.PathLike paths

* --keep now follows .gitignore rules

* Fixed ruff QA error

* Added support for .osxphotos_keep file

* Added reference to .osxphotos_keep

* Added tests for .osxphotos_keep

* Updated help text for --cleanup, --keep

tests/test_cli.py

@@ -3387,6 +3387,7 @@ def test_export_aae():
         files = glob.glob("*.*")
         assert sorted(files) == sorted(CLI_EXPORT_AAE_FILENAMES)
 
+
 def test_export_aae_as_hardlink():
     """Test export with --export-aae and --export-as-hardlink"""
 
@@ -3411,6 +3412,7 @@ def test_export_aae_as_hardlink():
         files = glob.glob("*.*")
         assert sorted(files) == sorted(CLI_EXPORT_AAE_FILENAMES)
 
+
 def test_export_sidecar():
     """test --sidecar"""
 
@@ -6564,13 +6566,14 @@ def test_export_cleanup_keep():
         assert pathlib.Path("./report.db").is_file()
 
 
-def test_export_cleanup_keep_relative_path():
-    """test export with --cleanup --keep options with relative paths"""
+def test_export_cleanup_keep_leading_slash():
+    """test export with --cleanup --keep options when pattern has leading slash"""
 
     runner = CliRunner()
     cwd = os.getcwd()
     # pylint: disable=not-context-manager
     with runner.isolated_filesystem():
+        tmpdir = os.getcwd()
         result = runner.invoke(export, [os.path.join(cwd, CLI_PHOTOS_DB), ".", "-V"])
         assert result.exit_code == 0
 
@@ -6602,11 +6605,11 @@ def test_export_cleanup_keep_relative_path():
                 "--update",
                 "--cleanup",
                 "--keep",
-                "keep_me",
+                f"/keep_me/",
                 "--keep",
-                "keep_me.txt",
+                f"/keep_me.txt",
                 "--keep",
-                "*.db",
+                f"/*.db",
                 "--dry-run",
             ],
         )
@@ -6625,11 +6628,11 @@ def test_export_cleanup_keep_relative_path():
                 "--update",
                 "--cleanup",
                 "--keep",
-                "keep_me",
+                f"/keep_me/",
                 "--keep",
-                "keep_me.txt",
+                f"/keep_me.txt",
                 "--keep",
-                "*.db",
+                f"/*.db",
             ],
         )
         assert "Deleted: 2 files, 2 directories" in result.output
@@ -6643,6 +6646,94 @@ def test_export_cleanup_keep_relative_path():
         assert pathlib.Path("./report.db").is_file()
 
 
+def test_export_cleanup_keep_relative_path():
+    """test export with --cleanup --keep options with relative paths"""
+
+    runner = CliRunner()
+    cwd = os.getcwd()
+    # pylint: disable=not-context-manager
+    with runner.isolated_filesystem():
+        result = runner.invoke(export, [os.path.join(cwd, CLI_PHOTOS_DB), ".", "-V"])
+        assert result.exit_code == 0
+
+        # create file and a directory that should be deleted
+        os.mkdir("./empty_dir")
+        os.mkdir("./delete_me_dir")
+        with open("./delete_me.txt", "w") as fd:
+            fd.write("delete me!")
+        with open("./delete_me_dir/delete_me.txt", "w") as fd:
+            fd.write("delete me!")
+
+        # create files and directories that should be kept
+        os.mkdir("./keep_me")
+        os.mkdir("./keep_me/keep_me_2")
+        with open("./keep_me.txt", "w") as fd:
+            fd.write("keep me!")
+        with open("./report.db", "w") as fd:
+            fd.write("keep me!")
+        with open("./keep_me/keep_me.txt", "w") as fd:
+            fd.write("keep me")
+
+        # for negation rule
+        with open("./keep_me/keep_me.db", "w") as fd:
+            fd.write("keep me")
+
+        # run cleanup with dry-run
+        result = runner.invoke(
+            export,
+            [
+                os.path.join(cwd, CLI_PHOTOS_DB),
+                ".",
+                "-V",
+                "--update",
+                "--cleanup",
+                "--keep",
+                "keep_me/",
+                "--keep",
+                "keep_me.txt",
+                "--keep",
+                "*.db",
+                "--dry-run",
+                "--keep",
+                "!keep_me/keep_me.db",
+            ],
+        )
+        assert "Deleted: 3 files, 1 directory" in result.output
+        assert pathlib.Path("./delete_me.txt").is_file()
+        assert pathlib.Path("./delete_me_dir/delete_me.txt").is_file()
+        assert pathlib.Path("./empty_dir").is_dir()
+
+        # run cleanup without dry-run
+        result = runner.invoke(
+            export,
+            [
+                os.path.join(cwd, CLI_PHOTOS_DB),
+                ".",
+                "-V",
+                "--update",
+                "--cleanup",
+                "--keep",
+                "keep_me/",
+                "--keep",
+                "keep_me.txt",
+                "--keep",
+                "*.db",
+                "--keep",
+                "!keep_me/keep_me.db",
+            ],
+        )
+        assert "Deleted: 3 files, 2 directories" in result.output
+        assert not pathlib.Path("./delete_me.txt").is_file()
+        assert not pathlib.Path("./delete_me_dir/delete_me_too.txt").is_file()
+        assert not pathlib.Path("./empty_dir").is_dir()
+        assert not pathlib.Path("./keep_me/keep_me.db").is_file()
+        assert pathlib.Path("./keep_me.txt").is_file()
+        assert pathlib.Path("./keep_me").is_dir()
+        assert pathlib.Path("./keep_me/keep_me.txt").is_file()
+        assert pathlib.Path("./keep_me/keep_me_2").is_dir()
+        assert pathlib.Path("./report.db").is_file()
+
+
 def test_export_cleanup_exportdb_report():
     """test export with --cleanup flag results show in exportdb --report"""
 
@@ -6682,6 +6773,159 @@ def test_export_cleanup_exportdb_report():
         assert len(deleted_files) == 2
 
 
+def test_export_cleanup_osxphotos_keep():
+    """test export with --cleanup with a .osxphotos_keep file"""
+
+    runner = CliRunner()
+    cwd = os.getcwd()
+    # pylint: disable=not-context-manager
+    with runner.isolated_filesystem():
+        tmpdir = os.getcwd()
+        result = runner.invoke(export, [os.path.join(cwd, CLI_PHOTOS_DB), ".", "-V"])
+        assert result.exit_code == 0
+
+        # create file and a directory that should be deleted
+        os.mkdir("./empty_dir")
+        os.mkdir("./delete_me_dir")
+        with open("./delete_me.txt", "w") as fd:
+            fd.write("delete me!")
+        with open("./delete_me_dir/delete_me.txt", "w") as fd:
+            fd.write("delete me!")
+
+        # create files and directories that should be kept
+        os.mkdir("./keep_me")
+        os.mkdir("./keep_me/keep_me_2")
+        with open("./keep_me.txt", "w") as fd:
+            fd.write("keep me!")
+        with open("./report.db", "w") as fd:
+            fd.write("keep me!")
+        with open("./keep_me/keep_me.txt", "w") as fd:
+            fd.write("keep me")
+
+        with open(".osxphotos_keep", "w") as fd:
+            fd.write("/keep_me/\n")
+            fd.write("/keep_me.txt\n")
+            fd.write("/*.db\n")
+
+        # run cleanup with dry-run
+        result = runner.invoke(
+            export,
+            [
+                ".",
+                "--library",
+                os.path.join(cwd, CLI_PHOTOS_DB),
+                "-V",
+                "--update",
+                "--cleanup",
+                "--dry-run",
+            ],
+        )
+        assert "Deleted: 2 files, 1 directory" in result.output
+        assert pathlib.Path("./delete_me.txt").is_file()
+        assert pathlib.Path("./delete_me_dir/delete_me.txt").is_file()
+        assert pathlib.Path("./empty_dir").is_dir()
+
+        # run cleanup without dry-run
+        result = runner.invoke(
+            export,
+            [
+                ".",
+                "--library",
+                os.path.join(cwd, CLI_PHOTOS_DB),
+                "-V",
+                "--update",
+                "--cleanup",
+            ],
+        )
+        assert "Deleted: 2 files, 2 directories" in result.output
+        assert not pathlib.Path("./delete_me.txt").is_file()
+        assert not pathlib.Path("./delete_me_dir/delete_me_too.txt").is_file()
+        assert not pathlib.Path("./empty_dir").is_dir()
+        assert pathlib.Path("./keep_me.txt").is_file()
+        assert pathlib.Path("./keep_me").is_dir()
+        assert pathlib.Path("./keep_me/keep_me.txt").is_file()
+        assert pathlib.Path("./keep_me/keep_me_2").is_dir()
+        assert pathlib.Path("./report.db").is_file()
+
+
+def test_export_cleanup_osxphotos_keep_keep():
+    """test export with --cleanup with a .osxphotos_keep file and --keep"""
+
+    runner = CliRunner()
+    cwd = os.getcwd()
+    # pylint: disable=not-context-manager
+    with runner.isolated_filesystem():
+        tmpdir = os.getcwd()
+        result = runner.invoke(export, [os.path.join(cwd, CLI_PHOTOS_DB), ".", "-V"])
+        assert result.exit_code == 0
+
+        # create file and a directory that should be deleted
+        os.mkdir("./empty_dir")
+        os.mkdir("./delete_me_dir")
+        with open("./delete_me.txt", "w") as fd:
+            fd.write("delete me!")
+        with open("./delete_me_dir/delete_me.txt", "w") as fd:
+            fd.write("delete me!")
+
+        # create files and directories that should be kept
+        os.mkdir("./keep_me")
+        os.mkdir("./keep_me/keep_me_2")
+        with open("./keep_me.txt", "w") as fd:
+            fd.write("keep me!")
+        with open("./report.db", "w") as fd:
+            fd.write("keep me!")
+        with open("./keep_me/keep_me.txt", "w") as fd:
+            fd.write("keep me")
+
+        with open(".osxphotos_keep", "w") as fd:
+            fd.write("/keep_me/\n")
+            fd.write("/keep_me.txt\n")
+
+        # run cleanup with dry-run
+        result = runner.invoke(
+            export,
+            [
+                ".",
+                "--library",
+                os.path.join(cwd, CLI_PHOTOS_DB),
+                "-V",
+                "--update",
+                "--cleanup",
+                "--dry-run",
+                "--keep",
+                "/*.db",
+            ],
+        )
+        assert "Deleted: 2 files, 1 directory" in result.output
+        assert pathlib.Path("./delete_me.txt").is_file()
+        assert pathlib.Path("./delete_me_dir/delete_me.txt").is_file()
+        assert pathlib.Path("./empty_dir").is_dir()
+
+        # run cleanup without dry-run
+        result = runner.invoke(
+            export,
+            [
+                ".",
+                "--library",
+                os.path.join(cwd, CLI_PHOTOS_DB),
+                "-V",
+                "--update",
+                "--cleanup",
+                "--keep",
+                "/*.db",
+            ],
+        )
+        assert "Deleted: 2 files, 2 directories" in result.output
+        assert not pathlib.Path("./delete_me.txt").is_file()
+        assert not pathlib.Path("./delete_me_dir/delete_me_too.txt").is_file()
+        assert not pathlib.Path("./empty_dir").is_dir()
+        assert pathlib.Path("./keep_me.txt").is_file()
+        assert pathlib.Path("./keep_me").is_dir()
+        assert pathlib.Path("./keep_me/keep_me.txt").is_file()
+        assert pathlib.Path("./keep_me/keep_me_2").is_dir()
+        assert pathlib.Path("./report.db").is_file()
+
+
 def test_save_load_config():
     """test --save-config, --load-config"""
 

tests/test_gitignorefile_cache.py

@@ -0,0 +1,137 @@
+import io
+import itertools
+import os
+import stat
+import tempfile
+import unittest
+import unittest.mock
+
+import osxphotos.gitignorefile
+
+
+class TestCache(unittest.TestCase):
+    def test_simple(self):
+        def normalize_path(path):
+            return os.path.abspath(path).replace(os.sep, "/")
+
+        class StatResult:
+            def __init__(self, is_file=False):
+                self.st_ino = id(self)
+                self.st_dev = 0
+                self.st_mode = stat.S_IFREG if is_file else stat.S_IFDIR
+
+            def isdir(self):
+                return self.st_mode == stat.S_IFDIR
+
+            def isfile(self):
+                return self.st_mode == stat.S_IFREG
+
+        class Stat:
+            def __init__(self, directories, files):
+                self.__filesystem = {}
+                for path in directories:
+                    self.__filesystem[normalize_path(path)] = StatResult()
+                for path in files:
+                    self.__filesystem[normalize_path(path)] = StatResult(True)
+
+            def __call__(self, path):
+                try:
+                    return self.__filesystem[normalize_path(path)]
+
+                except KeyError:
+                    raise FileNotFoundError()
+
+        for ignore_file_name in (".gitignore", ".mylovelytoolignore"):
+            with self.subTest(ignore_file_name=ignore_file_name):
+                my_stat = Stat(
+                    [
+                        "/home/vladimir/project/directory/subdirectory",
+                        "/home/vladimir/project/directory",
+                        "/home/vladimir/project",
+                        "/home/vladimir",
+                        "/home",
+                        "/",
+                    ],
+                    [
+                        "/home/vladimir/project/directory/subdirectory/subdirectory/file.txt",
+                        "/home/vladimir/project/directory/subdirectory/subdirectory/file2.txt",
+                        "/home/vladimir/project/directory/subdirectory/subdirectory/file3.txt",
+                        "/home/vladimir/project/directory/subdirectory/file.txt",
+                        "/home/vladimir/project/directory/subdirectory/file2.txt",
+                        "/home/vladimir/project/directory/%s" % ignore_file_name,
+                        "/home/vladimir/project/directory/file.txt",
+                        "/home/vladimir/project/directory/file2.txt",
+                        "/home/vladimir/project/file.txt",
+                        "/home/vladimir/project/%s" % ignore_file_name,
+                        "/home/vladimir/file.txt",
+                    ],
+                )
+
+                def mock_open(path):
+                    data = {
+                        normalize_path(
+                            "/home/vladimir/project/directory/%s" % ignore_file_name
+                        ): ["file.txt"],
+                        normalize_path(
+                            "/home/vladimir/project/%s" % ignore_file_name
+                        ): ["file2.txt"],
+                    }
+
+                    statistics["open"] += 1
+                    try:
+                        return io.StringIO("\n".join(data[normalize_path(path)]))
+
+                    except KeyError:
+                        raise FileNotFoundError()
+
+                def mock_isdir(path):
+                    statistics["isdir"] += 1
+                    try:
+                        return my_stat(path).isdir()
+                    except FileNotFoundError:
+                        return False
+
+                def mock_isfile(path):
+                    statistics["isfile"] += 1
+                    try:
+                        return my_stat(path).isfile()
+                    except FileNotFoundError:
+                        return False
+
+                data = {
+                    "/home/vladimir/project/directory/subdirectory/file.txt": True,
+                    "/home/vladimir/project/directory/subdirectory/file2.txt": True,
+                    "/home/vladimir/project/directory/subdirectory/subdirectory/file.txt": True,
+                    "/home/vladimir/project/directory/subdirectory/subdirectory/file2.txt": True,
+                    "/home/vladimir/project/directory/subdirectory/subdirectory/file3.txt": False,
+                    "/home/vladimir/project/directory/file.txt": True,
+                    "/home/vladimir/project/directory/file2.txt": True,
+                    "/home/vladimir/project/file.txt": False,
+                    "/home/vladimir/file.txt": False,  # No rules and no `isdir` calls for this file.
+                }
+
+                # 9! == 362880 combinations.
+                for permutation in itertools.islice(
+                    itertools.permutations(data.items()), 0, None, 6 * 8
+                ):
+                    statistics = {"open": 0, "isdir": 0, "isfile": 0}
+
+                    with unittest.mock.patch("builtins.open", mock_open):
+                        with unittest.mock.patch("os.path.isdir", mock_isdir):
+                            with unittest.mock.patch("os.path.isfile", mock_isfile):
+                                matches = osxphotos.gitignorefile.Cache(
+                                    ignore_names=[ignore_file_name]
+                                )
+                                for path, expected in permutation:
+                                    self.assertEqual(matches(path), expected)
+
+                    self.assertEqual(statistics["open"], 2)
+                    self.assertEqual(statistics["isdir"], len(data) - 1)
+                    self.assertEqual(statistics["isfile"], 7)  # Unique path fragments.
+
+    def test_wrong_symlink(self):
+        with tempfile.TemporaryDirectory() as d:
+            matches = osxphotos.gitignorefile.Cache()
+            os.makedirs(f"{d}/.venv/bin")
+            os.symlink(f"/nonexistent-path-{id(self)}", f"{d}/.venv/bin/python")
+            self.assertFalse(matches(f"{d}/.venv/bin/python"))

tests/test_gitignorefile_ignore.py

@@ -0,0 +1,83 @@
+import os
+import shutil
+import tempfile
+import unittest
+import unittest.mock
+
+import osxphotos.gitignorefile
+
+
+class TestIgnore(unittest.TestCase):
+    def test_robert_shutil_ignore_function(self):
+        with tempfile.TemporaryDirectory() as d:
+            for directory in [
+                "test__pycache__/excluded/excluded",
+                ".test_venv",
+                "not_excluded/test__pycache__",
+                "not_excluded/excluded_not",
+                "not_excluded/excluded",
+                "not_excluded/not_excluded2",
+            ]:
+                os.makedirs(f"{d}/example/{directory}")
+
+            for name in [
+                "test__pycache__/.test_gitignore",
+                "test__pycache__/excluded/excluded/excluded.txt",
+                "test__pycache__/excluded/excluded/test_inverse",
+                "test__pycache__/some_file.txt",
+                "test__pycache__/test",
+                ".test_gitignore",
+                ".test_venv/some_file.txt",
+                "not_excluded.txt",
+                "not_excluded/.test_gitignore",
+                "not_excluded/excluded_not/sub_excluded.txt",
+                "not_excluded/excluded/excluded.txt",
+                "not_excluded/not_excluded2.txt",
+                "not_excluded/not_excluded2/sub_excluded.txt",
+                "not_excluded/excluded_not.txt",
+                ".test_gitignore_empty",
+            ]:
+                with open(f"{d}/example/{name}", "w"):
+                    pass
+
+            with open(f"{d}/example/.gitignore", "w") as f:
+                print("test__pycache__", file=f)
+                print("*.py[cod]", file=f)
+                print(".test_venv/", file=f)
+                print(".test_venv/**", file=f)
+                print(".test_venv/*", file=f)
+                print("!test_inverse", file=f)
+
+            result = []
+            shutil.copytree(
+                f"{d}/example", f"{d}/target", ignore=osxphotos.gitignorefile.ignore()
+            )
+            for root, directories, files in os.walk(f"{d}/target"):
+                for directory in directories:
+                    result.append(os.path.join(root, directory))
+                for name in files:
+                    result.append(os.path.join(root, name))
+
+            result = sorted(
+                (os.path.relpath(x, f"{d}/target").replace(os.sep, "/") for x in result)
+            )
+
+            self.assertEqual(
+                result,
+                [
+                    ".gitignore",
+                    ".test_gitignore",
+                    ".test_gitignore_empty",
+                    "not_excluded",
+                    "not_excluded.txt",
+                    "not_excluded/.test_gitignore",
+                    "not_excluded/excluded",
+                    "not_excluded/excluded/excluded.txt",
+                    "not_excluded/excluded_not",
+                    "not_excluded/excluded_not.txt",
+                    "not_excluded/excluded_not/sub_excluded.txt",
+                    "not_excluded/not_excluded2",
+                    "not_excluded/not_excluded2.txt",
+                    "not_excluded/not_excluded2/sub_excluded.txt",
+                ],
+            )

tests/test_gitignorefile_ignored.py

@@ -0,0 +1,38 @@
+import os
+import unittest
+
+import osxphotos.gitignorefile
+
+
+class TestIgnored(unittest.TestCase):
+    def test_simple(self):
+        for is_dir in (None, False, True):
+            with self.subTest(i=is_dir):
+                self.assertFalse(
+                    osxphotos.gitignorefile.ignored(__file__, is_dir=is_dir)
+                )
+                if is_dir is not True:
+                    self.assertTrue(
+                        osxphotos.gitignorefile.ignored(
+                            f"{os.path.dirname(__file__)}/__pycache__/some.pyc",
+                            is_dir=is_dir,
+                        )
+                    )
+                self.assertFalse(
+                    osxphotos.gitignorefile.ignored(
+                        os.path.dirname(__file__), is_dir=is_dir
+                    )
+                )
+                if is_dir is not False:
+                    self.assertTrue(
+                        osxphotos.gitignorefile.ignored(
+                            f"{os.path.dirname(__file__)}/__pycache__", is_dir=is_dir
+                        )
+                    )
+                else:
+                    # Note: this test will fail if your .gitignore file does not contain __pycache__/
+                    self.assertFalse(
+                        osxphotos.gitignorefile.ignored(
+                            f"{os.path.dirname(__file__)}/__pycache__", is_dir=is_dir
+                        )
+                    )

tests/test_gitignorefile_match_non_str.py

@@ -0,0 +1,54 @@
+""" Test match with non-string arguments. """
+
+import io
+import pathlib
+import unittest
+import unittest.mock
+
+import osxphotos.gitignorefile
+
+
+class TestMatchNonStr(unittest.TestCase):
+    """Test match with non-string arguments."""
+
+    def test_simple_base_path(self):
+        """Test non-str pathlike arguments for base_path"""
+        matches = self.__parse_gitignore_string(
+            ["__pycache__/", "*.py[cod]"], mock_base_path=pathlib.Path("/home/michael")
+        )
+        for is_dir in (False, True):
+            with self.subTest(i=is_dir):
+                self.assertFalse(matches("/home/michael/main.py", is_dir=is_dir))
+                self.assertTrue(matches("/home/michael/main.pyc", is_dir=is_dir))
+                self.assertTrue(matches("/home/michael/dir/main.pyc", is_dir=is_dir))
+        self.assertFalse(matches("/home/michael/__pycache__", is_dir=False))
+        self.assertTrue(matches("/home/michael/__pycache__", is_dir=True))
+
+    def test_simple_matches(self):
+        """Test non-str pathlike arguments for match"""
+        matches = self.__parse_gitignore_string(
+            ["__pycache__/", "*.py[cod]"], mock_base_path=pathlib.Path("/home/michael")
+        )
+        for is_dir in (False, True):
+            with self.subTest(i=is_dir):
+                self.assertFalse(
+                    matches(pathlib.Path("/home/michael/main.py"), is_dir=is_dir)
+                )
+                self.assertTrue(
+                    matches(pathlib.Path("/home/michael/main.pyc"), is_dir=is_dir)
+                )
+                self.assertTrue(
+                    matches(pathlib.Path("/home/michael/dir/main.pyc"), is_dir=is_dir)
+                )
+        self.assertFalse(
+            matches(pathlib.Path("/home/michael/__pycache__"), is_dir=False)
+        )
+        self.assertTrue(matches(pathlib.Path("/home/michael/__pycache__"), is_dir=True))
+
+    def __parse_gitignore_string(self, data, mock_base_path):
+        with unittest.mock.patch(
+            "builtins.open", lambda _: io.StringIO("\n".join(data))
+        ):
+            return osxphotos.gitignorefile.parse(
+                f"{mock_base_path}/.gitignore", base_path=mock_base_path
+            )