cli/discover: remove local collections if the remote collection is deleted

This works when the destination backend is 'filesystem' and the source is
CalDAV-calendar-home-set.

pimutils#868

Author: dilyanpalauzov

CHANGELOG.rst

@@ -13,6 +13,10 @@ Version 0.19.0
 ==============
 
 - Add "shell" password fetch strategy to pass command string to a shell.
+- Add ``implicit`` option to storage section.  It creates/deletes implicitly
+  collections in the destinations, when new collections are created/deleted
+  in the source.  The deletion is implemented only for the "filesystem" storage.
+  See :ref:`storage_config`.
 - Add "description" and "order" as metadata.  These fetch the CalDAV:
   calendar-description, CardDAV:addressbook-description and apple-ns:calendar-order
   properties.

docs/config.rst

@@ -365,6 +365,8 @@ Local
       #encoding = "utf-8"
       #post_hook = null
       #fileignoreext = ".tmp"
+      #implicit = "create"
+      #implicit = ["create", "delete"]
 
     Can be used with `khal <http://lostpackets.de/khal/>`_. See :doc:`vdir` for
     a more formal description of the format.
@@ -387,6 +389,12 @@ Local
         new/updated file.
     :param fileeignoreext: The file extention to ignore. It is only useful
         if fileext is set to the empty string. The default is ``.tmp``.
+    :param implicit: When a new collection is created on the source,
+        create it in the destination without asking questions, when
+        the value is "create".  When the value is "delete" and a collection
+        is removed on the source, remove it in the destination.  The value
+        can be a string or an array of strings.  The deletion is implemented
+        only for the "filesystem" storage.
 
 .. storage:: singlefile
 

tests/system/cli/test_config.py

@@ -60,8 +60,9 @@ def test_read_config(read_config):
             "yesno": False,
             "number": 42,
             "instance_name": "bob_a",
+            "implicit": [],
         },
-        "bob_b": {"type": "carddav", "instance_name": "bob_b"},
+        "bob_b": {'type': "carddav", "instance_name": "bob_b", "implicit": []},
     }
 
 

tests/system/utils/test_main.py

@@ -20,7 +20,7 @@ def test_get_storage_init_args():
     from vdirsyncer.storage.memory import MemoryStorage
 
     all, required = utils.get_storage_init_args(MemoryStorage)
-    assert all == {"fileext", "collection", "read_only", "instance_name"}
+    assert all == {"fileext", "collection", "read_only", "instance_name", "implicit"}
     assert not required
 
 

vdirsyncer/cli/config.py

@@ -113,6 +113,13 @@ def _parse_section(self, section_type, name, options):
                 raise ValueError("More than one general section.")
             self._general = options
         elif section_type == "storage":
+            if "implicit" not in options:
+                options["implicit"] = []
+            elif isinstance(options["implicit"], str):
+                options["implicit"] = [options['implicit']]
+            elif not isinstance(options["implicit"], list):
+                raise ValueError(
+                    "`implicit` parameter must be a list, string or absent.")
             self._storages[name] = options
         elif section_type == "pair":
             self._pairs[name] = options

vdirsyncer/cli/discover.py

@@ -7,8 +7,10 @@
 import aiohttp
 import aiostream
 
+from . import cli_logger
 from .. import exceptions
 from .utils import handle_collection_not_found
+from .utils import handle_collection_was_removed
 from .utils import handle_storage_init_error
 from .utils import load_status
 from .utils import save_status
@@ -104,6 +106,29 @@ async def collections_for_pair(
             _handle_collection_not_found=handle_collection_not_found,
         )
     )
+    if "from b" in (pair.collections or []):
+        only_in_a = set((await a_discovered.get_self()).keys()) - set(
+            (await b_discovered.get_self()).keys())
+        if only_in_a and "delete" in pair.config_a["implicit"]:
+            for a in only_in_a:
+                try:
+                    handle_collection_was_removed(pair.config_a, a)
+                    save_status(status_path, pair.name, a, data_type="metadata")
+                    save_status(status_path, pair.name, a, data_type="items")
+                except NotImplementedError as e:
+                    cli_logger.error(e)
+
+    if "from a" in (pair.collections or []):
+        only_in_b = set((await b_discovered.get_self()).keys()) - set(
+            (await a_discovered.get_self()).keys())
+        if only_in_b and "delete" in pair.config_b["implicit"]:
+            for b in only_in_b:
+                try:
+                    handle_collection_was_removed(pair.config_b, b)
+                    save_status(status_path, pair.name, b, data_type="metadata")
+                    save_status(status_path, pair.name, b, data_type="items")
+                except NotImplementedError as e:
+                    cli_logger.error(e)
 
     await _sanity_check_collections(rv, connector=connector)
 

vdirsyncer/cli/utils.py

@@ -227,10 +227,15 @@ def manage_sync_status(base_path, pair_name, collection_name):
 
 def save_status(base_path, pair, collection=None, data_type=None, data=None):
     assert data_type is not None
-    assert data is not None
     status_name = get_status_name(pair, collection)
     path = expand_path(os.path.join(base_path, status_name)) + "." + data_type
     prepare_status_path(path)
+    if data is None:
+        try:
+            os.remove(path)
+        except OSError:  # the file has not existed
+            pass
+        return
 
     with atomic_write(path, mode="w", overwrite=True) as f:
         json.dump(data, f)
@@ -330,6 +335,19 @@ def assert_permissions(path, wanted):
         os.chmod(path, wanted)
 
 
+def handle_collection_was_removed(config, collection):
+    if "delete" in config["implicit"]:
+        storage_type = config["type"]
+        cls, config = storage_class_from_config(config)
+        config["collection"] = collection
+        try:
+            args = cls.delete_collection(**config)
+            args["type"] = storage_type
+            return args
+        except NotImplementedError as e:
+            cli_logger.error(e)
+
+
 async def handle_collection_not_found(config, collection, e=None):
     storage_name = config.get("instance_name", None)
 
@@ -339,7 +357,8 @@ async def handle_collection_not_found(config, collection, e=None):
         )
     )
 
-    if click.confirm("Should vdirsyncer attempt to create it?"):
+    if "create" in config["implicit"] or click.confirm(
+            "Should vdirsyncer attempt to create it?"):
         storage_type = config["type"]
         cls, config = storage_class_from_config(config)
         config["collection"] = collection

vdirsyncer/storage/base.py

@@ -52,6 +52,9 @@ class Storage(metaclass=StorageMeta):
 
     :param read_only: Whether the synchronization algorithm should avoid writes
         to this storage. Some storages accept no value other than ``True``.
+    :param implicit: Whether the synchronization shall create/delete collections
+        in the destination, when these were created/removed from the source.  Must
+        be a possibly empty list of strings.
     """
 
     fileext = ".txt"
@@ -75,9 +78,16 @@ class Storage(metaclass=StorageMeta):
     # The attribute values to show in the representation of the storage.
     _repr_attributes: List[str] = []
 
-    def __init__(self, instance_name=None, read_only=None, collection=None):
+    def __init__(self, instance_name=None, read_only=None, collection=None,
+                 implicit=None):
         if read_only is None:
             read_only = self.read_only
+        if implicit is None:
+            self.implicit = []
+        elif isinstance(implicit, str):
+            self.implicit = [implicit]
+        else:
+            self.implicit = implicit
         if self.read_only and not read_only:
             raise exceptions.UserError("This storage can only be read-only.")
         self.read_only = bool(read_only)
@@ -119,6 +129,18 @@ async def create_collection(cls, collection, **kwargs):
         """
         raise NotImplementedError()
 
+    @classmethod
+    def delete_collection(cls, collection, **kwargs):
+        '''
+        Delete the specified collection and return the new arguments.
+
+        ``collection=None`` means the arguments are already pointing to a
+        possible collection location.
+
+        The returned args should contain the collection name, for UI purposes.
+        '''
+        raise NotImplementedError()
+
     def __repr__(self):
         try:
             if self.instance_name:

vdirsyncer/storage/filesystem.py

@@ -1,6 +1,7 @@
 import errno
 import logging
 import os
+import shutil
 import subprocess
 
 from atomicwrites import atomic_write
@@ -62,9 +63,7 @@ async def discover(cls, path, **kwargs):
     def _validate_collection(cls, path):
         if not os.path.isdir(path) or os.path.islink(path):
             return False
-        if os.path.basename(path).startswith("."):
-            return False
-        return True
+        return not os.path.basename(path).startswith(".")
 
     @classmethod
     async def create_collection(cls, collection, **kwargs):
@@ -80,6 +79,19 @@ async def create_collection(cls, collection, **kwargs):
         kwargs["collection"] = collection
         return kwargs
 
+    @classmethod
+    def delete_collection(cls, collection, **kwargs):
+        kwargs = dict(kwargs)
+        path = kwargs['path']
+
+        if collection is not None:
+            path = os.path.join(path, collection)
+        shutil.rmtree(path, ignore_errors=True)
+
+        kwargs["path"] = path
+        kwargs["collection"] = collection
+        return kwargs
+
     def _get_filepath(self, href):
         return os.path.join(self.path, href)