[CA-183960][CA-196882]: GC in batch mode and immediate deletes in SRs

FileSR performs VDI.delete inline
Inline delete VDIs for LVHD SRs
GC waits five minutes after start before initiating work loop.
Use two locks in gcLoop, one to prevent multiple GC processs starting
and one to interlink with abort requests while work is actually occuring.
Inline delete orphaned nodes in coalesce
After handling garbage in the GC loop proceed to coalesce without returning
to scan and potentially finding more garbage whcih then prevents coalesce.

Fix bug in _kickGC where the running GC could be stopped but not start
a new one.

Signed-off-by: Mark Syms <[email protected]>
Signed-off-by: Stefano Panella <[email protected]>
Reviewed-by: Germano Percossi <[email protected]>

GitHub: closes #295

Author: MarkSymsCtx

drivers/FileSR.py

@@ -567,14 +567,7 @@ def delete(self, sr_uuid, vdi_uuid):
         if self.attached:
             raise xs_errors.XenError('VDIInUse')
 
-        if self.vdi_type == vhdutil.VDI_TYPE_VHD:
-            try:
-                util.ioretry(lambda: self._mark_hidden(self.path))
-            except util.CommandException, inst:
-                raise xs_errors.XenError('VDIDelete',
-                        opterr='error %d' % inst.code)
-        else:
-            os.unlink(self.path)
+        os.unlink(self.path)
 
         self.sr.deleted_vdi(vdi_uuid)
         self._db_forget()

drivers/LVHDSR.py

@@ -1365,8 +1365,6 @@ def _kickGC(self):
                     if e.code != errno.ETIMEDOUT:
                         raise
                     util.SMlog('failed to abort the GC')
-                finally:
-                    return
             else:
                 util.SMlog("A GC instance already running, not kicking")
                 return
@@ -1806,6 +1804,13 @@ def delete(self, sr_uuid, vdi_uuid):
         if self.sr.lvActivator.get(self.uuid, False):
             self.sr.lvActivator.deactivate(self.uuid, False)
 
+        try:
+            self.sr.lvmCache.remove(self.lvname)
+        except SR.SRException, e:
+            util.SMlog(
+                "Failed to remove the volume (maybe is leaf coalescing) "
+                "for %s err:%d" % (self.uuid, e.errno))
+
         self.sr._updateStats(self.sr.uuid, -self.size)
         self.sr._kickGC()
 

drivers/cleanup.py

@@ -62,6 +62,11 @@
 LOCK_TYPE_RUNNING = "running" 
 lockRunning = None
 
+# process "lock" to indicate that the GC process has been activated but may not
+# yet be running, stops a second process from being started.
+LOCK_TYPE_GC_ACTIVE = "gc_active"
+lockActive = None
+
 # Default coalesce error rate limit, in messages per minute. A zero value
 # disables throttling, and a negative value disables error reporting.
 DEFAULT_COALESCE_ERR_RATE = 1.0/60
@@ -1456,6 +1461,7 @@ def findCoalesceable(self):
         for vdi in self.vdis.values():
             if vdi.isCoalesceable() and vdi not in self._failedCoalesceTargets:
                 candidates.append(vdi)
+                Util.log("%s is coalescable" % vdi.uuid)
 
         # pick one in the tallest tree
         treeHeight = dict()
@@ -1748,6 +1754,7 @@ def _coalesce(self, vdi):
 
         vdi.parent._reloadChildren(vdi)
         self.journaler.remove(vdi.JRN_RELINK, vdi.uuid)
+        self.deleteVDI(vdi)
 
     def _coalesceLeaf(self, vdi):
         """Leaf-coalesce VDI vdi. Return true if we succeed, false if we cannot
@@ -1786,7 +1793,6 @@ def _snapshotCoalesce(self, vdi):
         Util.log("Coalescing parent %s" % tempSnap)
         util.fistpoint.activate("LVHDRT_coaleaf_delay_2", self.uuid)
         self._coalesce(tempSnap)
-        self.deleteVDI(tempSnap)
         if not vdi.isLeafCoalesceable():
             Util.log("The VDI tree appears to have been altered since")
             return False
@@ -2532,48 +2538,59 @@ def normalizeType(type):
     return type
 
 def _gcLoop(sr, dryRun):
-    failedCandidates = []
-    while True:
-        if not sr.xapi.isPluggedHere():
-            Util.log("SR no longer attached, exiting")
-            break
-        sr.scanLocked()
-        if not sr.hasWork():
-            Util.log("No work, exiting")
-            break
-
-        if not lockRunning.acquireNoblock():
-            Util.log("Another instance already running, exiting")
-            break
-        try:
-            if not sr.gcEnabled():
+    if not lockActive.acquireNoblock():
+        Util.log("Another GC instance already active, exiting")
+        return
+    try:
+        # TODO: make the delay configurable
+        Util.log("GC active, about to go quiet")
+        time.sleep(5 * 60)
+        Util.log("GC active, quiet period ended")
+
+        while True:
+            if not sr.xapi.isPluggedHere():
+                Util.log("SR no longer attached, exiting")
                 break
-            sr.cleanupCoalesceJournals()
             sr.scanLocked()
-            sr.updateBlockInfo()
-
-            if len(sr.findGarbage()) > 0:
-                sr.garbageCollect(dryRun)
-                sr.xapi.srUpdate()
-                continue
+            if not sr.hasWork():
+                Util.log("No work, exiting")
+                break
 
-            candidate = sr.findCoalesceable()
-            if candidate:
-                util.fistpoint.activate("LVHDRT_finding_a_suitable_pair",sr.uuid)
-                sr.coalesce(candidate, dryRun)
-                sr.xapi.srUpdate()
-                continue
+            if not lockRunning.acquireNoblock():
+                Util.log("Unable to acquire GC running lock.")
+                return
+            try:
+                if not sr.gcEnabled():
+                    break
+                sr.cleanupCoalesceJournals()
+                sr.scanLocked()
+                sr.updateBlockInfo()
+
+                howmany = len(sr.findGarbage())
+                if howmany > 0:
+                    Util.log("Found %d orphaned vdis" % howmany)
+                    sr.garbageCollect(dryRun)
+                    sr.xapi.srUpdate()
+
+                candidate = sr.findCoalesceable()
+                if candidate:
+                    util.fistpoint.activate(
+                        "LVHDRT_finding_a_suitable_pair", sr.uuid)
+                    sr.coalesce(candidate, dryRun)
+                    sr.xapi.srUpdate()
+                    continue
 
-            candidate = sr.findLeafCoalesceable()
-            if candidate:
-                sr.coalesceLeaf(candidate, dryRun)
-                sr.xapi.srUpdate()
-                continue
+                candidate = sr.findLeafCoalesceable()
+                if candidate:
+                    sr.coalesceLeaf(candidate, dryRun)
+                    sr.xapi.srUpdate()
+                    continue
 
-            Util.log("No work left")
-            sr.cleanup()
-        finally:
-            lockRunning.release()
+            finally:
+                lockRunning.release()
+    finally:
+        Util.log("GC process exiting, no work left")
+        lockActive.release()
 
 def _gc(session, srUuid, dryRun):
     init(srUuid)
@@ -2621,7 +2638,10 @@ def _abort(srUuid, soft=False):
 def init(srUuid):
     global lockRunning
     if not lockRunning:
-        lockRunning = lock.Lock(LOCK_TYPE_RUNNING, srUuid) 
+        lockRunning = lock.Lock(LOCK_TYPE_RUNNING, srUuid)
+    global lockActive
+    if not lockActive:
+        lockActive = lock.Lock(LOCK_TYPE_GC_ACTIVE, srUuid)
 
 def usage():
     output = """Garbage collect and/or coalesce VHDs in a VHD-based SR
@@ -2741,8 +2761,8 @@ def get_state(srUuid):
     locking.
     """
     init(srUuid)
-    if lockRunning.acquireNoblock():
-        lockRunning.release()
+    if lockActive.acquireNoblock():
+        lockActive.release()
         return False
     return True
 
@@ -2752,7 +2772,7 @@ def should_preempt(session, srUuid):
     if len(entries) == 0:
         return False
     elif len(entries) > 1:
-        raise util.SMException("More than one coalesce entry: " + entries)
+        raise util.SMException("More than one coalesce entry: " + str(entries))
     sr.scan()
     coalescedUuid = entries.popitem()[0]
     garbage = sr.findGarbage()