From 589b401f9449fa272d48fedd8f18a81fc63a0dba Mon Sep 17 00:00:00 2001 From: Angelica Date: Mon, 16 Sep 2024 14:43:16 -0400 Subject: [PATCH 01/10] Retired storage v6 --- lib/filesystem.py | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/lib/filesystem.py b/lib/filesystem.py index 21c26aa79..f4127dad2 100644 --- a/lib/filesystem.py +++ b/lib/filesystem.py @@ -1175,7 +1175,7 @@ def init(self): if (v_found == STORAGE_VERSION): ch.VERBOSE("found storage dir v%d: %s" % (STORAGE_VERSION, self.root)) self.lock() - elif (v_found in {None, 6}): # initialize/upgrade + elif (v_found in {None}): # initialize/upgrade ch.INFO("%s storage directory: v%d %s" % (op, STORAGE_VERSION, self.root)) self.root.mkdir() @@ -1188,16 +1188,6 @@ def init(self): self.build_large.mkdir() self.unpack_base.mkdir() self.upload_cache.mkdir() - if (v_found is not None): # upgrade - if (v_found == 6): - # Charliecloud 0.32 had a bug where symlinks to fat manifests - # that were really skinny were erroneously absolute, making the - # storage directory immovable (PR #1657). Remove all symlinks - # in dlcache; they’ll be re-created later. - for entry in self.download_cache.iterdir(): - if (entry.is_symlink()): - ch.DEBUG("deleting bad v6 symlink: %s" % entry) - entry.unlink() self.version_file.file_write("%d\n" % STORAGE_VERSION) else: # can’t upgrade ch.FATAL("incompatible storage directory v%d: %s" From eb646f21e1717df707fe0621f917211c699026ff Mon Sep 17 00:00:00 2001 From: Angelica Date: Mon, 16 Sep 2024 15:25:31 -0400 Subject: [PATCH 02/10] Retire support for v1 as well --- lib/filesystem.py | 18 +----------------- 1 file changed, 1 insertion(+), 17 deletions(-) diff --git a/lib/filesystem.py b/lib/filesystem.py index f4127dad2..1e9bd27ba 100644 --- a/lib/filesystem.py +++ b/lib/filesystem.py @@ -1175,20 +1175,6 @@ def init(self): if (v_found == STORAGE_VERSION): ch.VERBOSE("found storage dir v%d: %s" % (STORAGE_VERSION, self.root)) self.lock() - elif (v_found in {None}): # initialize/upgrade - ch.INFO("%s storage directory: v%d %s" - % (op, STORAGE_VERSION, self.root)) - self.root.mkdir() - self.lock() - # These directories appeared in various storage versions, but since - # the thing to do on upgrade is the same as initialize, we don’t - # track the details. - self.download_cache.mkdir() - self.build_cache.mkdir() - self.build_large.mkdir() - self.unpack_base.mkdir() - self.upload_cache.mkdir() - self.version_file.file_write("%d\n" % STORAGE_VERSION) else: # can’t upgrade ch.FATAL("incompatible storage directory v%d: %s" % (v_found, self.root), @@ -1284,10 +1270,8 @@ def validate_strict(self): % (msg_prefix, img), ch.BUG_REPORT_PLZ) def version_read(self): - # While support for storage v1 was dropped some time ago, let’s at least - # retain the ability to recognize it. if (not os.path.isfile(self.version_file)): - return 1 + return None text = self.version_file.file_read_all() try: return int(text) From ab1804afe6a31e4d9c8a651b843dc9150a634d2e Mon Sep 17 00:00:00 2001 From: Angelica Date: Mon, 16 Sep 2024 15:42:03 -0400 Subject: [PATCH 03/10] Fixing test fails --- lib/filesystem.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/filesystem.py b/lib/filesystem.py index 1e9bd27ba..1d3a3fdfd 100644 --- a/lib/filesystem.py +++ b/lib/filesystem.py @@ -1271,7 +1271,7 @@ def validate_strict(self): def version_read(self): if (not os.path.isfile(self.version_file)): - return None + ch.FATAL('Expected file "%s" not found' % self.version_fil e) text = self.version_file.file_read_all() try: return int(text) From 5f2053926684d969a9f58cc9cdd282bf4d4b59b8 Mon Sep 17 00:00:00 2001 From: Angelica Date: Mon, 16 Sep 2024 15:49:51 -0400 Subject: [PATCH 04/10] Fixing tests --- lib/filesystem.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/filesystem.py b/lib/filesystem.py index 1d3a3fdfd..526456c08 100644 --- a/lib/filesystem.py +++ b/lib/filesystem.py @@ -1271,7 +1271,7 @@ def validate_strict(self): def version_read(self): if (not os.path.isfile(self.version_file)): - ch.FATAL('Expected file "%s" not found' % self.version_fil e) + ch.FATAL('Expected file "%s" not found' % self.version_file) text = self.version_file.file_read_all() try: return int(text) From b75960abb223a2a5b2d970e27ebe1e4293d66f80 Mon Sep 17 00:00:00 2001 From: Angelica Date: Tue, 17 Sep 2024 13:35:58 -0400 Subject: [PATCH 05/10] Fixing NoneType test errors --- lib/filesystem.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/lib/filesystem.py b/lib/filesystem.py index 526456c08..8163afd71 100644 --- a/lib/filesystem.py +++ b/lib/filesystem.py @@ -1175,6 +1175,19 @@ def init(self): if (v_found == STORAGE_VERSION): ch.VERBOSE("found storage dir v%d: %s" % (STORAGE_VERSION, self.root)) self.lock() + elif (v_found is None): # initialize/upgrade + ch.INFO("%s storage directory: v%d %s" + % (op, STORAGE_VERSION, self.root)) + self.root.mkdir() + self.lock() + # These directories appeared in various storage versions, but since + # the thing to do on upgrade is the same as initialize, we don’t + # track the details. + self.download_cache.mkdir() + self.build_cache.mkdir() + self.build_large.mkdir() + self.unpack_base.mkdir() + self.upload_cache.mkdir() else: # can’t upgrade ch.FATAL("incompatible storage directory v%d: %s" % (v_found, self.root), From efee96556df94e53dd4ffa3633a29e145b275fff Mon Sep 17 00:00:00 2001 From: Angelica Date: Tue, 17 Sep 2024 15:15:30 -0400 Subject: [PATCH 06/10] Fixing test fails --- lib/filesystem.py | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/filesystem.py b/lib/filesystem.py index 8163afd71..fc20dc7e1 100644 --- a/lib/filesystem.py +++ b/lib/filesystem.py @@ -1188,6 +1188,7 @@ def init(self): self.build_large.mkdir() self.unpack_base.mkdir() self.upload_cache.mkdir() + self.version_file.file_write("%d\n" % STORAGE_VERSION) else: # can’t upgrade ch.FATAL("incompatible storage directory v%d: %s" % (v_found, self.root), From 22c0ec485083e928a43abd50cc126e880a78cdeb Mon Sep 17 00:00:00 2001 From: Angelica Date: Tue, 24 Sep 2024 12:45:58 -0400 Subject: [PATCH 07/10] Use None with set notation Co-authored-by: Reid Priedhorsky --- lib/filesystem.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/filesystem.py b/lib/filesystem.py index fc20dc7e1..2f3fafed9 100644 --- a/lib/filesystem.py +++ b/lib/filesystem.py @@ -1175,7 +1175,7 @@ def init(self): if (v_found == STORAGE_VERSION): ch.VERBOSE("found storage dir v%d: %s" % (STORAGE_VERSION, self.root)) self.lock() - elif (v_found is None): # initialize/upgrade + elif (v_found in {None,}): # None if initializing, or upgraded version ch.INFO("%s storage directory: v%d %s" % (op, STORAGE_VERSION, self.root)) self.root.mkdir() From 107e7e60d600e3297e6ab255f553ac745031b88b Mon Sep 17 00:00:00 2001 From: Angelica Date: Mon, 30 Sep 2024 14:53:45 -0400 Subject: [PATCH 08/10] Added version file check to valid_p method --- lib/filesystem.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/filesystem.py b/lib/filesystem.py index fc20dc7e1..846987cec 100644 --- a/lib/filesystem.py +++ b/lib/filesystem.py @@ -1129,7 +1129,9 @@ def valid_p(self): less any Charliecloud storage directory we might feasibly come across, even if it can’t be upgraded. See also #1147.""" return (os.path.isdir(self.unpack_base) and - os.path.isdir(self.download_cache)) + os.path.isdir(self.download_cache) and + os.path.isfile(self.version_file)) and + self.version_read(self.version_file) >= STORAGE_VERSION) @property def version_file(self): @@ -1284,8 +1286,6 @@ def validate_strict(self): % (msg_prefix, img), ch.BUG_REPORT_PLZ) def version_read(self): - if (not os.path.isfile(self.version_file)): - ch.FATAL('Expected file "%s" not found' % self.version_file) text = self.version_file.file_read_all() try: return int(text) From 2cf62f4ae36bbd2fe1b4df57bc245039179fb9fe Mon Sep 17 00:00:00 2001 From: Angelica Date: Mon, 30 Sep 2024 16:31:44 -0400 Subject: [PATCH 09/10] Created a STORAGE_MIN_VERSION variable --- lib/filesystem.py | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/lib/filesystem.py b/lib/filesystem.py index 846987cec..3eac2c486 100644 --- a/lib/filesystem.py +++ b/lib/filesystem.py @@ -23,8 +23,9 @@ # # To see the directory formats in released versions: # -# $ git grep -E '^STORAGE_VERSION =' $(git tag | sort -V) -STORAGE_VERSION = 7 +# $ git grep -E '^STORAGE_CURRENT_VERSION =' $(git tag | sort -V) +STORAGE_CURRENT_VERSION = 7 +STORAGE_MIN_VERSION = 7 ## Globals ## @@ -1131,7 +1132,7 @@ def valid_p(self): return (os.path.isdir(self.unpack_base) and os.path.isdir(self.download_cache) and os.path.isfile(self.version_file)) and - self.version_read(self.version_file) >= STORAGE_VERSION) + self.version_read(self.version_file) >= STORAGE_MIN_VERSION) @property def version_file(self): @@ -1174,12 +1175,12 @@ def init(self): hint = None ch.FATAL("storage directory seems invalid: %s" % self.root, hint) v_found = self.version_read() - if (v_found == STORAGE_VERSION): - ch.VERBOSE("found storage dir v%d: %s" % (STORAGE_VERSION, self.root)) + if (v_found == STORAGE_CURRENT_VERSION): + ch.VERBOSE("found storage dir v%d: %s" % (STORAGE_CURRENT_VERSION, self.root)) self.lock() elif (v_found is None): # initialize/upgrade ch.INFO("%s storage directory: v%d %s" - % (op, STORAGE_VERSION, self.root)) + % (op, STORAGE_CURRENT_VERSION, self.root)) self.root.mkdir() self.lock() # These directories appeared in various storage versions, but since @@ -1190,7 +1191,7 @@ def init(self): self.build_large.mkdir() self.unpack_base.mkdir() self.upload_cache.mkdir() - self.version_file.file_write("%d\n" % STORAGE_VERSION) + self.version_file.file_write("%d\n" % STORAGE_CURRENT_VERSION) else: # can’t upgrade ch.FATAL("incompatible storage directory v%d: %s" % (v_found, self.root), @@ -1273,9 +1274,9 @@ def validate_strict(self): % (msg_prefix, " ".join(sorted(entries)))) # check version v_found = self.version_read() - if (v_found != STORAGE_VERSION): + if (v_found != STORAGE_CURRENT_VERSION): ch.FATAL("%s: version mismatch: %d expected, %d found" - % (msg_prefix, STORAGE_VERSION, v_found)) + % (msg_prefix, STORAGE_CURRENT_VERSION, v_found)) # check that no image directories have “:” in filename assert isinstance(self.unpack_base, Path) # remove if test suite passes imgs = self.unpack_base.listdir() From 3234bf7bc4d96db8be07199d852d55d480031775 Mon Sep 17 00:00:00 2001 From: Angelica Date: Tue, 1 Oct 2024 13:08:57 -0400 Subject: [PATCH 10/10] Syntax typo --- lib/filesystem.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/filesystem.py b/lib/filesystem.py index 3eac2c486..0d828c3ee 100644 --- a/lib/filesystem.py +++ b/lib/filesystem.py @@ -1131,7 +1131,7 @@ def valid_p(self): across, even if it can’t be upgraded. See also #1147.""" return (os.path.isdir(self.unpack_base) and os.path.isdir(self.download_cache) and - os.path.isfile(self.version_file)) and + os.path.isfile(self.version_file) and self.version_read(self.version_file) >= STORAGE_MIN_VERSION) @property