From ce8b059bca6f078107ed1b52ab741dd61705e5a7 Mon Sep 17 00:00:00 2001 From: Kurt Schwehr Date: Sun, 6 Jul 2025 15:09:31 +0000 Subject: [PATCH 1/2] Update to python 3.9. - Some hand editing - `pyupgrade --py39-plus` --- pyproject.toml | 3 +- setup.cfg | 3 +- shapefile.py | 243 ++++++++++++++++------------------------------ test_shapefile.py | 6 +- 4 files changed, 89 insertions(+), 166 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index df8e737..fed78f7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -39,7 +39,7 @@ line-length = 88 indent-width = 4 # Assume Python 3.9 -target-version = "py37" +target-version = "py39" [tool.ruff.lint] # Enable Pyflakes (`F`) and a subset of the pycodestyle (`E`) codes by default. @@ -67,7 +67,6 @@ skip-magic-trailing-comma = false line-ending = "auto" - [tool.pylint.MASTER] load-plugins=[ "pylint_per_file_ignores", diff --git a/setup.cfg b/setup.cfg index 906abd3..d13d43b 100644 --- a/setup.cfg +++ b/setup.cfg @@ -16,7 +16,6 @@ keywords = gis, geospatial, geographic, shapefile, shapefiles classifiers = Development Status :: 5 - Production/Stable Programming Language :: Python - Programming Language :: Python :: 2.7 Programming Language :: Python :: 3 Topic :: Scientific/Engineering :: GIS Topic :: Software Development :: Libraries @@ -24,7 +23,7 @@ classifiers = [options] py_modules = shapefile -python_requires = >=2.7 +python_requires = >=3.9 [bdist_wheel] universal=1 diff --git a/shapefile.py b/shapefile.py index 211fd48..fdd49cd 100644 --- a/shapefile.py +++ b/shapefile.py @@ -3,21 +3,25 @@ Provides read and write support for ESRI Shapefiles. authors: jlawheadgeospatialpython.com maintainer: karim.bahgat.norwaygmail.com -Compatible with Python versions 2.7-3.x +Compatible with Python versions >=3.9 """ __version__ = "2.4.0" import array +from datetime import date import io import logging import os import sys import tempfile import time -import zipfile -from datetime import date from struct import Struct, calcsize, error, pack, unpack +import zipfile + +from urllib.error import HTTPError +from urllib.parse import urlparse, urlunparse +from urllib.request import Request, urlopen # Create named logger logger = logging.getLogger(__name__) @@ -79,118 +83,48 @@ 5: "RING", } - -# Python 2-3 handling - -PYTHON3 = sys.version_info[0] == 3 - -if PYTHON3: - xrange = range - izip = zip - - from urllib.error import HTTPError - from urllib.parse import urlparse, urlunparse - from urllib.request import Request, urlopen - -else: - from itertools import izip - - from urllib2 import HTTPError, Request, urlopen - from urlparse import urlparse, urlunparse - - # Helpers MISSING = [None, ""] NODATA = -10e38 # as per the ESRI shapefile spec, only used for m-values. -if PYTHON3: - - def b(v, encoding="utf-8", encodingErrors="strict"): - if isinstance(v, str): - # For python 3 encode str to bytes. - return v.encode(encoding, encodingErrors) - elif isinstance(v, bytes): - # Already bytes. - return v - elif v is None: - # Since we're dealing with text, interpret None as "" - return b"" - else: - # Force string representation. - return str(v).encode(encoding, encodingErrors) - - def u(v, encoding="utf-8", encodingErrors="strict"): - if isinstance(v, bytes): - # For python 3 decode bytes to str. - return v.decode(encoding, encodingErrors) - elif isinstance(v, str): - # Already str. - return v - elif v is None: - # Since we're dealing with text, interpret None as "" - return "" - else: - # Force string representation. - return bytes(v).decode(encoding, encodingErrors) - - def is_string(v): - return isinstance(v, str) - -else: - - def b(v, encoding="utf-8", encodingErrors="strict"): - if isinstance(v, unicode): - # For python 2 encode unicode to bytes. - return v.encode(encoding, encodingErrors) - elif isinstance(v, bytes): - # Already bytes. - return v - elif v is None: - # Since we're dealing with text, interpret None as "" - return "" - else: - # Force string representation. - return unicode(v).encode(encoding, encodingErrors) - - def u(v, encoding="utf-8", encodingErrors="strict"): - if isinstance(v, bytes): - # For python 2 decode bytes to unicode. - return v.decode(encoding, encodingErrors) - elif isinstance(v, unicode): - # Already unicode. - return v - elif v is None: - # Since we're dealing with text, interpret None as "" - return "" - else: - # Force string representation. - return bytes(v).decode(encoding, encodingErrors) - - def is_string(v): - return isinstance(v, basestring) - -if sys.version_info[0:2] >= (3, 6): +def b(v, encoding="utf-8", encodingErrors="strict"): + if isinstance(v, str): + # For python 3 encode str to bytes. + return v.encode(encoding, encodingErrors) + elif isinstance(v, bytes): + # Already bytes. + return v + elif v is None: + # Since we're dealing with text, interpret None as "" + return b"" + else: + # Force string representation. + return str(v).encode(encoding, encodingErrors) + +def u(v, encoding="utf-8", encodingErrors="strict"): + if isinstance(v, bytes): + # For python 3 decode bytes to str. + return v.decode(encoding, encodingErrors) + elif isinstance(v, str): + # Already str. + return v + elif v is None: + # Since we're dealing with text, interpret None as "" + return "" + else: + # Force string representation. + return bytes(v).decode(encoding, encodingErrors) - def pathlike_obj(path): - if isinstance(path, os.PathLike): - return os.fsdecode(path) - else: - return path -else: - - def pathlike_obj(path): - if is_string(path): - return path - elif hasattr(path, "__fspath__"): - return path.__fspath__() - else: - try: - return str(path) - except: - return path +def is_string(v): + return isinstance(v, str) +def pathlike_obj(path): + if isinstance(path, os.PathLike): + return os.fsdecode(path) + else: + return path # Begin @@ -311,8 +245,7 @@ def ring_sample(coords, ccw=False): def itercoords(): # iterate full closed ring - for p in coords: - yield p + yield from coords # finally, yield the second coordinate to the end to allow checking the last triplet yield coords[1] @@ -350,7 +283,7 @@ def itercoords(): def ring_contains_ring(coords1, coords2): """Returns True if all vertexes in coords2 are fully inside coords1.""" - return all((ring_contains_point(coords1, p2) for p2 in coords2)) + return all(ring_contains_point(coords1, p2) for p2 in coords2) def organize_polygon_rings(rings, return_errors=None): @@ -398,7 +331,7 @@ def organize_polygon_rings(rings, return_errors=None): return polys # first determine each hole's candidate exteriors based on simple bbox contains test - hole_exteriors = dict([(hole_i, []) for hole_i in xrange(len(holes))]) + hole_exteriors = {hole_i: [] for hole_i in range(len(holes))} exterior_bboxes = [ring_bbox(ring) for ring in exteriors] for hole_i in hole_exteriors.keys(): hole_bbox = ring_bbox(holes[hole_i]) @@ -478,7 +411,7 @@ def organize_polygon_rings(rings, return_errors=None): return polys -class Shape(object): +class Shape: def __init__( self, shapeType=NULL, points=None, parts=None, partTypes=None, oid=None ): @@ -566,7 +499,7 @@ def __geo_interface__(self): else: # get all polygon rings rings = [] - for i in xrange(len(self.parts)): + for i in range(len(self.parts)): # get indexes of start and end points of the ring start = self.parts[i] try: @@ -712,7 +645,7 @@ def shapeTypeName(self): return SHAPETYPE_LOOKUP[self.shapeType] def __repr__(self): - return "Shape #{}: {}".format(self.__oid, self.shapeTypeName) + return f"Shape #{self.__oid}: {self.shapeTypeName}" class _Record(list): @@ -763,10 +696,10 @@ def __getattr__(self, item): index = self.__field_positions[item] return list.__getitem__(self, index) except KeyError: - raise AttributeError("{} is not a field name".format(item)) + raise AttributeError(f"{item} is not a field name") except IndexError: raise IndexError( - "{} found as a field but not enough values available.".format(item) + f"{item} found as a field but not enough values available." ) def __setattr__(self, key, value): @@ -783,7 +716,7 @@ def __setattr__(self, key, value): index = self.__field_positions[key] return list.__setitem__(self, index, value) except KeyError: - raise AttributeError("{} is not a field name".format(key)) + raise AttributeError(f"{key} is not a field name") def __getitem__(self, item): """ @@ -804,7 +737,7 @@ def __getitem__(self, item): if index is not None: return list.__getitem__(self, index) else: - raise IndexError('"{}" is not a field name and not an int'.format(item)) + raise IndexError(f'"{item}" is not a field name and not an int') def __setitem__(self, key, value): """ @@ -822,7 +755,7 @@ def __setitem__(self, key, value): if index is not None: return list.__setitem__(self, index, value) else: - raise IndexError("{} is not a field name and not an int".format(key)) + raise IndexError(f"{key} is not a field name and not an int") @property def oid(self): @@ -834,15 +767,15 @@ def as_dict(self, date_strings=False): Returns this Record as a dictionary using the field names as keys :return: dict """ - dct = dict((f, self[i]) for f, i in self.__field_positions.items()) + dct = {f: self[i] for f, i in self.__field_positions.items()} if date_strings: for k, v in dct.items(): if isinstance(v, date): - dct[k] = "{:04d}{:02d}{:02d}".format(v.year, v.month, v.day) + dct[k] = f"{v.year:04d}{v.month:02d}{v.day:02d}" return dct def __repr__(self): - return "Record #{}: {}".format(self.__oid, list(self)) + return f"Record #{self.__oid}: {list(self)}" def __dir__(self): """ @@ -866,7 +799,7 @@ def __eq__(self, other): return list.__eq__(self, other) -class ShapeRecord(object): +class ShapeRecord: """A ShapeRecord object containing a shape along with its attributes. Provides the GeoJSON __geo_interface__ to return a Feature dictionary.""" @@ -892,7 +825,7 @@ class Shapes(list): to return a GeometryCollection dictionary.""" def __repr__(self): - return "Shapes: {}".format(list(self)) + return f"Shapes: {list(self)}" @property def __geo_interface__(self): @@ -912,7 +845,7 @@ class ShapeRecords(list): to return a FeatureCollection dictionary.""" def __repr__(self): - return "ShapeRecords: {}".format(list(self)) + return f"ShapeRecords: {list(self)}" @property def __geo_interface__(self): @@ -929,7 +862,7 @@ class ShapefileException(Exception): pass -class Reader(object): +class Reader: """Reads the three files of a shapefile as a unit or separately. If one of the three files (.shp, .shx, .dbf) is missing no exception is thrown until you try @@ -1157,7 +1090,7 @@ def __str__(self): ) if self.dbf: info.append( - " {} records ({} fields)".format(len(self), len(self.fields)) + f" {len(self)} records ({len(self.fields)} fields)" ) return "\n".join(info) @@ -1224,8 +1157,7 @@ def __len__(self): def __iter__(self): """Iterates through the shapes/records in the shapefile.""" - for shaperec in self.iterShapeRecords(): - yield shaperec + yield from self.iterShapeRecords() @property def __geo_interface__(self): @@ -1250,7 +1182,7 @@ def load(self, shapefile=None): self.load_dbf(shapeName) if not (self.shp or self.dbf): raise ShapefileException( - "Unable to open %s.dbf or %s.shp." % (shapeName, shapeName) + f"Unable to open {shapeName}.dbf or {shapeName}.shp." ) if self.shp: self.__shpHeader() @@ -1265,13 +1197,13 @@ def load_shp(self, shapefile_name): """ shp_ext = "shp" try: - self.shp = open("%s.%s" % (shapefile_name, shp_ext), "rb") + self.shp = open(f"{shapefile_name}.{shp_ext}", "rb") self._files_to_close.append(self.shp) - except IOError: + except OSError: try: - self.shp = open("%s.%s" % (shapefile_name, shp_ext.upper()), "rb") + self.shp = open(f"{shapefile_name}.{shp_ext.upper()}", "rb") self._files_to_close.append(self.shp) - except IOError: + except OSError: pass def load_shx(self, shapefile_name): @@ -1280,13 +1212,13 @@ def load_shx(self, shapefile_name): """ shx_ext = "shx" try: - self.shx = open("%s.%s" % (shapefile_name, shx_ext), "rb") + self.shx = open(f"{shapefile_name}.{shx_ext}", "rb") self._files_to_close.append(self.shx) - except IOError: + except OSError: try: - self.shx = open("%s.%s" % (shapefile_name, shx_ext.upper()), "rb") + self.shx = open(f"{shapefile_name}.{shx_ext.upper()}", "rb") self._files_to_close.append(self.shx) - except IOError: + except OSError: pass def load_dbf(self, shapefile_name): @@ -1295,13 +1227,13 @@ def load_dbf(self, shapefile_name): """ dbf_ext = "dbf" try: - self.dbf = open("%s.%s" % (shapefile_name, dbf_ext), "rb") + self.dbf = open(f"{shapefile_name}.{dbf_ext}", "rb") self._files_to_close.append(self.dbf) - except IOError: + except OSError: try: - self.dbf = open("%s.%s" % (shapefile_name, dbf_ext.upper()), "rb") + self.dbf = open(f"{shapefile_name}.{dbf_ext.upper()}", "rb") self._files_to_close.append(self.dbf) - except IOError: + except OSError: pass def __del__(self): @@ -1313,7 +1245,7 @@ def close(self): if hasattr(attribute, "close"): try: attribute.close() - except IOError: + except OSError: pass self._files_to_close = [] @@ -1337,7 +1269,7 @@ def __restrictIndex(self, i): rmax = self.numRecords - 1 if abs(i) > rmax: raise IndexError( - "Shape or Record index: %s out of range. Max index: %s" % (i, rmax) + f"Shape or Record index: {i} out of range. Max index: {rmax}" ) if i < 0: i = range(self.numRecords)[i] @@ -1561,7 +1493,7 @@ def iterShapes(self, bbox=None): if self.numShapes: # Iterate exactly the number of shapes from shx header - for i in xrange(self.numShapes): + for i in range(self.numShapes): # MAYBE: check if more left of file or exit early? shape = self.__shape(oid=i, bbox=bbox) if shape: @@ -1624,7 +1556,7 @@ def __dbfHeader(self): # store all field positions for easy lookups # note: fieldLookup gives the index position of a field inside Reader.fields - self.__fieldLookup = dict((f[0], i) for i, f in enumerate(self.fields)) + self.__fieldLookup = {f[0]: i for i, f in enumerate(self.fields)} # by default, read all fields except the deletion flag, hence "[1:]" # note: recLookup gives the index position of a field inside a _Record list @@ -1676,7 +1608,7 @@ def __recordFields(self, fields=None): # make sure the given fieldnames exist for name in fields: if name not in self.__fieldLookup or name == "DeletionFlag": - raise ValueError('"{}" is not a valid field name'.format(name)) + raise ValueError(f'"{name}" is not a valid field name') # fetch relevant field info tuples fieldTuples = [] for fieldinfo in self.fields[1:]: @@ -1684,7 +1616,7 @@ def __recordFields(self, fields=None): if name in fields: fieldTuples.append(fieldinfo) # store the field positions - recLookup = dict((f[0], i) for i, f in enumerate(fieldTuples)) + recLookup = {f[0]: i for i, f in enumerate(fieldTuples)} else: # use all the dbf fields fieldTuples = self.fields[1:] # sans deletion flag @@ -1850,7 +1782,7 @@ def iterRecords(self, fields=None, start=0, stop=None): recSize = self.__recordLength f.seek(self.__dbfHdrLength + (start * recSize)) fieldTuples, recLookup, recStruct = self.__recordFields(fields) - for i in xrange(start, stop): + for i in range(start, stop): r = self.__record( oid=i, fieldTuples=fieldTuples, recLookup=recLookup, recStruct=recStruct ) @@ -1891,7 +1823,7 @@ def iterShapeRecords(self, fields=None, bbox=None): """ if bbox is None: # iterate through all shapes and records - for shape, record in izip( + for shape, record in zip( self.iterShapes(), self.iterRecords(fields=fields) ): yield ShapeRecord(shape=shape, record=record) @@ -1908,7 +1840,7 @@ def iterShapeRecords(self, fields=None, bbox=None): yield ShapeRecord(shape=shape, record=record) -class Writer(object): +class Writer: """Provides write support for ESRI Shapefiles.""" def __init__(self, target=None, shapeType=None, autoBalance=False, **kwargs): @@ -2015,7 +1947,7 @@ def close(self): ): try: attribute.flush() - except IOError: + except OSError: pass # Close any files that the writer opened (but not those given by user) @@ -2023,7 +1955,7 @@ def close(self): if hasattr(attribute, "close"): try: attribute.close() - except IOError: + except OSError: pass self._files_to_close = [] @@ -2494,7 +2426,7 @@ def record(self, *recordList, **recordDict): if self.autoBalance and self.recNum > self.shpNum: self.balance() - fieldCount = sum((1 for field in self.fields if field[0] != "DeletionFlag")) + fieldCount = sum(1 for field in self.fields if field[0] != "DeletionFlag") if recordList: record = list(recordList) while len(record) < fieldCount: @@ -2909,9 +2841,6 @@ def _test(args=sys.argv[1:], verbosity=0): class Py23DocChecker(doctest.OutputChecker): def check_output(self, want, got, optionflags): - if sys.version_info[0] == 2: - got = re.sub("u'(.*?)'", "'\\1'", got) - got = re.sub('u"(.*?)"', '"\\1"', got) res = doctest.OutputChecker.check_output(self, want, got, optionflags) return res diff --git a/test_shapefile.py b/test_shapefile.py index 1b7182f..b55c1f7 100644 --- a/test_shapefile.py +++ b/test_shapefile.py @@ -6,11 +6,7 @@ import json import os.path -try: - from pathlib import Path -except ImportError: - # pathlib2 is a dependency of pytest >= 3.7 - from pathlib2 import Path +from pathlib import Path # third party imports import pytest From 3e3462089712f8d4f49415ac5baee0eb360cf0d3 Mon Sep 17 00:00:00 2001 From: Kurt Schwehr Date: Sun, 6 Jul 2025 15:12:52 +0000 Subject: [PATCH 2/2] =?UTF-8?q?izip=20=E2=86=92=20zip?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- shapefile.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shapefile.py b/shapefile.py index fdd49cd..57f3630 100644 --- a/shapefile.py +++ b/shapefile.py @@ -1338,7 +1338,7 @@ def __shape(self, oid=None, bbox=None): # Read points - produces a list of [x,y] values if nPoints: flat = unpack("<%sd" % (2 * nPoints), f.read(16 * nPoints)) - record.points = list(izip(*(iter(flat),) * 2)) + record.points = list(zip(*(iter(flat),) * 2)) # Read z extremes and values if shapeType in (13, 15, 18, 31): (zmin, zmax) = unpack("<2d", f.read(16))