diff --git a/errata/sources/distros/alma.py b/errata/sources/distros/alma.py index 193127e4..e0f2d4ae 100644 --- a/errata/sources/distros/alma.py +++ b/errata/sources/distros/alma.py @@ -17,6 +17,8 @@ import concurrent.futures import json +from django.db import connections + from operatingsystems.utils import get_or_create_osrelease from packages.models import Package from packages.utils import get_or_create_package, parse_package_string @@ -74,6 +76,7 @@ def process_alma_errata_serially(release, advisories): def process_alma_errata_concurrently(release, advisories): """ Process Alma Linux Errata concurrently """ + connections.close_all() elen = len(advisories) pbar_start.send(sender=None, ptext=f'Processing {elen} Alma {release} Errata', plen=elen) i = 0 diff --git a/errata/sources/distros/arch.py b/errata/sources/distros/arch.py index c0330dc8..40d0dada 100644 --- a/errata/sources/distros/arch.py +++ b/errata/sources/distros/arch.py @@ -17,6 +17,8 @@ import concurrent.futures import json +from django.db import connections + from operatingsystems.utils import get_or_create_osrelease from patchman.signals import error_message, pbar_start, pbar_update from packages.models import Package @@ -66,6 +68,7 @@ def parse_arch_errata_concurrently(advisories): """ Parse Arch Linux Errata Advisories concurrently """ osrelease = get_or_create_osrelease(name='Arch Linux') + connections.close_all() elen = len(advisories) pbar_start.send(sender=None, ptext=f'Processing {elen} Arch Advisories', plen=elen) i = 0 diff --git a/errata/sources/distros/debian.py b/errata/sources/distros/debian.py index 710faeeb..29ec5312 100644 --- a/errata/sources/distros/debian.py +++ b/errata/sources/distros/debian.py @@ -21,6 +21,8 @@ from debian.deb822 import Dsc from io import StringIO +from django.db import connections + from operatingsystems.models import OSRelease from operatingsystems.utils import get_or_create_osrelease from packages.models import Package @@ -181,6 +183,7 @@ def create_debian_errata_serially(errata, accepted_codenames): def create_debian_errata_concurrently(errata, accepted_codenames): """ Create Debian Errata concurrently """ + connections.close_all() elen = len(errata) pbar_start.send(sender=None, ptext=f'Processing {elen} Debian Errata', plen=elen) i = 0 diff --git a/errata/sources/distros/rocky.py b/errata/sources/distros/rocky.py index 6231e9e9..693d7b0c 100644 --- a/errata/sources/distros/rocky.py +++ b/errata/sources/distros/rocky.py @@ -18,6 +18,7 @@ import concurrent.futures from tenacity import retry, retry_if_exception_type, stop_after_attempt, wait_exponential +from django.db import connections from django.db.utils import OperationalError from operatingsystems.utils import get_or_create_osrelease @@ -158,6 +159,7 @@ def process_rocky_errata_serially(advisories): def process_rocky_errata_concurrently(advisories): """ Process Rocky Linux errata concurrently """ + connections.close_all() elen = len(advisories) pbar_start.send(sender=None, ptext=f'Processing {elen} Rocky Errata', plen=elen) i = 0 diff --git a/errata/sources/distros/ubuntu.py b/errata/sources/distros/ubuntu.py index 8580e12b..7f50962c 100644 --- a/errata/sources/distros/ubuntu.py +++ b/errata/sources/distros/ubuntu.py @@ -21,6 +21,8 @@ from io import StringIO from urllib.parse import urlparse +from django.db import connections + from operatingsystems.models import OSRelease, OSVariant from operatingsystems.utils import get_or_create_osrelease from packages.models import Package @@ -87,6 +89,7 @@ def parse_usn_data_serially(advisories, accepted_releases): def parse_usn_data_concurrently(advisories, accepted_releases): """ Parse the Ubuntu USN data concurrently """ + connections.close_all() elen = len(advisories) pbar_start.send(sender=None, ptext=f'Processing {elen} Ubuntu Errata', plen=elen) i = 0 diff --git a/errata/sources/repos/yum.py b/errata/sources/repos/yum.py index 242e6d1a..dfeed879 100644 --- a/errata/sources/repos/yum.py +++ b/errata/sources/repos/yum.py @@ -18,6 +18,8 @@ from io import BytesIO from defusedxml import ElementTree +from django.db import connections + from operatingsystems.utils import get_or_create_osrelease from packages.models import Package from packages.utils import get_or_create_package @@ -55,6 +57,7 @@ def extract_updateinfo_serially(updates, elen): def extract_updateinfo_concurrently(updates, elen): """ Parses updateinfo.xml and extracts package/errata information concurrently """ + connections.close_all() pbar_start.send(sender=None, ptext=f'Extracting {elen} updateinfo Errata', plen=elen) i = 0 with concurrent.futures.ProcessPoolExecutor(max_workers=100) as executor: diff --git a/etc/patchman/local_settings.py b/etc/patchman/local_settings.py index 33a7d52f..181c4c4d 100644 --- a/etc/patchman/local_settings.py +++ b/etc/patchman/local_settings.py @@ -35,6 +35,9 @@ # Maximum number of mirrors to add or refresh per repo MAX_MIRRORS = 2 +# Maximum number of failures before disabling a mirror, set to -1 to never disable mirrors +MAX_MIRROR_FAILURES = 14 + # Number of days to wait before raising that a host has not reported DAYS_WITHOUT_REPORT = 14