Feature/rebase upstream

.github/workflows/codeql-analysis.yml

@@ -2,9 +2,9 @@ name: "Code Scanning - Action"
 
 on:
   push:
-    branches: [master]
+    branches: [main]
   pull_request:
-    branches: [master]
+    branches: [main]
 
 jobs:
   CodeQL-Build:

.github/workflows/create-release-and-upload-assets.yml

@@ -11,7 +11,7 @@ jobs:
       - uses: actions/checkout@v4
         with:
           fetch-depth: 0
-          ref: master
+          ref: main
       - name: git fetch --all
         run: |
           git fetch --all
@@ -40,7 +40,7 @@ jobs:
       - uses: actions/checkout@v4
         with:
           fetch-depth: 0
-          ref: master
+          ref: main
       - name: git fetch --all
         run: |
           git fetch --all
@@ -62,7 +62,7 @@ jobs:
           echo "${{ github.ref }}" | cut -dv -f2 > VERSION.txt
           git add VERSION.txt
           git diff --quiet && git diff --staged --quiet || git commit -m "${COMMIT_MSG}"
-          git push origin master
+          git push origin main
       - name: Update debian changelog
         env:
           EMAIL: [email protected]
@@ -72,7 +72,7 @@ jobs:
             skip-checks: true
         run: |
           gbp dch --new-version=$(cat VERSION.txt)-1 --release --distribution=stable --spawn-editor=never --commit --commit-msg="${COMMIT_MSG}"
-          git push origin master
+          git push origin main
   build-and-upload-deb-assets:
     needs: update-version-and-changelog
     runs-on: ubuntu-latest
@@ -91,7 +91,7 @@ jobs:
       - uses: actions/checkout@v4
         with:
           fetch-depth: 0
-          ref: master
+          ref: main
       - name: git fetch --all
         run: |
           git config --global --add safe.directory /__w/patchman/patchman
@@ -156,7 +156,7 @@ jobs:
       - uses: actions/checkout@v4
         with:
           fetch-depth: 0
-          ref: master
+          ref: main
       - name: git fetch --all
         run: |
           git config --global --add safe.directory /__w/patchman/patchman

.gitignore

@@ -14,3 +14,5 @@ dist
 run
 pyvenv.cfg
 .vscode
+.venv
+*.xml

MANIFEST.in

@@ -16,5 +16,7 @@ recursive-include packages *
 recursive-include repos *
 recursive-include reports *
 recursive-include modules *
+recursive-include errata *
+recursive-include security *
 recursive-include sbin *
 recursive-include etc *

README.md

@@ -106,6 +106,7 @@ python3-requests
 python3-colorama
 python3-magic
 python3-humanize
+python3-yaml
 ```
 
 The server can optionally make use of celery to asynchronously process the

TODO

@@ -1,15 +1,21 @@
-* allow sending updates from Red Hat / SuSE machines
-* web interface support for updating repos, finding updates
 * add checkrestart-style options to see which services need restarting
-* CVE/OVAL apps
+* OVAL/OSCAP apps
 * CA support (tinyca?)
-* native python client, using apt/yum/debian libraries
+* native python/go client, using apt/yum/debian libraries
 * record the history of installed packages on a host
 * also store package descriptions/tags/urls
 * check for unused repos
 * suggest names for repos with the same checksum
 * helper script to change paths (e.g. /usr/lib/python3/dist-packages/patchman)
 * Dockerfile/Dockerimage
 * compressed reports
-* add cronjobs to built packages
-* install celery/rabbit/memcache with packages
+* add cronjobs to build packages
+* dnf5 support
+* proxy support
+* GLSA support
+* only use date for errata issue date?
+* parallelize package extraction
+* use django-tables2
+* autonaming for deb repos
+* associate repos with gentoo hosts
+* populate authenticated repos with package lists from hosts?

VERSION.txt

@@ -1 +1 @@
-3.0.15
+3.0.19

arch/utils.py

@@ -0,0 +1,52 @@
+# Copyright 2025 Marcus Furlong <[email protected]>
+#
+# This file is part of Patchman.
+#
+# Patchman is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, version 3 only.
+#
+# Patchman is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Patchman. If not, see <http://www.gnu.org/licenses/>
+
+from arch.models import PackageArchitecture, MachineArchitecture
+from patchman.signals import info_message
+
+
+def clean_package_architectures():
+    """ Remove package architectures that are no longer in use
+    """
+    parches = PackageArchitecture.objects.filter(package__isnull=True)
+    plen = parches.count()
+    if plen == 0:
+        info_message.send(sender=None, text='No orphaned PackageArchitectures found.')
+    else:
+        info_message.send(sender=None, text=f'Removing {plen} orphaned PackageArchitectures')
+        parches.delete()
+
+
+def clean_machine_architectures():
+    """ Remove machine architectures that are no longer in use
+    """
+    marches = MachineArchitecture.objects.filter(
+        host__isnull=True,
+        repository__isnull=True,
+    )
+    mlen = marches.count()
+    if mlen == 0:
+        info_message.send(sender=None, text='No orphaned MachineArchitectures found.')
+    else:
+        info_message.send(sender=None, text=f'Removing {mlen} orphaned MachineArchitectures')
+        marches.delete()
+
+
+def clean_architectures():
+    """ Remove architectures that are no longer in use
+    """
+    clean_package_architectures()
+    clean_machine_architectures()

client/patchman-client

@@ -297,10 +297,20 @@ get_installed_archlinux_packages() {
     fi
 }
 
+get_installed_gentoo_packages() {
+    if check_command_exists qkeyword ; then
+        gentoo_package_arch=$(qkeyword -A)
+    fi
+    if check_command_exists qlist ; then
+        qlist -Ic -F "'%{PN}' '%{SLOT}' '%{PV}' REL'%{PR}' '${gentoo_package_arch}' 'gentoo' '%{CAT}' '%{REPO}'" | sed -e "s/REL'r/'/g"  >> "${tmpfile_pkg}"
+    fi
+}
+
 get_packages() {
     get_installed_rpm_packages
     get_installed_deb_packages
     get_installed_archlinux_packages
+    get_installed_gentoo_packages
 }
 
 get_modules() {
@@ -346,7 +356,9 @@ get_host_data() {
             os="${PRETTY_NAME}"
         elif [ "${ID}" == "arch" ] ; then
             os="${NAME}"
-        elif [[ "${ID}" =~ "suse" ]] ; then
+        elif [ "${ID}" == "gentoo" ] ; then
+            os="${PRETTY_NAME} ${VERSION_ID}"
+        elif [[ "${ID_LIKE}" =~ "suse" ]] ; then
             os="${PRETTY_NAME}"
         elif [ "${ID}" == "astra" ] ; then
             os="${NAME} $(cat /etc/astra_version)"
@@ -386,6 +398,9 @@ get_host_data() {
             fi
         done
     fi
+    if [ ! -z "${CPE_NAME}" ] ; then
+        os="${os} [${CPE_NAME}]"
+    fi
     if ${verbose} ; then
         echo "Kernel:   ${host_kernel}"
         echo "Arch:     ${host_arch}"
@@ -452,7 +467,7 @@ get_repos() {
         fi
         # replace this with a dedicated awk or simple python script?
         yum_repolist=$(yum repolist enabled --verbose 2>/dev/null | sed -e "s/:\? *([0-9]\+ more)$//g" -e "s/ ([0-9]\+$//g" -e "s/:\? more)$//g" -e "s/'//g" -e "s/%/%%/g")
-        for i in $(echo "${yum_repolist}" | awk '{ if ($1=="Repo-id") {printf "'"'"'"; for (i=3; i<NF; i++) printf $i " "; printf $NF"'"'"' "} if ($1=="Repo-name") {printf "'"'"'"; for (i=3; i<NF; i++) printf $i " "; printf $NF"'" ${host_arch}'"' "} if ($1=="Repo-mirrors" || $1=="Repo-metalink:") {printf "'"'"'"; for (i=3; i<NF; i++) printf $i " "; printf $NF"'"'"' "} if ($1=="Repo-baseurl" || $1=="Repo-baseurl:") { url=1; comma=match($NF,","); if (comma) out=substr($NF,1,comma-1); else out=$NF; printf "'"'"'"out"'"'"' "; } else { if (url==1) { if ($1==":") { comma=match($NF,","); if (comma) out=substr($NF,1,comma-1); else out=$NF; printf "'"'"'"out"'"'"' "; } else {url=0; print "";} } } }' | sed -e "s/\/'/'/g" | sed -e "s/ ' /' /") ; do
+        for i in $(echo "${yum_repolist}" | awk '{ if ($1=="Repo-id") {printf "'"'"'"; for (i=3; i<NF; i++) printf $i " "; printf $NF"'"'"' "} if ($1=="Repo-name") {printf "'"'"'"; for (i=3; i<NF; i++) printf $i " "; printf $NF"'" ${host_arch}'"' "} if ($1=="Repo-mirrors" || $1=="Repo-metalink") {printf "'"'"'"; for (i=3; i<NF; i++) printf $i " "; printf $NF"'"'"' "} if ($1=="Repo-baseurl" || $1=="Repo-baseurl:") { url=1; comma=match($NF,","); if (comma) out=substr($NF,1,comma-1); else out=$NF; printf "'"'"'"out"'"'"' "; } else { if (url==1) { if ($1==":") { comma=match($NF,","); if (comma) out=substr($NF,1,comma-1); else out=$NF; printf "'"'"'"out"'"'"' "; } else {url=0; print "";} } } }' | sed -e "s/\/'/'/g" | sed -e "s/ ' /' /") ; do
             full_id=$(echo ${i} | cut -d \' -f 2)
             id=$(echo ${i} | cut -d \' -f 2 | cut -d \/ -f 1)
             name=$(echo ${i} | cut -d \' -f 4)
@@ -463,7 +478,7 @@ get_repos() {
             if [ "${priority}" == "" ] ; then
                 priority=99
             fi
-            redhat_repo=$(echo ${i} | grep -e "https://.*/XMLRPC.*\|https://cdn.redhat.com/.*")
+            redhat_repo=$(echo ${i} | grep -e "https://.*/XMLRPC.*\|https://cdn[-[a-z]*]*.redhat.com/.*")
             if [ ${?} == 0 ] || ${local_updates} ; then
                 if ${verbose} ; then
                     echo "Finding updates locally for ${id}"
@@ -473,7 +488,7 @@ get_repos() {
             if [ ! -z ${CPE_NAME} ] ; then
                 id="${CPE_NAME}-${id}"
             fi
-            j=$(echo ${i} | sed -e "s#'${full_id}' '${name}'#'${name}' '${id}' '${priority}'#")
+            j=$(echo ${i} | sed -e "s#'${full_id}' '${name}'#'${name}' '${id}' '${priority}'#" | sed -e "s/'\[/'/g" -e "s/\]'/'/g")
             echo "'rpm' ${j}" >> "${tmpfile_rep}"
             unset priority
         done
@@ -484,7 +499,8 @@ get_repos() {
         if ${verbose} ; then
             echo 'Finding apt repos...'
         fi
-        IFS=${FULL_IFS} read -r osname shortversion <<<$(echo "${os}" | awk '{print $1,$2}' | cut -d . -f 1,2)
+        osname=$(echo ${os} | cut -d " " -f 1)
+        shortversion=${VERSION_ID}
         repo_string="'deb\' \'${osname} ${shortversion} ${host_arch} repo at"
         repos=$(apt-cache policy | grep -v Translation | grep -E "^ *[0-9]{1,5}" | grep -E " mirror\+file|http(s)?:" | sed -e "s/^ *//g" -e "s/ *$//g" | cut -d " " -f 1,2,3,4)
         non_mirror_repos=$(echo "${repos}" | grep -Ev "mirror\+file")
@@ -510,11 +526,11 @@ get_repos() {
             echo 'Finding zypper repos...'
         fi
         if [ $(zypper -q --no-refresh lr --details | head -n 1 | grep Keep) ] ; then
-            zypper_lr_cols="2,3,8,10"
+            zypper_lr_cols='{print "${os}" $3 "|" $2 "|" $8 "|" $10}'
         else
-            zypper_lr_cols="2,3,7,9"
+            zypper_lr_cols='{print "${os}" $3 "|" $2 "|" $7 "|" $9}'
         fi
-        for i in $(zypper -q --no-refresh lr -E -u --details | grep -v ^$ | tail -n +3 | cut -d "|" -f ${zypper_lr_cols} | sed -e "s/ *|/ ${host_arch} |/" -e "s/\?[a-zA-Z0-9_-]* *$//" -e "s/^ /'/g" -e "s/ *| */' '/g" -e "s/ *$/'/g") ; do
+        for i in $(zypper -q --no-refresh lr -E -u --details | grep -v ^$ | tail -n +3 | awk -F"|" "${zypper_lr_cols}" | sed -e "s/\${os}/${PRETTY_NAME}/" -e "s/ *|/ ${host_arch} |/" -e "s/\?[a-zA-Z0-9_-]* *$//" -e "s/^/'/g" -e "s/ *| */' '/g" -e "s/ *$/'/g") ; do
             echo \'rpm\' ${i} >> "${tmpfile_rep}"
             id=$(echo ${i} | cut -d \' -f 4)
             suse_repo=$(echo ${i} | grep -e "https://updates.suse.com/.*")
@@ -557,6 +573,56 @@ get_repos() {
         done
     fi
 
+    # Gentoo
+    if [[ "${os}" =~ "Gentoo" ]] ; then
+        if [ ${verbose} == 1 ] ; then
+            echo 'Finding portage repos...'
+        fi
+        declare -A repo_info
+        repos_output=$(portageq repos_config /)
+        repo_name=""
+        priority=""
+        sync_uri=""
+
+        while IFS= read -r line; do
+            # if the line starts with a section header (e.g., [gentoo], [guru]), it's the repo name
+            if [[ "${line}" =~ ^\[(.*)\] ]]; then
+                # if we already have a repo_name, save the previous entry
+                if [[ -n "${repo_name}" && -n "${sync_uri}" ]]; then
+                    repo_info["${repo_name}"]="${priority},${sync_uri}"
+                fi
+                # else start new repo parsing, resetting vars
+                repo_name="${BASH_REMATCH[1]}"
+                priority=""
+                sync_uri=""
+            fi
+
+            # if the line contains "priority", extract the value, 0 if it doesnt exist
+            if [[ "${line}" =~ "priority" ]]; then
+                priority=$(echo "${line}" | cut -d'=' -f2 | xargs)
+            fi
+
+            # if the line contains "sync-uri", extract the value
+            if [[ "${line}" =~ "sync-uri" ]]; then
+                sync_uri=$(echo "${line}" | cut -d'=' -f2 | xargs)
+            fi
+        done <<< "${repos_output}"
+
+        # save the last repository entry if it's available
+        if [[ -n "${repo_name}" && -n "${sync_uri}" ]]; then
+            repo_info["${repo_name}"]="${priority},${sync_uri}"
+        fi
+
+        for repo in "${!repo_info[@]}"; do
+            priority=$(echo ${repo_info[$repo]} | cut -d',' -f1)
+            sync_uri=$(echo ${repo_info[$repo]} | cut -d',' -f2)
+            if [ "${priority}" == "" ] ; then
+                priority=0
+            fi
+            echo "'gentoo' 'Gentoo Linux ${repo} Repo ${host_arch}' '${repo}' '${priority}' '${sync_uri}'" >> "${tmpfile_rep}"
+        done
+    fi
+
     IFS=${FULL_IFS}
 
     sed -i -e '/^$/d' "${tmpfile_rep}"
@@ -629,10 +695,11 @@ post_data() {
 }
 
 if ! check_command_exists which || \
+   ! check_command_exists awk || \
    ! check_command_exists mktemp || \
    ! check_command_exists curl || \
    ! check_command_exists flock ; then
-    echo "which, mktemp, flock or curl was not found, exiting."
+    echo "which, awk, mktemp, flock or curl was not found, exiting."
     exit 1
 fi
 

debian/changelog

@@ -1,3 +1,35 @@
+patchman (3.0.19-1) stable; urgency=medium
+
+  * switch default branch from master to main
+  * auto-commit to update version skip-checks: true
+
+ -- Marcus Furlong <[email protected]>  Sat, 01 Mar 2025 20:45:02 +0000
+
+patchman (3.0.18-1) stable; urgency=medium
+
+  * recognize https mirrors in mirrorlists
+  * auto-commit to update version skip-checks: true
+
+ -- Marcus Furlong <[email protected]>  Sat, 01 Mar 2025 20:36:58 +0000
+
+patchman (3.0.17-1) stable; urgency=medium
+
+  [ Hugo Deprez ]
+  * add python3-yaml dependency
+  * update readme for depenency
+
+  [ Marcus Furlong ]
+  * auto-commit to update version skip-checks: true
+
+ -- Marcus Furlong <[email protected]>  Thu, 27 Feb 2025 16:07:15 +0000
+
+patchman (3.0.16-1) stable; urgency=medium
+
+  * change compression format to support older rpm versions
+  * auto-commit to update version skip-checks: true
+
+ -- Marcus Furlong <[email protected]>  Wed, 26 Feb 2025 05:25:29 +0000
+
 patchman (3.0.15-1) stable; urgency=medium
 
   [ Vladimir Lettiev ]