CP-46944: Update yum plugins to dnf plugins

.github/workflows/main.yml

@@ -79,6 +79,7 @@ jobs:
           --cov-report xml:.git/coverage${{matrix.python-version}}.xml
         env:
           PYTHONDEVMODE: yes
+          PYTHONPATH: "python3:python3/tests/stubs"
 
       - name: Upload Python ${{matrix.python-version}} coverage report to Codecov
         uses: codecov/codecov-action@v3

python3/Makefile

@@ -3,7 +3,11 @@ include ../config.mk
 SITE3_DIR=$(shell python3 -c "from distutils.sysconfig import get_python_lib; print(get_python_lib())")
 
 IDATA=install -m 644
+DNF_PLUGIN_DIR=dnf-plugins
 
 install:
 	mkdir -p $(DESTDIR)$(SITE3_DIR)
+	mkdir -p $(DESTDIR)$(SITE3_DIR)/$(DNF_PLUGIN_DIR)
 	$(IDATA) packages/observer.py $(DESTDIR)$(SITE3_DIR)/
+	$(IDATA) dnf_plugins/accesstoken.py $(DESTDIR)$(SITE3_DIR)/$(DNF_PLUGIN_DIR)/
+	$(IDATA) dnf_plugins/ptoken.py $(DESTDIR)$(SITE3_DIR)/$(DNF_PLUGIN_DIR)/

python3/README.md

@@ -10,3 +10,4 @@ run by xapi and other daemons.
 - packages: This contains files to be installed in python's site-packages and are meant
 to be modules and packages to be imported by other scripts or executed via python3 -m
 - plugins: This contains files that are meant to be xapi plugins
+- dnf-plugins: This contains dnf-plugins and are meant to be called automatically by dnf

python3/dnf_plugins/accesstoken.py

@@ -0,0 +1,45 @@
+"""dnf plugin to set accesstoken http header for enabled repos"""
+import json
+import logging
+# Disable the error, it can be import in production env
+# and mocked out in unitttest
+#pylint: disable=import-error
+import dnf
+import urlgrabber
+
+
+class InvalidToken(Exception):
+    """Token is invlaid"""
+    def __init__(self, token):
+        super().__init__(f"Invalid token: {token}")
+
+
+#pylint: disable=too-few-public-methods
+class AccessToken(dnf.Plugin):
+    """dnf accesstoken plugin class"""
+
+    name = "accesstoken"
+
+    def config(self):
+        """ DNF plugin config hook,
+        refer to https://dnf.readthedocs.io/en/latest/api_plugins.html"""
+
+        for repo_name in self.base.repos:
+            repo = self.base.repos[repo_name]
+
+            token_url = repo.accesstoken
+            if not token_url or token_url == '':
+                continue
+            try:
+                token_str = urlgrabber.urlopen(token_url).read().strip()
+                token = json.loads(token_str)
+            except Exception:  #pylint: disable=broad-except
+                logging.debug("Failed to load token from: %s", token_url)
+                continue
+
+            if not (token.get('token') and token.get('token_id')):
+                raise InvalidToken(token)
+
+            access_token = f'X-Access-Token:{str(token["token"])}'
+            referer = f'Referer:{str(token["token_id"])}'
+            repo.set_http_headers([access_token, referer])

python3/dnf_plugins/ptoken.py

@@ -0,0 +1,29 @@
+"""dnf plugin to add ptoken for repos"""
+import logging
+import dnf
+
+PTOKEN_PATH = "/etc/xensource/ptoken"
+
+#pylint: disable=too-few-public-methods
+class Ptoken(dnf.Plugin):
+    """ptoken plugin class"""
+
+    name = "ptoken"
+
+    def config(self):
+        """ DNF plugin config hook,
+        refer to https://dnf.readthedocs.io/en/latest/api_plugins.html"""
+        try:
+            with open('/etc/xensource/ptoken', encoding="utf-8") as file:
+                ptoken = file.read().strip()
+        except Exception: #pylint: disable=broad-exception-caught
+            logging.error("Failed to open %s", PTOKEN_PATH)
+            raise
+
+        for repo_name in self.base.repos:
+            repo = self.base.repos[repo_name]
+
+            if len(repo.baseurl) > 0 and repo.baseurl[0].startswith("http://127.0.0.1") \
+                and repo.ptoken:
+                secret = "pool_secret=" + ptoken
+                repo.set_http_headers([f'cookie:{secret}'])

python3/tests/stubs/dnf.py

@@ -0,0 +1,12 @@
+"""This is a stub module for dnf.base
+
+This module is introduced to decouple the dependencies from dnf
+modules during unittest, as the github CI ubuntu container has
+issues with the python-dnf and python3.11
+"""
+#pylint: disable=too-few-public-methods
+class Plugin:
+    """Dnf plugin interface"""
+    def __init__(self, base, cli):
+        self.base = base
+        self.cli = cli

python3/tests/test_dnf_plugins.py

@@ -0,0 +1,103 @@
+"""Test module for dnf accesstoken"""
+import unittest
+import sys
+import json
+from unittest.mock import MagicMock, patch
+
+sys.modules["urlgrabber"] = MagicMock()
+
+# Disable wrong import postition as need to mock some sys modules first
+#pylint: disable=wrong-import-position
+
+# Disable unused-argument as some mock obj is not used
+#pylint: disable=unused-argument
+
+# Some test case does not use self
+
+from dnf_plugins import accesstoken
+from dnf_plugins import ptoken
+
+REPO_NAME = "testrepo"
+
+
+def _mock_repo(a_token=None, p_token=None, baseurl=None):
+    mock_repo = MagicMock()
+    mock_repo.accesstoken = a_token
+    mock_repo.ptoken = p_token
+    mock_repo.baseurl = baseurl
+    mock_base = MagicMock()
+    mock_base.repos = {REPO_NAME: mock_repo}
+    mock_repo.base = mock_base
+    return mock_repo
+
+
+@patch("dnf_plugins.accesstoken.urlgrabber")
+class TestAccesstoken(unittest.TestCase):
+    """Test class for dnf access plugin"""
+
+    def test_set_http_header_with_access_token(self, mock_grabber):
+        """test config succeed with accesstokan"""
+        mock_repo = _mock_repo(a_token="file:///mock_accesstoken_url")
+        mock_grabber.urlopen.return_value.read.return_value = json.dumps({
+            "token": "valid_token",
+            "token_id": "valid_token_id",
+        })
+        accesstoken.AccessToken(mock_repo.base, MagicMock()).config()
+        mock_repo.set_http_headers.assert_called_with(
+            ['X-Access-Token:valid_token','Referer:valid_token_id']
+        )
+
+    def test_repo_without_access_token(self, mock_grabber):
+        """If repo has not accestoken, it should not be blocked"""
+        mock_repo = _mock_repo()
+        accesstoken.AccessToken(mock_repo.base, MagicMock()).config()
+        mock_repo.set_http_headers.assert_not_called()
+
+    def test_ignore_invalid_token_url(self, mock_grabber):
+        """If repo provided an invalid token url, it should be ignored"""
+        mock_repo = _mock_repo(a_token="Not_existed")
+        mock_grabber.urlopen.side_effect = FileNotFoundError('')
+        accesstoken.AccessToken(mock_repo.base, MagicMock()).config()
+        assert not mock_repo.set_http_headers.called
+
+    def test_invalid_token_raise_exception(self, mock_grabber):
+        """Token with right json format, bad content should raise"""
+        mock_repo = _mock_repo(a_token="file:///file_contain_invalid_token")
+        mock_grabber.urlopen.return_value.read.return_value = json.dumps({
+            "bad_token": "I am bad guy"
+        })
+        with self.assertRaises(accesstoken.InvalidToken):
+            accesstoken.AccessToken(mock_repo.base, MagicMock()).config()
+
+
+class TestPtoken(unittest.TestCase):
+    """Test class for ptoken dnf plugin"""
+    def test_failed_to_open_ptoken_file(self):
+        """Exception should raised if the system does not have PTOKEN_PATH"""
+        ptoken.PTOKEN_PATH = "/some/not/exist/path"
+        with self.assertRaises(Exception):
+            ptoken.Ptoken(MagicMock(), MagicMock()).config()
+
+    @patch("builtins.open")
+    def test_set_ptoken_to_http_header(self, mock_open):
+        """Local repo with ptoken enabled should set the ptoken to its http header"""
+        mock_open.return_value.__enter__.return_value.read.return_value = "valid_ptoken"
+        mock_repo = _mock_repo(p_token=True, baseurl=["http://127.0.0.1/some_local_path"])
+        ptoken.Ptoken(mock_repo.base, MagicMock()).config()
+        mock_repo.set_http_headers.assert_called_with(["cookie:pool_secret=valid_ptoken"])
+
+    @patch("builtins.open")
+    def test_remote_repo_ignore_ptoken(self, mock_open):
+        """non-local repo should just ignore the ptoken"""
+        mock_open.return_value.__enter__.return_value.read.return_value = "valid_ptoken"
+        mock_repo = _mock_repo(p_token=True, baseurl=["http://some_remote_token/some_local_path"])
+        ptoken.Ptoken(mock_repo.base, MagicMock()).config()
+        assert not  mock_repo.set_http_headers.called
+
+    @patch("builtins.open")
+    def test_local_repo_does_not_enable_ptoken_should_ignore_ptoken(self, mock_open):
+        """local repo which has not enabled ptoken should just ignore the ptoken"""
+        mock_open.return_value.__enter__.return_value.read.return_value = "valid_ptoken"
+        mock_repo = _mock_repo(p_token=False, baseurl=["http://127.0.0.1/some_local_path"])
+        ptoken.Ptoken(mock_repo.base, MagicMock()).config()
+        assert not  mock_repo.set_http_headers.called