Chore/python clean up, and unused variable check to pre commit

.pre-commit-config.yaml

@@ -63,6 +63,13 @@ repos:
     hooks:
       - id: black
 
+  - repo: https://github.com/PyCQA/autoflake
+    rev: v2.3.1
+    hooks:
+      - id: autoflake
+        files: tests/.*\.py$
+        args: ['--in-place', '--remove-all-unused-imports', '--remove-unused-variable']
+
   - repo: https://github.com/python-jsonschema/check-jsonschema
     rev: 0.28.6
     hooks:

perf-tests/suite/ap_request_perf.py

@@ -1,7 +1,7 @@
 import os
 
 import yaml
-from locust import HttpUser, TaskSet, task
+from locust import HttpUser, task
 
 host = ""
 

perf-tests/suite/test_ap_reload_perf.py

@@ -1,5 +1,4 @@
 import json
-import logging
 import os
 import re
 import subprocess
@@ -9,7 +8,7 @@
 import requests
 import yaml
 from kubernetes.client import V1ContainerPort
-from settings import DEPLOYMENTS, TEST_DATA
+from settings import TEST_DATA
 from suite.utils.ap_resources_utils import (
     create_ap_logconf_from_yaml,
     create_ap_policy_from_yaml,
@@ -25,14 +24,9 @@
     delete_items_from_yaml,
     ensure_connection_to_public_endpoint,
     ensure_response_from_backend,
-    get_events,
-    get_file_contents,
-    get_first_pod_name,
-    get_ingress_nginx_template_conf,
     get_resource_metrics,
     replace_ingress_with_ap_annotations,
     wait_before_test,
-    wait_for_event_increment,
     wait_until_all_pods_are_ready,
 )
 from suite.utils.yaml_utils import get_first_ingress_host_from_yaml

perf-tests/suite/test_vs_perf.py

@@ -1,7 +1,4 @@
 import json
-import re
-import subprocess
-from datetime import datetime
 
 import pytest
 import requests

tests/suite/fixtures/ic_fixtures.py

@@ -102,7 +102,7 @@ def crd_ingress_controller(
             ingress_controller_endpoint.port,
             ingress_controller_endpoint.port_ssl,
         )
-    except ApiException as ex:
+    except ApiException:
         # Finalizer method doesn't start if fixture creation was incomplete, ensure clean up here
         print("Restore the ClusterRole:")
         patch_rbac(kube_apis.rbac_v1, f"{DEPLOYMENTS}/rbac/rbac.yaml")
@@ -267,7 +267,6 @@ def crd_ingress_controller_with_dos(
 
         print("------------------------- Create syslog svc -----------------------")
         src_syslog_yaml = f"{TEST_DATA}/dos/dos-syslog.yaml"
-        log_loc = f"/var/log/messages"
         create_items_from_yaml(kube_apis, src_syslog_yaml, namespace)
         before = time.time()
         wait_until_all_pods_are_ready(kube_apis.v1, namespace)
@@ -397,7 +396,7 @@ def crd_ingress_controller_with_ed(
             ingress_controller_prerequisites.namespace,
             cm_source,
         )
-    except ApiException as ex:
+    except ApiException:
         # Finalizer method doesn't start if fixture creation was incomplete, ensure clean up here
         print("Restore the ClusterRole:")
         patch_rbac(kube_apis.rbac_v1, f"{DEPLOYMENTS}/rbac/rbac.yaml")

tests/suite/grpc/helloworld_pb2_grpc.py

@@ -16,10 +16,10 @@ def __init__(self, channel):
             channel: A grpc.Channel.
         """
         self.SayHello = channel.unary_unary(
-                '/helloworld.Greeter/SayHello',
-                request_serializer=helloworld__pb2.HelloRequest.SerializeToString,
-                response_deserializer=helloworld__pb2.HelloReply.FromString,
-                )
+            '/helloworld.Greeter/SayHello',
+            request_serializer=helloworld__pb2.HelloRequest.SerializeToString,
+            response_deserializer=helloworld__pb2.HelloReply.FromString,
+        )
 
 
 class GreeterServicer(object):
@@ -36,35 +36,36 @@ def SayHello(self, request, context):
 
 def add_GreeterServicer_to_server(servicer, server):
     rpc_method_handlers = {
-            'SayHello': grpc.unary_unary_rpc_method_handler(
-                    servicer.SayHello,
-                    request_deserializer=helloworld__pb2.HelloRequest.FromString,
-                    response_serializer=helloworld__pb2.HelloReply.SerializeToString,
-            ),
+        'SayHello': grpc.unary_unary_rpc_method_handler(
+            servicer.SayHello,
+            request_deserializer=helloworld__pb2.HelloRequest.FromString,
+            response_serializer=helloworld__pb2.HelloReply.SerializeToString,
+        ),
     }
     generic_handler = grpc.method_handlers_generic_handler(
-            'helloworld.Greeter', rpc_method_handlers)
+        'helloworld.Greeter', rpc_method_handlers)
     server.add_generic_rpc_handlers((generic_handler,))
 
-
  # This class is part of an EXPERIMENTAL API.
+
+
 class Greeter(object):
     """The greeting service definition.
     """
 
     @staticmethod
     def SayHello(request,
-            target,
-            options=(),
-            channel_credentials=None,
-            call_credentials=None,
-            insecure=False,
-            compression=None,
-            wait_for_ready=None,
-            timeout=None,
-            metadata=None):
+                 target,
+                 options=(),
+                 channel_credentials=None,
+                 call_credentials=None,
+                 insecure=False,
+                 compression=None,
+                 wait_for_ready=None,
+                 timeout=None,
+                 metadata=None):
         return grpc.experimental.unary_unary(request, target, '/helloworld.Greeter/SayHello',
-            helloworld__pb2.HelloRequest.SerializeToString,
-            helloworld__pb2.HelloReply.FromString,
-            options, channel_credentials,
-            insecure, call_credentials, compression, wait_for_ready, timeout, metadata)
+                                             helloworld__pb2.HelloRequest.SerializeToString,
+                                             helloworld__pb2.HelloReply.FromString,
+                                             options, channel_credentials,
+                                             insecure, call_credentials, compression, wait_for_ready, timeout, metadata)

tests/suite/test_annotations.py

@@ -565,8 +565,6 @@ def test_standard_ingress(
         expected_strings,
         unexpected_strings,
     ):
-        initial_events = get_events(kube_apis.v1, annotations_setup.namespace)
-        initial_count = get_event_count(annotations_setup.ingress_event_text, initial_events)
         print("Case 8: standard ingress")
         replace_ingresses_from_yaml(kube_apis.networking_v1, annotations_setup.namespace, yaml_file)
         wait_before_test(1)
@@ -577,7 +575,6 @@ def test_standard_ingress(
             annotations_setup.ingress_pod_name,
             ingress_controller_prerequisites.namespace,
         )
-        new_events = get_events(kube_apis.v1, annotations_setup.namespace)
         for _ in expected_strings:
             assert _ in result_conf
         for _ in unexpected_strings:

tests/suite/test_app_protect_integration.py

@@ -1,7 +1,7 @@
 import pytest
 import requests
 import yaml
-from settings import CRDS, DEPLOYMENTS, TEST_DATA
+from settings import CRDS, TEST_DATA
 from suite.utils.ap_resources_utils import (
     create_ap_logconf_from_yaml,
     create_ap_policy_from_yaml,

tests/suite/test_app_protect_waf_policies.py

@@ -177,7 +177,7 @@ def test_ap_waf_policy_block(
                 "syslog:server=127.0.0.1:514",
             )
         elif waf == waf_pol_default_src:
-            pol_name = create_policy_from_yaml(kube_apis.custom_objects, waf, test_namespace)
+            create_policy_from_yaml(kube_apis.custom_objects, waf, test_namespace)
         else:
             pytest.fail(f"Invalid argument")
 

tests/suite/test_app_protect_waf_policies_grpc.py

@@ -78,7 +78,7 @@ def appprotect_setup(
         elif vs_or_vsr == "vsr":
             (src_pol_name, vsr_ns, vs_host, vs_name, vsr) = ap_vsr_setup(kube_apis, test_namespace, policy_method)
         wait_before_test(120)
-    except Exception as ex:
+    except Exception:
         cleanup(kube_apis, ingress_controller_prerequisites, src_pol_name, test_namespace, vs_or_vsr, vs_name, vsr)
 
     def fin():

tests/suite/test_build_info.py

@@ -50,7 +50,7 @@ def send_build_info(self, kube_apis, ingress_controller_prerequisites) -> str:
         try:
             _info = _log[_log.find("Version") :].strip()
             logging.info(f"Version and GitCommit info: {_info}")
-        except Exception as e:
+        except Exception:
             logging.exception(f"Tag labels not found")
 
         return _info

tests/suite/test_default_server.py

@@ -3,7 +3,7 @@
 import pytest
 import requests
 from requests.exceptions import ConnectionError
-from settings import BASEDIR, DEPLOYMENTS, TEST_DATA
+from settings import TEST_DATA
 from suite.utils.resources_utils import (
     create_secret_from_yaml,
     delete_secret,

tests/suite/test_egress_mtls.py

@@ -1,16 +1,9 @@
 import pytest
-import requests
 from settings import TEST_DATA
 from suite.utils.policy_resources_utils import create_policy_from_yaml, delete_policy
 from suite.utils.resources_utils import create_secret_from_yaml, delete_secret, wait_before_test
 from suite.utils.ssl_utils import create_sni_session
-from suite.utils.vs_vsr_resources_utils import (
-    delete_and_create_vs_from_yaml,
-    patch_v_s_route_from_yaml,
-    patch_virtual_server_from_yaml,
-    read_vs,
-    read_vsr,
-)
+from suite.utils.vs_vsr_resources_utils import delete_and_create_vs_from_yaml, patch_virtual_server_from_yaml, read_vs
 
 std_vs_src = f"{TEST_DATA}/virtual-server/standard/virtual-server.yaml"
 std_vsr_src = f"{TEST_DATA}/virtual-server-route/route-multiple.yaml"

tests/suite/test_jwt_policies_jwksuri.py

@@ -1,4 +1,3 @@
-import time
 from unittest import mock
 
 import pytest
@@ -10,7 +9,6 @@
     create_virtual_server_from_yaml,
     delete_and_create_vs_from_yaml,
     delete_virtual_server,
-    patch_v_s_route_from_yaml,
 )
 
 std_vs_src = f"{TEST_DATA}/virtual-server/standard/virtual-server.yaml"

tests/suite/test_jwt_policies_jwksuri_vsr.py

@@ -1,4 +1,3 @@
-import time
 from unittest import mock
 
 import pytest

tests/suite/test_multiple_ns_perf.py

@@ -51,7 +51,7 @@ def ingress_ns_setup(
         watched_namespace = create_namespace_with_name_from_yaml(kube_apis.v1, f"ns-{i}", f"{TEST_DATA}/common/ns.yaml")
         multi_ns = multi_ns + f"{watched_namespace},"
         create_example_app(kube_apis, "simple", watched_namespace)
-        secret_name = create_secret_from_yaml(kube_apis.v1, watched_namespace, f"{TEST_DATA}/smoke/smoke-secret.yaml")
+        create_secret_from_yaml(kube_apis.v1, watched_namespace, f"{TEST_DATA}/smoke/smoke-secret.yaml")
         with open(manifest) as f:
             doc = yaml.safe_load(f)
             doc["metadata"]["name"] = f"smoke-ingress-{i}"

tests/suite/test_rl_ingress.py

@@ -2,11 +2,8 @@
 
 import pytest
 import requests
-import yaml
-from kubernetes.client import NetworkingV1Api
-from settings import DEPLOYMENTS, TEST_DATA
+from settings import TEST_DATA
 from suite.fixtures.fixtures import PublicEndpoint
-from suite.utils.custom_assertions import assert_event_count_increased
 from suite.utils.resources_utils import (
     are_all_pods_in_ready_state,
     create_example_app,

tests/suite/test_rl_policies.py

@@ -5,13 +5,7 @@
 from settings import TEST_DATA
 from suite.utils.custom_resources_utils import read_custom_resource
 from suite.utils.policy_resources_utils import create_policy_from_yaml, delete_policy
-from suite.utils.resources_utils import (
-    get_first_pod_name,
-    get_pod_list,
-    get_vs_nginx_template_conf,
-    scale_deployment,
-    wait_before_test,
-)
+from suite.utils.resources_utils import get_pod_list, get_vs_nginx_template_conf, scale_deployment, wait_before_test
 from suite.utils.vs_vsr_resources_utils import (
     create_virtual_server_from_yaml,
     delete_virtual_server,

tests/suite/test_transport_server_backup_service.py

@@ -1,9 +1,7 @@
-from pprint import pprint
-
 import pytest
-from settings import DEPLOYMENTS, TEST_DATA
+from settings import TEST_DATA
 from suite.fixtures.fixtures import PublicEndpoint
-from suite.utils.custom_resources_utils import create_ts_from_yaml, delete_ts, patch_ts_from_yaml, read_ts
+from suite.utils.custom_resources_utils import create_ts_from_yaml, delete_ts, patch_ts_from_yaml
 from suite.utils.resources_utils import (
     create_configmap_from_yaml,
     create_items_from_yaml,

tests/suite/test_transport_server_service_insight.py

@@ -1,24 +1,20 @@
-from pprint import pprint
 from unittest import mock
 
 import pytest
 import requests
-from settings import DEPLOYMENTS, TEST_DATA
+from settings import TEST_DATA
 from suite.fixtures.fixtures import PublicEndpoint
-from suite.utils.custom_resources_utils import create_ts_from_yaml, delete_ts, read_ts
+from suite.utils.custom_resources_utils import create_ts_from_yaml, delete_ts
 from suite.utils.resources_utils import (
     create_items_from_yaml,
     create_secret_from_yaml,
     delete_items_from_yaml,
     delete_secret,
     get_first_pod_name,
-    get_nginx_template_conf,
-    replace_configmap_from_yaml,
     wait_before_test,
     wait_until_all_pods_are_ready,
 )
 from suite.utils.ssl_utils import create_sni_session
-from suite.utils.vs_vsr_resources_utils import create_virtual_server_from_yaml, delete_virtual_server, read_vs
 from suite.utils.yaml_utils import get_first_host_from_yaml
 
 

tests/suite/test_upgrade_resources.py

@@ -3,15 +3,9 @@
 
 import pytest
 import yaml
-from settings import DEPLOYMENTS, TEST_DATA
-from suite.utils.custom_resources_utils import create_resource_from_manifest, read_custom_resource
-from suite.utils.resources_utils import (
-    create_ingress,
-    create_items_from_yaml,
-    create_namespace,
-    delete_namespace,
-    wait_before_test,
-)
+from settings import TEST_DATA
+from suite.utils.custom_resources_utils import create_resource_from_manifest
+from suite.utils.resources_utils import create_ingress, create_items_from_yaml, create_namespace, delete_namespace
 from suite.utils.vs_vsr_resources_utils import create_virtual_server
 
 tcp_deployment = f"{TEST_DATA}/upgrade-test-resources/tcp-deployment.yaml"

tests/suite/test_use_cluster_ip.py

@@ -3,17 +3,14 @@
 from suite.utils.resources_utils import (
     create_example_app,
     create_ingress_from_yaml,
-    create_secret_from_yaml,
     delete_common_app,
     delete_items_from_yaml,
-    delete_secret,
     ensure_connection_to_public_endpoint,
     get_reload_count,
-    replace_secret,
     scale_deployment,
     wait_before_test,
 )
-from suite.utils.yaml_utils import get_first_ingress_host_from_yaml, get_name_from_yaml
+from suite.utils.yaml_utils import get_first_ingress_host_from_yaml
 
 from tests.suite.utils.custom_assertions import assert_pods_scaled_to_count