diff --git a/rest_framework_jwt/utils.py b/rest_framework_jwt/utils.py
index c72197bc..56e6a460 100644
--- a/rest_framework_jwt/utils.py
+++ b/rest_framework_jwt/utils.py
@@ -97,15 +97,21 @@ def jwt_encode_handler(payload):
 
 
 def jwt_decode_handler(token):
+    key = api_settings.JWT_PUBLIC_KEY
+    if not key:
+        if api_settings.JWT_GET_USER_SECRET_KEY:
+            # get user from token, BEFORE verification, to get user secret key
+            unverified_payload = jwt.decode(token, None, False)
+            key = jwt_get_secret_key(unverified_payload)
+        else:
+            key = api_settings.JWT_SECRET_KEY
+
     options = {
         'verify_exp': api_settings.JWT_VERIFY_EXPIRATION,
     }
-    # get user from token, BEFORE verification, to get user secret key
-    unverified_payload = jwt.decode(token, None, False)
-    secret_key = jwt_get_secret_key(unverified_payload)
     return jwt.decode(
         token,
-        api_settings.JWT_PUBLIC_KEY or secret_key,
+        key,
         api_settings.JWT_VERIFY,
         options=options,
         leeway=api_settings.JWT_LEEWAY,