[SPARK-46808][PYTHON] Refine error classes in Python with automatic sorting function

### What changes were proposed in this pull request?

This PR proposes:
- Add the automated way of writing `error_classes.py` file, `from pyspark.errors.exceptions import _write_self; _write_self()`
- Fix the formatting of the JSON file to be consistent
- Fix typos within the error messages
- Fix parameter names to be consistent (it fixes some, not all)

### Why are the changes needed?

- Now, it is difficult to add a new error class because it enforces alphabetical order or error classes, etc. When you add multiple error classes, you should manually fix and move them around which is troublesome.
- In addition, the current JSON format isn't very consistent.
- For consistency. This PR includes the changes of *some* of parameter naming.

### Does this PR introduce _any_ user-facing change?

Yes, it fixes a couple of typos.

### How was this patch tested?

Unittests were fixed together.

### Was this patch authored or co-authored using generative AI tooling?

No.

Closes #44848 from HyukjinKwon/SPARK-46808.

Authored-by: Hyukjin Kwon <[email protected]>
Signed-off-by: Hyukjin Kwon <[email protected]>

Author: HyukjinKwon

python/pyspark/errors/error_classes.py

@@ -14,3 +14,43 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
+
+
+def _write_self() -> None:
+    import json
+    from pyspark.errors import error_classes
+
+    with open("python/pyspark/errors/error_classes.py", "w") as f:
+        error_class_py_file = """#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# NOTE: Automatically sort this file via
+# - cd $SPARK_HOME
+# - bin/pyspark
+# - from pyspark.errors.exceptions import _write_self; _write_self()
+import json
+
+
+ERROR_CLASSES_JSON = '''
+%s
+'''
+
+ERROR_CLASSES_MAP = json.loads(ERROR_CLASSES_JSON)
+""" % json.dumps(
+            error_classes.ERROR_CLASSES_MAP, sort_keys=True, indent=2
+        )
+        f.write(error_class_py_file)

python/pyspark/errors/exceptions/__init__.py

@@ -1663,7 +1663,7 @@ def sampleBy(
                         "arg_name": "fractions",
                         "arg_type": type(fractions).__name__,
                         "allowed_types": "float, int, str",
-                        "return_type": type(k).__name__,
+                        "item_type": type(k).__name__,
                     },
                 )
             fractions[k] = float(v)

python/pyspark/sql/connect/dataframe.py

@@ -353,7 +353,7 @@ def createDataFrame(
         if isinstance(data, DataFrame):
             raise PySparkTypeError(
                 error_class="INVALID_TYPE",
-                message_parameters={"arg_name": "data", "data_type": "DataFrame"},
+                message_parameters={"arg_name": "data", "arg_type": "DataFrame"},
             )
 
         _schema: Optional[Union[AtomicType, StructType]] = None

python/pyspark/sql/connect/session.py

@@ -1319,6 +1319,7 @@ def hint(
                     error_class="DISALLOWED_TYPE_FOR_CONTAINER",
                     message_parameters={
                         "arg_name": "parameters",
+                        "arg_type": type(parameters).__name__,
                         "allowed_types": allowed_types_repr,
                         "item_type": type(p).__name__,
                     },
@@ -1329,6 +1330,7 @@ def hint(
                         error_class="DISALLOWED_TYPE_FOR_CONTAINER",
                         message_parameters={
                             "arg_name": "parameters",
+                            "arg_type": type(parameters).__name__,
                             "allowed_types": allowed_types_repr,
                             "item_type": type(p).__name__ + "[" + type(p[0]).__name__ + "]",
                         },
@@ -2385,7 +2387,7 @@ def sampleBy(
                         "arg_name": "fractions",
                         "arg_type": type(fractions).__name__,
                         "allowed_types": "float, int, str",
-                        "return_type": type(k).__name__,
+                        "item_type": type(k).__name__,
                     },
                 )
             fractions[k] = float(v)
@@ -5839,7 +5841,7 @@ def approxQuantile(
                         "arg_name": "col",
                         "arg_type": type(col).__name__,
                         "allowed_types": "str",
-                        "return_type": type(c).__name__,
+                        "item_type": type(c).__name__,
                     },
                 )
         col = _to_list(self._sc, cast(List["ColumnOrName"], col))

python/pyspark/sql/dataframe.py

@@ -1421,7 +1421,7 @@ def createDataFrame(  # type: ignore[misc]
         if isinstance(data, DataFrame):
             raise PySparkTypeError(
                 error_class="INVALID_TYPE",
-                message_parameters={"arg_name": "data", "data_type": "DataFrame"},
+                message_parameters={"arg_name": "data", "arg_type": "DataFrame"},
             )
 
         if isinstance(schema, str):

python/pyspark/sql/session.py

@@ -2282,7 +2282,7 @@ def test_stat_sample_by(self):
                 "arg_name": "fractions",
                 "arg_type": "dict",
                 "allowed_types": "float, int, str",
-                "return_type": "NoneType",
+                "item_type": "NoneType",
             },
         )
 

python/pyspark/sql/tests/connect/test_connect_basic.py

@@ -1111,7 +1111,7 @@ def test_observe(self):
         # observation requires name (if given) to be non empty string
         with self.assertRaisesRegex(TypeError, "`name` should be a str, got int"):
             Observation(123)
-        with self.assertRaisesRegex(ValueError, "`name` must be a non empty string, got ''."):
+        with self.assertRaisesRegex(ValueError, "`name` must be a non-empty string, got ''."):
             Observation("")
 
         # dataframe.observe requires at least one expr
@@ -2044,7 +2044,7 @@ def test_invalid_argument_create_dataframe(self):
         self.check_error(
             exception=pe.exception,
             error_class="INVALID_TYPE",
-            message_parameters={"arg_name": "data", "data_type": "DataFrame"},
+            message_parameters={"arg_name": "data", "arg_type": "DataFrame"},
         )
 
 

python/pyspark/sql/tests/test_dataframe.py

@@ -190,7 +190,7 @@ def test_sampleby(self):
                 "arg_name": "fractions",
                 "arg_type": "dict",
                 "allowed_types": "float, int, str",
-                "return_type": "NoneType",
+                "item_type": "NoneType",
             },
         )
 

python/pyspark/sql/tests/test_functions.py

@@ -2236,7 +2236,7 @@ def verify_udf(obj: Any) -> None:
                     error_class="NOT_INSTANCE_OF",
                     message_parameters={
                         "value": str(obj),
-                        "data_type": str(dataType),
+                        "type": str(dataType),
                     },
                 )
             verifier(dataType.toInternal(obj))