Merge branch 'main' into saumya/streaming-fetchone

Author: gargsaumya

.coveragerc

@@ -1,6 +1,8 @@
 [run]
 omit =
-    mssql_python/testing_ddbc_bindings.py
+    main.py
+    setup.py
+    bcp_options.py
     tests/*
 
 [report]

.github/workflows/pr-format-check.yml

@@ -57,9 +57,9 @@ jobs:
               // Extract the summary content
               const summaryContent = summaryMatch[1];
 
-              // Remove all HTML comments including the template placeholder
+              // Remove all HTML comments including unclosed ones (template placeholders)
               const contentWithoutComments = 
-                summaryContent.replace(/<!--[\s\S]*?-->/g, '');
+                summaryContent.replace(/<!--[\s\S]*?(?:-->|$)/g, '');
 
               // Remove whitespace and check if there's actual text content
               const trimmedContent = contentWithoutComments.trim();

.gitignore

@@ -1,6 +1,3 @@
-# Ignore all files in the pybind/build directory
-mssql_python/pybind/build/
-
 # Ignore pycache files and folders
 __pycache__/
 **/__pycache__/
@@ -23,6 +20,7 @@ test-*.xml
 
 # Ignore the build & mssql_python.egg-info directories
 build/
+**/build/
 mssql_python.egg-info/
 
 # Python bytecode
@@ -46,4 +44,19 @@ build/
 *.swp
 
 # .DS_Store files
-.DS_Store
+.DS_Store
+
+# wheel files
+*.whl
+*.tar.gz
+*.zip
+
+# Dockerfiles and images (root only)
+/Dockerfile*
+/docker-compose.yml
+/docker-compose.override.yml
+/docker-compose.*.yml
+
+# Virtual environments
+*venv*/
+**/*venv*/

PyPI_Description.md

@@ -1,20 +1,31 @@
 # mssql-python
 
-This is a new Python driver for Microsoft SQL Server currently in Alpha phase.
+This is a new Python driver for Microsoft SQL Server currently in Public Preview phase.
 
 ## Public Preview Release
 
-We are making progress - The Public Preview of our driver is now available! This marks a significant milestone in our development journey. While we saw a few early adopters of our alpha release, we are introducing the following functionalities to support your applications in a more robust and reliable manner.
+We are making progress - The Public Preview of our driver is now available! This marks a significant milestone in our development journey. While we saw a few early adopters of our public preview release, we are introducing the following functionalities to support your applications in a more robust and reliable manner.
 
 ### What's Included:
 
 - Everything from previous releases
-- **Alpine Linux Support:** Added full support for Alpine Linux distribution (musllinux) with specialized driver handling and fixes for musl libc compatibility.
-- **Connection Management Improvements:** Fixed autocommit to be False by default and added automatic rollback on connection close for better transaction control.
-- **PyODBC Compatibility:** Enhanced type objects and constructor compatibility with pyodbc for seamless migration and interoperability.
+- **SUSE Linux Support:** Added full support for SUSE and openSUSE distributions alongside existing other Linux distros support, broadening enterprise Linux compatibility.
+- **Context Manager Support:** Implemented Python `with` statement support for Connection and Cursor classes with automatic transaction management and resource cleanup.
+- **Large Text Streaming:** Added Data At Execution (DAE) support for streaming large text parameters (`NVARCHAR(MAX)`, `VARCHAR(MAX)`), eliminating memory constraints for bulk text `execute()` operations.
+  - `VARBINARY(MAX)` support to follow alongwith streaming support for fetch operations.
+- **Enhanced Unicode Handling:** Improved emoji and international character support with robust UTF-16 encoding for reliable multilingual data processing.
+- **PyODBC Compatibility:** Enhanced API compatibility with pyodbc including:
+  - DB-API 2.0 exception classes: `Warning`, `Error`, `InterfaceError`, `DatabaseError`, `DataError`, `OperationalError`, `IntegrityError`, `InternalError`, `ProgrammingError`, `NotSupportedError`
+  - Context manager support with `with` statements for Connection and Cursor
+  - Encoding configuration APIs: `setencoding()`, `getencoding()`, `setdecoding()`, `getdecoding()`
+  - Cursor navigation APIs: `next()`, `__iter__()`, `scroll()`, `skip()`, `fetchval()`
+  - Cursor attributes: `rownumber`, `messages`
+  - Additional methods: `cursor.commit()`, `cursor.rollback()`, `table()`
 
 For more information, please visit the project link on Github: https://github.com/microsoft/mssql-python
 
+If you have any feedback, questions or need support please mail us at [email protected].
+
 ### What's Next:
 
 As we continue to develop and refine the driver, you can expect regular updates that will introduce new features, optimizations, and bug fixes. We encourage you to contribute, provide feedback and report any issues you encounter, as this will help us improve the driver for the final release.

eng/pipelines/build-whl-pipeline.yml

@@ -258,6 +258,9 @@ jobs:
     # Install CMake on macOS
     - script: |
         brew update
+        # Uninstall existing CMake to avoid tap conflicts
+        brew uninstall cmake --ignore-dependencies || echo "CMake not installed or already removed"
+        # Install CMake from homebrew/core
         brew install cmake
       displayName: 'Install CMake'
 
@@ -337,7 +340,7 @@ jobs:
         python -m pytest -v
       displayName: 'Run Pytest to validate bindings'
       env:
-        DB_CONNECTION_STRING: 'Driver=ODBC Driver 18 for SQL Server;Server=localhost;Database=master;Uid=SA;Pwd=$(DB_PASSWORD);TrustServerCertificate=yes'
+        DB_CONNECTION_STRING: 'Driver=ODBC Driver 18 for SQL Server;Server=tcp:127.0.0.1,1433;Database=master;Uid=SA;Pwd=$(DB_PASSWORD);TrustServerCertificate=yes'
 
     # Build wheel package for universal2
     - script: |

eng/pipelines/pr-validation-pipeline.yml

@@ -152,7 +152,7 @@ jobs:
       python -m pytest -v --junitxml=test-results.xml --cov=. --cov-report=xml --capture=tee-sys --cache-clear
     displayName: 'Run pytest with coverage'
     env:
-      DB_CONNECTION_STRING: 'Driver=ODBC Driver 18 for SQL Server;Server=localhost;Database=master;Uid=SA;Pwd=$(DB_PASSWORD);TrustServerCertificate=yes'
+      DB_CONNECTION_STRING: 'Driver=ODBC Driver 18 for SQL Server;Server=tcp:127.0.0.1,1433;Database=master;Uid=SA;Pwd=$(DB_PASSWORD);TrustServerCertificate=yes'
       DB_PASSWORD: $(DB_PASSWORD)
 
   - task: PublishTestResults@2

mssql_python-0.8.1-cp312-cp312-musllinux_1_2_aarch64.whl

@@ -380,33 +380,21 @@ def _map_sql_type(self, param, parameters_list, i):
             )
 
         if isinstance(param, bytes):
-            if len(param) > 8000:  # Assuming VARBINARY(MAX) for long byte arrays
-                return (
-                    ddbc_sql_const.SQL_VARBINARY.value,
-                    ddbc_sql_const.SQL_C_BINARY.value,
-                    len(param),
-                    0,
-                    False,
-                )
+            # Use VARBINARY for Python bytes/bytearray since they are variable-length by nature.
+            # This avoids storage waste from BINARY's zero-padding and matches Python's semantics.
             return (
-                ddbc_sql_const.SQL_BINARY.value,
+                ddbc_sql_const.SQL_VARBINARY.value,
                 ddbc_sql_const.SQL_C_BINARY.value,
                 len(param),
                 0,
                 False,
             )
 
         if isinstance(param, bytearray):
-            if len(param) > 8000:  # Assuming VARBINARY(MAX) for long byte arrays
-                return (
-                    ddbc_sql_const.SQL_VARBINARY.value,
-                    ddbc_sql_const.SQL_C_BINARY.value,
-                    len(param),
-                    0,
-                    True,
-                )
+            # Use VARBINARY for Python bytes/bytearray since they are variable-length by nature.
+            # This avoids storage waste from BINARY's zero-padding and matches Python's semantics.
             return (
-                ddbc_sql_const.SQL_BINARY.value,
+                ddbc_sql_const.SQL_VARBINARY.value,
                 ddbc_sql_const.SQL_C_BINARY.value,
                 len(param),
                 0,
@@ -471,17 +459,16 @@ def _reset_cursor(self) -> None:
 
     def close(self) -> None:
         """
-        Close the cursor now (rather than whenever __del__ is called).
+        Close the connection now (rather than whenever .__del__() is called).
+        Idempotent: subsequent calls have no effect and will be no-ops.
 
         The cursor will be unusable from this point forward; an InterfaceError
-        will be raised if any operation is attempted with the cursor.
-        
-        Note:
-            Unlike the current behavior, this method can be called multiple times safely.
-            Subsequent calls to close() on an already closed cursor will have no effect.
+        will be raised if any operation (other than close) is attempted with the cursor.
+        This is a deviation from pyodbc, which raises an exception if the cursor is already closed.
         """
         if self.closed:
-            return 
+            # Do nothing - not calling _check_closed() here since we want this to be idempotent
+            return
 
         # Clear messages per DBAPI
         self.messages = []
@@ -498,12 +485,12 @@ def _check_closed(self):
         Check if the cursor is closed and raise an exception if it is.
 
         Raises:
-            InterfaceError: If the cursor is closed.
+            ProgrammingError: If the cursor is closed.
         """
         if self.closed:
-            raise InterfaceError(
-                driver_error="Operation cannot be performed: the cursor is closed.",
-                ddbc_error="Operation cannot be performed: the cursor is closed."
+            raise ProgrammingError(
+                driver_error="Operation cannot be performed: The cursor is closed.",
+                ddbc_error=""
             )
 
     def _create_parameter_types_list(self, parameter, param_info, parameters_list, i):
@@ -849,6 +836,8 @@ def _select_best_sample_value(column):
             return max(non_nulls, key=lambda s: len(str(s)))
         if all(isinstance(v, datetime.datetime) for v in non_nulls):
             return datetime.datetime.now()
+        if all(isinstance(v, (bytes, bytearray)) for v in non_nulls):
+            return max(non_nulls, key=lambda b: len(b))
         if all(isinstance(v, datetime.date) for v in non_nulls):
             return datetime.date.today()
         return non_nulls[0]  # fallback
@@ -1185,13 +1174,19 @@ def __del__(self):
         Destructor to ensure the cursor is closed when it is no longer needed.
         This is a safety net to ensure resources are cleaned up
         even if close() was not called explicitly.
+        If the cursor is already closed, it will not raise an exception during cleanup.
         """
-        if "_closed" not in self.__dict__ or not self._closed:
+        if "closed" not in self.__dict__ or not self.closed:
             try:
                 self.close()
             except Exception as e:
                 # Don't raise an exception in __del__, just log it
-                log('error', "Error during cursor cleanup in __del__: %s", e)
+                # If interpreter is shutting down, we might not have logging set up
+                import sys
+                if sys and sys._is_finalizing():
+                    # Suppress logging during interpreter shutdown
+                    return
+                log('debug', "Exception during cursor cleanup in __del__: %s", e)
 
     def scroll(self, value: int, mode: str = 'relative') -> None:
         """
@@ -1430,4 +1425,4 @@ def tables(self, table=None, catalog=None, schema=None, tableType=None):
             row = Row(row_data, self.description, column_map)
             result_rows.append(row)
 
-        return result_rows
+        return result_rows

mssql_python/cursor.py

@@ -17,9 +17,14 @@ class Exception(Exception):
     def __init__(self, driver_error, ddbc_error) -> None:
         self.driver_error = driver_error
         self.ddbc_error = truncate_error_message(ddbc_error)
-        self.message = (
-            f"Driver Error: {self.driver_error}; DDBC Error: {self.ddbc_error}"
-        )
+        if self.ddbc_error:
+            # Both driver and DDBC errors are present
+            self.message = (
+                f"Driver Error: {self.driver_error}; DDBC Error: {self.ddbc_error}"
+            )
+        else:
+            # Errors raised by the driver itself should not have a DDBC error message
+            self.message = f"Driver Error: {self.driver_error}"
         super().__init__(self.message)