[3.8] bpo-40014: Fix os.getgrouplist() (GH-19126)

Misc/NEWS.d/next/Library/2020-03-23-17-52-00.bpo-40014.Ya70VG.rst

@@ -1,3 +1,4 @@
-Fix ``os.getgrouplist()``: on macOS, the ``getgrouplist()`` function returns a
-non-zero value without setting ``errno`` if the group list is too small. Double
-the list size and call it again in this case.
+Fix ``os.getgrouplist()``: if ``getgrouplist()`` function fails because the
+group list is too small, retry with a larger group list. On failure, the glibc
+implementation of ``getgrouplist()`` sets ``ngroups`` to the total number of
+groups. For other implementations, double the group list size.

Modules/posixmodule.c

@@ -6906,37 +6906,40 @@ posix_getgrouplist(PyObject *self, PyObject *args)
         return NULL;
 #endif
 
+    while (1) {
 #ifdef __APPLE__
-    groups = PyMem_New(int, ngroups);
+        groups = PyMem_New(int, ngroups);
 #else
-    groups = PyMem_New(gid_t, ngroups);
+        groups = PyMem_New(gid_t, ngroups);
 #endif
-    if (groups == NULL)
-        return PyErr_NoMemory();
+        if (groups == NULL) {
+            return PyErr_NoMemory();
+        }
 
-#ifdef __APPLE__
-    while (getgrouplist(user, basegid, groups, &ngroups)) {
-        /* On macOS, getgrouplist() returns a non-zero value without setting
-           errno if the group list is too small. Double the list size and call
-           it again in this case. */
+        int old_ngroups = ngroups;
+        if (getgrouplist(user, basegid, groups, &ngroups) != -1) {
+            /* Success */
+            break;
+        }
+
+        /* getgrouplist() fails if the group list is too small */
         PyMem_Free(groups);
 
-        if (ngroups > INT_MAX / 2) {
-            return PyErr_NoMemory();
+        if (ngroups > old_ngroups) {
+            /* If the group list is too small, the glibc implementation of
+               getgrouplist() sets ngroups to the total number of groups and
+               returns -1. */
         }
-        ngroups *= 2;
-
-        groups = PyMem_New(int, ngroups);
-        if (groups == NULL) {
-            return PyErr_NoMemory();
+        else {
+            /* Double the group list size */
+            if (ngroups > INT_MAX / 2) {
+                return PyErr_NoMemory();
+            }
+            ngroups *= 2;
         }
+
+        /* Retry getgrouplist() with a larger group list */
     }
-#else
-    if (getgrouplist(user, basegid, groups, &ngroups) == -1) {
-        PyMem_Del(groups);
-        return posix_error();
-    }
-#endif
 
 #ifdef _Py_MEMORY_SANITIZER
     /* Clang memory sanitizer libc intercepts don't know getgrouplist. */