Skip to content

bpo-42213: Remove redundant cyclic GC hack in sqlite3 #26462

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 13 commits into from
Closed

bpo-42213: Remove redundant cyclic GC hack in sqlite3 #26462

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
13 commits
Select commit Hold shift + click to select a range
70511f3
bpo-42213: Remove redundant cyclic GC hack in sqlite3
May 29, 2021
6cb435b
Split open tests into their own test case, and cleanup more explicitly
Jun 2, 2021
1e3f589
clean up test_trace_callback_content explicitly
Jun 2, 2021
5ff1a80
Document workarounds
Jun 2, 2021
866e226
Use test.support.gc_collect iso. gc.collect
Jun 2, 2021
063878c
Revert gc workarounds
Jun 2, 2021
b53aacb
Address review: Explicitly close opened database files
Jun 2, 2021
e0ffe76
Try to harden connection close
Jun 2, 2021
4ebb593
sqlite3_close_v2() always returns SQLITE_OK
Jun 2, 2021
7d8014a
Check connection on __enter__
Jun 2, 2021
1b23df1
Add tests that exercise stuff against a closed database
Jun 2, 2021
d31403e
Harden test clean up
Jun 2, 2021
aba8dd7
Add managed resource helper
Jun 2, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
59 changes: 46 additions & 13 deletions Lib/sqlite3/test/dbapi.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,14 +20,26 @@
# misrepresented as being the original software.
# 3. This notice may not be removed or altered from any source distribution.

import threading
import unittest
import contextlib
import sqlite3 as sqlite
import sys
import threading
import unittest

from test.support.os_helper import TESTFN, unlink


# Helper for tests using TESTFN
@contextlib.contextmanager
def managed_connect(*args, **kwargs):
cx = sqlite.connect(*args, **kwargs)
try:
yield cx
finally:
cx.close()
unlink(TESTFN)


class ModuleTests(unittest.TestCase):
def test_api_level(self):
self.assertEqual(sqlite.apilevel, "2.0",
Expand Down Expand Up @@ -135,6 +147,25 @@ def test_failed_open(self):
def test_close(self):
self.cx.close()

def test_use_after_close(self):
sql = "select 1"
cu = self.cx.cursor()
res = cu.execute(sql)
self.cx.close()
self.assertRaises(sqlite.ProgrammingError, res.fetchall)
self.assertRaises(sqlite.ProgrammingError, cu.execute, sql)
self.assertRaises(sqlite.ProgrammingError, cu.executemany, sql, [])
self.assertRaises(sqlite.ProgrammingError, cu.executescript, sql)
self.assertRaises(sqlite.ProgrammingError, self.cx.execute, sql)
self.assertRaises(sqlite.ProgrammingError,
self.cx.executemany, sql, [])
self.assertRaises(sqlite.ProgrammingError, self.cx.executescript, sql)
self.assertRaises(sqlite.ProgrammingError,
self.cx.create_function, "t", 1, lambda x: x)
with self.assertRaises(sqlite.ProgrammingError):
with self.cx:
pass

def test_exceptions(self):
# Optional DB-API extension.
self.assertEqual(self.cx.Warning, sqlite.Warning)
Expand Down Expand Up @@ -170,26 +201,27 @@ def test_in_transaction_ro(self):
with self.assertRaises(AttributeError):
self.cx.in_transaction = True

class OpenTests(unittest.TestCase):
_sql = "create table test(id integer)"

def test_open_with_path_like_object(self):
""" Checks that we can successfully connect to a database using an object that
is PathLike, i.e. has __fspath__(). """
self.addCleanup(unlink, TESTFN)
class Path:
def __fspath__(self):
return TESTFN
path = Path()
with sqlite.connect(path) as cx:
cx.execute('create table test(id integer)')
with managed_connect(path) as cx:
cx.execute(self._sql)

def test_open_uri(self):
self.addCleanup(unlink, TESTFN)
with sqlite.connect(TESTFN) as cx:
cx.execute('create table test(id integer)')
with sqlite.connect('file:' + TESTFN, uri=True) as cx:
cx.execute('insert into test(id) values(0)')
with sqlite.connect('file:' + TESTFN + '?mode=ro', uri=True) as cx:
with self.assertRaises(sqlite.OperationalError):
cx.execute('insert into test(id) values(1)')
with managed_connect(TESTFN) as cx:
cx.execute(self._sql)
with managed_connect(f"file:{TESTFN}", uri=True) as cx:
cx.execute(self._sql)
with self.assertRaises(sqlite.OperationalError):
with managed_connect(f"file:{TESTFN}?mode=ro", uri=True) as cx:
cx.execute(self._sql)


class CursorTests(unittest.TestCase):
Expand Down Expand Up @@ -942,6 +974,7 @@ def suite():
CursorTests,
ExtensionTests,
ModuleTests,
OpenTests,
SqliteOnConflictTests,
ThreadTests,
]
Expand Down
14 changes: 9 additions & 5 deletions Lib/sqlite3/test/hooks.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -254,11 +254,15 @@ def trace(statement):
self.addCleanup(unlink, TESTFN)
con1 = sqlite.connect(TESTFN, isolation_level=None)
con2 = sqlite.connect(TESTFN)
con1.set_trace_callback(trace)
cur = con1.cursor()
cur.execute(queries[0])
con2.execute("create table bar(x)")
cur.execute(queries[1])
try:
con1.set_trace_callback(trace)
cur = con1.cursor()
cur.execute(queries[0])
con2.execute("create table bar(x)")
cur.execute(queries[1])
finally:
con1.close()
con2.close()
self.assertEqual(traced_statements, queries)


Expand Down
11 changes: 2 additions & 9 deletions Modules/_sqlite/cache.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -97,9 +97,6 @@ pysqlite_cache_init(pysqlite_Cache *self, PyObject *args, PyObject *kwargs)
}

self->factory = Py_NewRef(factory);

self->decref_factory = 1;

return 0;
}

Expand All @@ -108,9 +105,7 @@ cache_traverse(pysqlite_Cache *self, visitproc visit, void *arg)
{
Py_VISIT(Py_TYPE(self));
Py_VISIT(self->mapping);
if (self->decref_factory) {
Py_VISIT(self->factory);
}
Py_VISIT(self->factory);

pysqlite_Node *node = self->first;
while (node) {
Expand All @@ -124,9 +119,7 @@ static int
cache_clear(pysqlite_Cache *self)
{
Py_CLEAR(self->mapping);
if (self->decref_factory) {
Py_CLEAR(self->factory);
}
Py_CLEAR(self->factory);

/* iterate over all nodes and deallocate them */
pysqlite_Node *node = self->first;
Expand Down
4 changes: 0 additions & 4 deletions Modules/_sqlite/cache.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -52,10 +52,6 @@ typedef struct

pysqlite_Node* first;
pysqlite_Node* last;

/* if set, decrement the factory function when the Cache is deallocated.
* this is almost always desirable, but not in the pysqlite context */
int decref_factory;
} pysqlite_Cache;

extern PyTypeObject *pysqlite_NodeType;
Expand Down
42 changes: 20 additions & 22 deletions Modules/_sqlite/connection.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -149,14 +149,6 @@ pysqlite_connection_init(pysqlite_Connection *self, PyObject *args,
return -1;
}

/* By default, the Cache class INCREFs the factory in its initializer, and
* decrefs it in its deallocator method. Since this would create a circular
* reference here, we're breaking it by decrementing self, and telling the
* cache class to not decref the factory (self) in its deallocator.
*/
self->statement_cache->decref_factory = 0;
Py_DECREF(self);

self->detect_types = detect_types;
self->timeout = timeout;
(void)sqlite3_busy_timeout(self->db, (int)(timeout*1000));
Expand Down Expand Up @@ -258,6 +250,16 @@ connection_clear(pysqlite_Connection *self)
return 0;
}

static void
connection_close(pysqlite_Connection *self)
{
if (self->db) {
int rc = sqlite3_close_v2(self->db);
assert(rc == SQLITE_OK);
self->db = NULL;
}
}

static void
connection_dealloc(pysqlite_Connection *self)
{
Expand All @@ -266,9 +268,7 @@ connection_dealloc(pysqlite_Connection *self)
tp->tp_clear((PyObject *)self);

/* Clean up if user has not called .close() explicitly. */
if (self->db) {
sqlite3_close_v2(self->db);
}
connection_close(self);

tp->tp_free(self);
Py_DECREF(tp);
Expand Down Expand Up @@ -353,23 +353,18 @@ static PyObject *
pysqlite_connection_close_impl(pysqlite_Connection *self)
/*[clinic end generated code: output=a546a0da212c9b97 input=3d58064bbffaa3d3]*/
{
int rc;

if (!pysqlite_check_thread(self)) {
return NULL;
}

pysqlite_do_all_statements(self, ACTION_FINALIZE, 1);

if (self->db) {
rc = sqlite3_close_v2(self->db);
/* Free pending statements before closing. This implies also cleaning
* up cursors, as they may have strong refs to statements. */
Py_CLEAR(self->statement_cache);
Py_CLEAR(self->statements);
Py_CLEAR(self->cursors);
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

What happens if you use the connection after you call close()? Are we checking for NULL for these veriables everywhere else? (Cannot check myself as I'm on my phone)

Copy link
Contributor Author

@erlend-aasland erlend-aasland Jun 2, 2021 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

You'll get a sqlite3.ProgrammingError. There's a lot of sanity checks in the code for this. The sqlite3 test suite runs fine, but of course, we do not have 100% code coverage yet. (Digression: see issue 43553 for improving sqlite3 code coverage.)

After close, if you try to use a cursor, fetch from a pending statement, or manipulate the connection, you'll get a sqlite3.ProgrammingError:

>>> cx = sqlite3.connect(":memory:")
>>> cu = cx.cursor()
>>> res = cu.execute("select 1")
>>> cx.close()
>>> res.fetchall()
Traceback (most recent call last):
  File "<stdin>", line 1, in <module>
sqlite3.ProgrammingError: Cannot operate on a closed database.
>>> cu.execute("select 1")
Traceback (most recent call last):
  File "<stdin>", line 1, in <module>
sqlite3.ProgrammingError: Cannot operate on a closed database.
>>> cx.create_function("test", 1, lambda x: x)
Traceback (most recent call last):
  File "<stdin>", line 1, in <module>
sqlite3.ProgrammingError: Cannot operate on a closed database.
>>>

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I added some more tests wrt. this: 1b23df1
Also, there's some regression tests that exercise operations on closed connections (and closed cursors).

Let me know if you want me to add more tests.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Here's a strange observation:
I did a ref leak test on Windows yesterday, and it segfaulted on test_bpo31770. After some testing, I reverted the change highlighted in this conversation; I removed the three Py_CLEAR, added pysqlite_do_all_statements(self, ACTION_FINALIZE, 1) back, but kept connection_close(). That fixed the ref leak test on Windows, and the test suite still worked; it did not "leak" open test database files. So, for the new PR, I'll keep this modification. We'll see how the CI fares :)

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

cc. @vstinner

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

CI seems to be fine with this: erlend-aasland#9


if (rc != SQLITE_OK) {
_pysqlite_seterror(self->db);
return NULL;
} else {
self->db = NULL;
}
connection_close(self);
}

Py_RETURN_NONE;
Expand Down Expand Up @@ -1820,6 +1815,9 @@ static PyObject *
pysqlite_connection_enter_impl(pysqlite_Connection *self)
/*[clinic end generated code: output=457b09726d3e9dcd input=127d7a4f17e86d8f]*/
{
if (!pysqlite_check_connection(self)) {
return NULL;
}
return Py_NewRef((PyObject *)self);
}

Expand Down