From bf9677c7cf697d50c81f4ddcb41b93918c01bec9 Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Tue, 26 Oct 2021 09:25:04 +0200 Subject: [PATCH 01/12] bpo-45613: Dynamically set sqlite3.threadsafety Use the compile-time selected default SQLite threaded mode to set the DB-API 2.0 attribute 'threadsafety' Mappings: - SQLITE_THREADSAFE=0 => threadsafety=0 - SQLITE_THREADSAFE=1 => threadsafety=3 - SQLITE_THREADSAFE=2 => threadsafety=1 --- Lib/sqlite3/dbapi2.py | 32 +++++++++++++++++++++++++++++++- Lib/sqlite3/test/test_dbapi.py | 5 +++-- 2 files changed, 34 insertions(+), 3 deletions(-) diff --git a/Lib/sqlite3/dbapi2.py b/Lib/sqlite3/dbapi2.py index cfe6225f46efc0..44cebd3847d4dc 100644 --- a/Lib/sqlite3/dbapi2.py +++ b/Lib/sqlite3/dbapi2.py @@ -28,7 +28,37 @@ paramstyle = "qmark" -threadsafety = 1 + +def fetch_compile_options(): + try: + cx = connect(":memory:") + res = cx.execute("select * from pragma_compile_options") + opts = {i[0] for i in res} + finally: + cx.close() + return opts + +_compile_options = fetch_compile_options() +del fetch_compile_options + +def _threadsafety(): + if "THREADSAFE=0" in _compile_options: # SQLite single-thread mode + # Translates to DB-API threadsafety level 0, meaning threads may not + # share the module. + return 0 + elif "THREADSAFE=2" in _compile_options: + # SQLITE_THREADSAFE=2 (multi-thread mode) translates to DB-API + # threadsafety level 1, meaning threads may share the module, but not + # connections. + return 1 + + # Default to SQLITE_THREADSAFE=1 (serialized mode), which translates to + # DB-API threadsafety level 3, meaning threads may share the module, + # connections and cursors. This is the default threaded mode of SQLite. + return 3 + +threadsafety = _threadsafety() +del _threadsafety apilevel = "2.0" diff --git a/Lib/sqlite3/test/test_dbapi.py b/Lib/sqlite3/test/test_dbapi.py index d82543663d18bb..3b79c904bfc172 100644 --- a/Lib/sqlite3/test/test_dbapi.py +++ b/Lib/sqlite3/test/test_dbapi.py @@ -54,8 +54,9 @@ def test_api_level(self): "apilevel is %s, should be 2.0" % sqlite.apilevel) def test_thread_safety(self): - self.assertEqual(sqlite.threadsafety, 1, - "threadsafety is %d, should be 1" % sqlite.threadsafety) + self.assertIn(sqlite.threadsafety, {0, 1, 3}, + "threadsafety is %d, should be 0, 1 or 3" % + sqlite.threadsafety) def test_param_style(self): self.assertEqual(sqlite.paramstyle, "qmark", From 9747c80eb7bac7c8116fe222a99e5b5f487026ff Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Tue, 26 Oct 2021 14:31:26 +0200 Subject: [PATCH 02/12] Add NEWS and What's New --- Doc/whatsnew/3.11.rst | 6 ++++++ .../next/Library/2021-10-26-14-29-54.bpo-45613.55Ie3c.rst | 3 +++ 2 files changed, 9 insertions(+) create mode 100644 Misc/NEWS.d/next/Library/2021-10-26-14-29-54.bpo-45613.55Ie3c.rst diff --git a/Doc/whatsnew/3.11.rst b/Doc/whatsnew/3.11.rst index 7b3ce9bc7feaf6..44c7b43e51fa6c 100644 --- a/Doc/whatsnew/3.11.rst +++ b/Doc/whatsnew/3.11.rst @@ -248,6 +248,12 @@ sqlite3 (Contributed by Aviv Palivoda, Daniel Shahaf, and Erlend E. Aasland in :issue:`16379`.) +* :mod:`sqlite3` now sets :attr:`sqlite3.threadsafety` dynamically, based on + the default threaded mode the underlying SQLite library has been compiled + with. + (Contributed by Erlend E. Aasland in :issue:`45613`.) + + threading --------- diff --git a/Misc/NEWS.d/next/Library/2021-10-26-14-29-54.bpo-45613.55Ie3c.rst b/Misc/NEWS.d/next/Library/2021-10-26-14-29-54.bpo-45613.55Ie3c.rst new file mode 100644 index 00000000000000..720969df4cc50e --- /dev/null +++ b/Misc/NEWS.d/next/Library/2021-10-26-14-29-54.bpo-45613.55Ie3c.rst @@ -0,0 +1,3 @@ +:mod:`sqlite3` now sets :attr:`sqlite3.threadsafety` dynamically, based on +the default threaded mode the underlying SQLite library has been compiled +with. Patch by Erlend E. Aasland. From 9d2aa820f20c0adfa131572a736e8e6a63559e6d Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Tue, 26 Oct 2021 14:55:51 +0200 Subject: [PATCH 03/12] Compatibility with older SQLite releases --- Lib/sqlite3/dbapi2.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/sqlite3/dbapi2.py b/Lib/sqlite3/dbapi2.py index 44cebd3847d4dc..032ec632ae1360 100644 --- a/Lib/sqlite3/dbapi2.py +++ b/Lib/sqlite3/dbapi2.py @@ -32,7 +32,7 @@ def fetch_compile_options(): try: cx = connect(":memory:") - res = cx.execute("select * from pragma_compile_options") + res = cx.execute("pragma compile_options") opts = {i[0] for i in res} finally: cx.close() From 06022ec9119c287095e62d71c4c3386e84b3ffab Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Tue, 26 Oct 2021 15:07:49 +0200 Subject: [PATCH 04/12] Adjust wording: threaded => threading --- Doc/whatsnew/3.11.rst | 2 +- Lib/sqlite3/dbapi2.py | 2 +- .../next/Library/2021-10-26-14-29-54.bpo-45613.55Ie3c.rst | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Doc/whatsnew/3.11.rst b/Doc/whatsnew/3.11.rst index 44c7b43e51fa6c..191ddfd7c5c591 100644 --- a/Doc/whatsnew/3.11.rst +++ b/Doc/whatsnew/3.11.rst @@ -249,7 +249,7 @@ sqlite3 :issue:`16379`.) * :mod:`sqlite3` now sets :attr:`sqlite3.threadsafety` dynamically, based on - the default threaded mode the underlying SQLite library has been compiled + the default threading mode the underlying SQLite library has been compiled with. (Contributed by Erlend E. Aasland in :issue:`45613`.) diff --git a/Lib/sqlite3/dbapi2.py b/Lib/sqlite3/dbapi2.py index 032ec632ae1360..199dc93d7a88fe 100644 --- a/Lib/sqlite3/dbapi2.py +++ b/Lib/sqlite3/dbapi2.py @@ -54,7 +54,7 @@ def _threadsafety(): # Default to SQLITE_THREADSAFE=1 (serialized mode), which translates to # DB-API threadsafety level 3, meaning threads may share the module, - # connections and cursors. This is the default threaded mode of SQLite. + # connections and cursors. This is the default threading mode of SQLite. return 3 threadsafety = _threadsafety() diff --git a/Misc/NEWS.d/next/Library/2021-10-26-14-29-54.bpo-45613.55Ie3c.rst b/Misc/NEWS.d/next/Library/2021-10-26-14-29-54.bpo-45613.55Ie3c.rst index 720969df4cc50e..f1d3a98c6b65be 100644 --- a/Misc/NEWS.d/next/Library/2021-10-26-14-29-54.bpo-45613.55Ie3c.rst +++ b/Misc/NEWS.d/next/Library/2021-10-26-14-29-54.bpo-45613.55Ie3c.rst @@ -1,3 +1,3 @@ :mod:`sqlite3` now sets :attr:`sqlite3.threadsafety` dynamically, based on -the default threaded mode the underlying SQLite library has been compiled +the default threading mode the underlying SQLite library has been compiled with. Patch by Erlend E. Aasland. From 4706e176d2ba2b6b38f7fbacd71ae132a4cc6fb9 Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Thu, 28 Oct 2021 22:44:37 +0200 Subject: [PATCH 05/12] Update docs --- Doc/library/sqlite3.rst | 48 +++++++++++++++++++++++++++-------------- 1 file changed, 32 insertions(+), 16 deletions(-) diff --git a/Doc/library/sqlite3.rst b/Doc/library/sqlite3.rst index fe1b64ade95612..9cae6410657a88 100644 --- a/Doc/library/sqlite3.rst +++ b/Doc/library/sqlite3.rst @@ -159,22 +159,38 @@ Module functions and constants .. data:: threadsafety - Integer constant required by the DB-API, stating the level of thread safety - the :mod:`sqlite3` module supports. Currently hard-coded to ``1``, meaning - *"Threads may share the module, but not connections."* However, this may not - always be true. You can check the underlying SQLite library's compile-time - threaded mode using the following query:: - - import sqlite3 - con = sqlite3.connect(":memory:") - con.execute(""" - select * from pragma_compile_options - where compile_options like 'THREADSAFE=%' - """).fetchall() - - Note that the `SQLITE_THREADSAFE levels - `_ do not match the DB-API 2.0 - ``threadsafety`` levels. + Integer constant required by the DB-API 2.0, stating the level of thread + safety the :mod:`sqlite3` module supports. This attribute is set based on + the default `threading mode `_ the + underlying SQLite library is compiled with. The SQLite threading modes are: + + 1. **Single-thread**: In this mode, all mutexes are disabled and SQLite is + unsafe to use in more than a single thread at once. + 2. **Multi-thread**: In this mode, SQLite can be safely used by multiple + threads provided that no single database connection is used + simultaneously in two or more threads. + 3. **Serialized**: In serialized mode, SQLite can be safely used by + multiple threads with no restriction. + + The mappings from SQLite threading modes to DB-API 2.0 threadsafety levels + are as follows: + + +------------------+-----------------+----------------------+-------------------------------+ + | SQLite threading | `threadsafety`_ | `SQLITE_THREADSAFE`_ | DB-API 2.0 meaning | + | mode | | | | + +==================+=================+======================+===============================+ + | single-thread | 0 | 0 | Threads may not share the | + | | | | module | + +------------------+-----------------+----------------------+-------------------------------+ + | multi-thread | 1 | 2 | Threads may share the module, | + | | | | but not connections | + +------------------+-----------------+----------------------+-------------------------------+ + | serialized | 3 | 1 | Threads may share the module, | + | | | | connections and cursors | + +------------------+-----------------+----------------------+-------------------------------+ + + .. _threadsafety: https://www.python.org/dev/peps/pep-0249/#threadsafety + .. _SQLITE_THREADSAFE: https://sqlite.org/compile.html#threadsafe .. data:: PARSE_DECLTYPES From d45e27ab40e383288d91ccefbec74df3d9ab437c Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Thu, 28 Oct 2021 22:51:52 +0200 Subject: [PATCH 06/12] Nitpick --- Lib/sqlite3/dbapi2.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Lib/sqlite3/dbapi2.py b/Lib/sqlite3/dbapi2.py index 199dc93d7a88fe..dc3149bfa64e16 100644 --- a/Lib/sqlite3/dbapi2.py +++ b/Lib/sqlite3/dbapi2.py @@ -42,9 +42,9 @@ def fetch_compile_options(): del fetch_compile_options def _threadsafety(): - if "THREADSAFE=0" in _compile_options: # SQLite single-thread mode - # Translates to DB-API threadsafety level 0, meaning threads may not - # share the module. + if "THREADSAFE=0" in _compile_options: + # SQLITE_THREADSAFE=0 (single-thread mode) translates to DB-API + # threadsafety level 0, meaning threads may not share the module. return 0 elif "THREADSAFE=2" in _compile_options: # SQLITE_THREADSAFE=2 (multi-thread mode) translates to DB-API From db01838c130db3a54379f6f017836e94bdf27c8d Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Thu, 28 Oct 2021 23:04:54 +0200 Subject: [PATCH 07/12] Test for exact safety level --- Lib/sqlite3/test/test_dbapi.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/Lib/sqlite3/test/test_dbapi.py b/Lib/sqlite3/test/test_dbapi.py index 3b79c904bfc172..5be5eb9d2e56c5 100644 --- a/Lib/sqlite3/test/test_dbapi.py +++ b/Lib/sqlite3/test/test_dbapi.py @@ -54,9 +54,17 @@ def test_api_level(self): "apilevel is %s, should be 2.0" % sqlite.apilevel) def test_thread_safety(self): - self.assertIn(sqlite.threadsafety, {0, 1, 3}, - "threadsafety is %d, should be 0, 1 or 3" % - sqlite.threadsafety) + from sqlite3.dbapi2 import _compile_options + msg = "threadsafety is %d, should be %d in %s mode" + if "THREADSAFE=0" in _compile_options: + self.assertEqual(sqlite.threadsafety, 0, + msg % (sqlite.threadsafety, 0, "single-thread")) + elif "THREADSAFE=2" in _compile_options: + self.assertEqual(sqlite.threadsafety, 1, + msg % (sqlite.threadsafety, 1, "multi-thread")) + else: # THREADSAFE=1 (default) + self.assertEqual(sqlite.threadsafety, 3, + msg % (sqlite.threadsafety, 3, "serialized")) def test_param_style(self): self.assertEqual(sqlite.paramstyle, "qmark", From 396abd4cd679f3a79f2daea48b64ef5fddf64562 Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Thu, 28 Oct 2021 23:07:42 +0200 Subject: [PATCH 08/12] Reword NEWS/What's New --- Doc/whatsnew/3.11.rst | 5 ++--- .../next/Library/2021-10-26-14-29-54.bpo-45613.55Ie3c.rst | 6 +++--- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/Doc/whatsnew/3.11.rst b/Doc/whatsnew/3.11.rst index 6678186ffff604..a66d35ced77869 100644 --- a/Doc/whatsnew/3.11.rst +++ b/Doc/whatsnew/3.11.rst @@ -248,9 +248,8 @@ sqlite3 (Contributed by Aviv Palivoda, Daniel Shahaf, and Erlend E. Aasland in :issue:`16379`.) -* :mod:`sqlite3` now sets :attr:`sqlite3.threadsafety` dynamically, based on - the default threading mode the underlying SQLite library has been compiled - with. +* :mod:`sqlite3` now sets :attr:`sqlite3.threadsafety` based on the default + threading mode the underlying SQLite library has been compiled with. (Contributed by Erlend E. Aasland in :issue:`45613`.) diff --git a/Misc/NEWS.d/next/Library/2021-10-26-14-29-54.bpo-45613.55Ie3c.rst b/Misc/NEWS.d/next/Library/2021-10-26-14-29-54.bpo-45613.55Ie3c.rst index f1d3a98c6b65be..ac0937b54aeeab 100644 --- a/Misc/NEWS.d/next/Library/2021-10-26-14-29-54.bpo-45613.55Ie3c.rst +++ b/Misc/NEWS.d/next/Library/2021-10-26-14-29-54.bpo-45613.55Ie3c.rst @@ -1,3 +1,3 @@ -:mod:`sqlite3` now sets :attr:`sqlite3.threadsafety` dynamically, based on -the default threading mode the underlying SQLite library has been compiled -with. Patch by Erlend E. Aasland. +:mod:`sqlite3` now sets :attr:`sqlite3.threadsafety` based on the default +threading mode the underlying SQLite library has been compiled with. Patch by +Erlend E. Aasland. From 084173a7658fd47a1773c1bb02274412ffdb644e Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Tue, 2 Nov 2021 21:48:59 +0100 Subject: [PATCH 09/12] Use sqlite3_treadsafe() instead of querying compile_options --- Lib/sqlite3/dbapi2.py | 32 ----------------------------- Lib/test/test_sqlite3/test_dbapi.py | 14 +++---------- Modules/_sqlite/module.c | 27 ++++++++++++++++++++++++ 3 files changed, 30 insertions(+), 43 deletions(-) diff --git a/Lib/sqlite3/dbapi2.py b/Lib/sqlite3/dbapi2.py index dc3149bfa64e16..7cf4dd32d541dd 100644 --- a/Lib/sqlite3/dbapi2.py +++ b/Lib/sqlite3/dbapi2.py @@ -28,38 +28,6 @@ paramstyle = "qmark" - -def fetch_compile_options(): - try: - cx = connect(":memory:") - res = cx.execute("pragma compile_options") - opts = {i[0] for i in res} - finally: - cx.close() - return opts - -_compile_options = fetch_compile_options() -del fetch_compile_options - -def _threadsafety(): - if "THREADSAFE=0" in _compile_options: - # SQLITE_THREADSAFE=0 (single-thread mode) translates to DB-API - # threadsafety level 0, meaning threads may not share the module. - return 0 - elif "THREADSAFE=2" in _compile_options: - # SQLITE_THREADSAFE=2 (multi-thread mode) translates to DB-API - # threadsafety level 1, meaning threads may share the module, but not - # connections. - return 1 - - # Default to SQLITE_THREADSAFE=1 (serialized mode), which translates to - # DB-API threadsafety level 3, meaning threads may share the module, - # connections and cursors. This is the default threading mode of SQLite. - return 3 - -threadsafety = _threadsafety() -del _threadsafety - apilevel = "2.0" Date = datetime.date diff --git a/Lib/test/test_sqlite3/test_dbapi.py b/Lib/test/test_sqlite3/test_dbapi.py index e1690969fe625b..66515aa0d72967 100644 --- a/Lib/test/test_sqlite3/test_dbapi.py +++ b/Lib/test/test_sqlite3/test_dbapi.py @@ -54,17 +54,9 @@ def test_api_level(self): "apilevel is %s, should be 2.0" % sqlite.apilevel) def test_thread_safety(self): - from sqlite3.dbapi2 import _compile_options - msg = "threadsafety is %d, should be %d in %s mode" - if "THREADSAFE=0" in _compile_options: - self.assertEqual(sqlite.threadsafety, 0, - msg % (sqlite.threadsafety, 0, "single-thread")) - elif "THREADSAFE=2" in _compile_options: - self.assertEqual(sqlite.threadsafety, 1, - msg % (sqlite.threadsafety, 1, "multi-thread")) - else: # THREADSAFE=1 (default) - self.assertEqual(sqlite.threadsafety, 3, - msg % (sqlite.threadsafety, 3, "serialized")) + self.assertIn(sqlite.threadsafety, {0, 1, 3}, + "threadsafety is %d, should be 0, 1 or 3" % + sqlite.threadsafety) def test_param_style(self): self.assertEqual(sqlite.paramstyle, "qmark", diff --git a/Modules/_sqlite/module.c b/Modules/_sqlite/module.c index 65229623b84d66..368823dd55435c 100644 --- a/Modules/_sqlite/module.c +++ b/Modules/_sqlite/module.c @@ -415,6 +415,28 @@ add_integer_constants(PyObject *module) { return 0; } +/* Convert SQLite default threading mode (as set by the compile-time constant + * SQLITE_THREADSAFE) to the corresponding DB-API 2.0 (PEP 249) threadsafety + * level. */ +static int +get_threadsafety(pysqlite_state *state) +{ + int mode = sqlite3_threadsafe(); + switch (mode) { + case 0: // SQLite single-thread mode; threads may not share the + return 0; // module. + case 1: // SQLite serialized mode; threads may share the module, + return 3; // connections and cursors. + case 2: // SQLite multi-thread mode; threads may share the module, + return 1; // but not connections. + default: + PyErr_Format(state->InterfaceError, + "Unable to interpret SQLite threadsafety mode. Got %d, " + "expected 0, 1, or 2", mode); + return -1; + } +} + static int module_traverse(PyObject *module, visitproc visit, void *arg) { @@ -564,6 +586,11 @@ module_exec(PyObject *module) goto error; } + int threadsafety = get_threadsafety(state); + if (PyModule_AddIntConstant(module, "threadsafety", threadsafety) < 0) { + goto error; + } + /* initialize microprotocols layer */ if (pysqlite_microprotocols_init(module) < 0) { goto error; From abb7baec2b90072420ab0590d33b935a17eba823 Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Tue, 2 Nov 2021 21:57:08 +0100 Subject: [PATCH 10/12] Adjust comments --- Modules/_sqlite/module.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/Modules/_sqlite/module.c b/Modules/_sqlite/module.c index 368823dd55435c..237146d336c28b 100644 --- a/Modules/_sqlite/module.c +++ b/Modules/_sqlite/module.c @@ -423,12 +423,12 @@ get_threadsafety(pysqlite_state *state) { int mode = sqlite3_threadsafe(); switch (mode) { - case 0: // SQLite single-thread mode; threads may not share the - return 0; // module. - case 1: // SQLite serialized mode; threads may share the module, - return 3; // connections and cursors. - case 2: // SQLite multi-thread mode; threads may share the module, - return 1; // but not connections. + case 0: // Single-thread mode; threads may not share the module. + return 0; + case 1: // Serialized mode; threads may share the module, + return 3; // connections, and cursors. + case 2: // Multi-thread mode; threads may share the module, but not + return 1; // connections. default: PyErr_Format(state->InterfaceError, "Unable to interpret SQLite threadsafety mode. Got %d, " From 686d70ff9423016ff989713fa32f3dfd82dc3bd6 Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Tue, 2 Nov 2021 22:00:37 +0100 Subject: [PATCH 11/12] Bail if get_threadsafety() fails --- Modules/_sqlite/module.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Modules/_sqlite/module.c b/Modules/_sqlite/module.c index 237146d336c28b..5046df43b36f56 100644 --- a/Modules/_sqlite/module.c +++ b/Modules/_sqlite/module.c @@ -587,6 +587,9 @@ module_exec(PyObject *module) } int threadsafety = get_threadsafety(state); + if (threadsafety < 0) { + goto error; + } if (PyModule_AddIntConstant(module, "threadsafety", threadsafety) < 0) { goto error; } From 710a12e354c8f4092a9812062342ae0eb5ec96c3 Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Wed, 3 Nov 2021 20:54:00 +0100 Subject: [PATCH 12/12] Add version changed annotation --- Doc/library/sqlite3.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Doc/library/sqlite3.rst b/Doc/library/sqlite3.rst index 7e4d9059d3b4b6..91bf7b0e7c879e 100644 --- a/Doc/library/sqlite3.rst +++ b/Doc/library/sqlite3.rst @@ -192,6 +192,8 @@ Module functions and constants .. _threadsafety: https://www.python.org/dev/peps/pep-0249/#threadsafety .. _SQLITE_THREADSAFE: https://sqlite.org/compile.html#threadsafe + .. versionchanged:: 3.11 + Set *threadsafety* dynamically instead of hard-coding it to ``1``. .. data:: PARSE_DECLTYPES