From 22417958afd9ee6b19c1f23276388c6a1df446db Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Tue, 3 Aug 2021 23:57:13 +0200 Subject: [PATCH 1/8] bpo-44822: Pass string size to sqlite3_result_text() --- Modules/_sqlite/connection.c | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/Modules/_sqlite/connection.c b/Modules/_sqlite/connection.c index f75cf833807114..cf6831aada5c33 100644 --- a/Modules/_sqlite/connection.c +++ b/Modules/_sqlite/connection.c @@ -519,10 +519,15 @@ _pysqlite_set_result(sqlite3_context* context, PyObject* py_val) } else if (PyFloat_Check(py_val)) { sqlite3_result_double(context, PyFloat_AsDouble(py_val)); } else if (PyUnicode_Check(py_val)) { - const char *str = PyUnicode_AsUTF8(py_val); - if (str == NULL) + Py_ssize_t sz; + const char *str = PyUnicode_AsUTF8AndSize(py_val, &sz); + if (str == NULL) { return -1; - sqlite3_result_text(context, str, -1, SQLITE_TRANSIENT); + } + if (sz > INT_MAX) { + sz = -1; // Let SQLite compute string length + } + sqlite3_result_text(context, str, (int)sz, SQLITE_TRANSIENT); } else if (PyObject_CheckBuffer(py_val)) { Py_buffer view; if (PyObject_GetBuffer(py_val, &view, PyBUF_SIMPLE) != 0) { From 58d324417e7e21b1751e8ec03abf63ea5025f605 Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Wed, 4 Aug 2021 11:07:55 +0200 Subject: [PATCH 2/8] Raise OverflowError if size > INT_MAX --- Modules/_sqlite/connection.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Modules/_sqlite/connection.c b/Modules/_sqlite/connection.c index cf6831aada5c33..c458c457f5897a 100644 --- a/Modules/_sqlite/connection.c +++ b/Modules/_sqlite/connection.c @@ -525,7 +525,9 @@ _pysqlite_set_result(sqlite3_context* context, PyObject* py_val) return -1; } if (sz > INT_MAX) { - sz = -1; // Let SQLite compute string length + PyErr_SetString(PyExc_OverflowError, + "String is longer than INT_MAX bytes"); + return -1; } sqlite3_result_text(context, str, (int)sz, SQLITE_TRANSIENT); } else if (PyObject_CheckBuffer(py_val)) { From 73639f12ed432562164f99a2f02182f84af794c0 Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Wed, 4 Aug 2021 11:59:57 +0200 Subject: [PATCH 3/8] Add test --- Lib/sqlite3/test/userfunctions.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/Lib/sqlite3/test/userfunctions.py b/Lib/sqlite3/test/userfunctions.py index 1ed090e3d92568..30a50dd1e40227 100644 --- a/Lib/sqlite3/test/userfunctions.py +++ b/Lib/sqlite3/test/userfunctions.py @@ -53,6 +53,8 @@ def wrapper(self, *args, **kwargs): def func_returntext(): return "foo" +def func_returntextwithnull(): + return "1\x002" def func_returnunicode(): return "bar" def func_returnint(): @@ -168,6 +170,7 @@ def setUp(self): self.con = sqlite.connect(":memory:") self.con.create_function("returntext", 0, func_returntext) + self.con.create_function("returntextwithnull", 0, func_returntextwithnull) self.con.create_function("returnunicode", 0, func_returnunicode) self.con.create_function("returnint", 0, func_returnint) self.con.create_function("returnfloat", 0, func_returnfloat) @@ -211,6 +214,12 @@ def test_func_return_text(self): self.assertEqual(type(val), str) self.assertEqual(val, "foo") + def test_func_return_text_with_null_char(self): + cur = self.con.cursor() + res = cur.execute("select returntextwithnull()").fetchone()[0] + self.assertEqual(type(res), str) + self.assertEqual(res, "1\x002") + def test_func_return_unicode(self): cur = self.con.cursor() cur.execute("select returnunicode()") From 60d94f959ef357c18b2855f0e2c61da51d613281 Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Wed, 4 Aug 2021 12:29:11 +0200 Subject: [PATCH 4/8] Add NEWS entry --- .../next/Library/2021-08-04-12-29-00.bpo-44822.zePNXA.rst | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 Misc/NEWS.d/next/Library/2021-08-04-12-29-00.bpo-44822.zePNXA.rst diff --git a/Misc/NEWS.d/next/Library/2021-08-04-12-29-00.bpo-44822.zePNXA.rst b/Misc/NEWS.d/next/Library/2021-08-04-12-29-00.bpo-44822.zePNXA.rst new file mode 100644 index 00000000000000..98a35445942e7c --- /dev/null +++ b/Misc/NEWS.d/next/Library/2021-08-04-12-29-00.bpo-44822.zePNXA.rst @@ -0,0 +1,3 @@ +:mod:`sqlite3` now raises :exc:`OverflowError` if an user-defined function +returns a :class:`str` larger than ``INT_MAX`` bytes. Patch by Erlend E. +Aasland. From 10fbc4bf34d4416684df8a286c5492f7f51413d7 Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Wed, 4 Aug 2021 12:34:21 +0200 Subject: [PATCH 5/8] Don't capitalize error string --- Modules/_sqlite/connection.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Modules/_sqlite/connection.c b/Modules/_sqlite/connection.c index c458c457f5897a..aae6c66d63faba 100644 --- a/Modules/_sqlite/connection.c +++ b/Modules/_sqlite/connection.c @@ -526,7 +526,7 @@ _pysqlite_set_result(sqlite3_context* context, PyObject* py_val) } if (sz > INT_MAX) { PyErr_SetString(PyExc_OverflowError, - "String is longer than INT_MAX bytes"); + "string is longer than INT_MAX bytes"); return -1; } sqlite3_result_text(context, str, (int)sz, SQLITE_TRANSIENT); From a4830525c0aecb58839f9fe9c5173903dfebdcbc Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Wed, 4 Aug 2021 15:57:31 +0200 Subject: [PATCH 6/8] Add aggregate test --- Lib/sqlite3/test/userfunctions.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/Lib/sqlite3/test/userfunctions.py b/Lib/sqlite3/test/userfunctions.py index 30a50dd1e40227..b3da3c425b8035 100644 --- a/Lib/sqlite3/test/userfunctions.py +++ b/Lib/sqlite3/test/userfunctions.py @@ -165,6 +165,15 @@ def step(self, val): def finalize(self): return self.val +class AggrText: + def __init__(self): + self.txt = "" + def step(self, txt): + self.txt = self.txt + txt + def finalize(self): + return self.txt + + class FunctionTests(unittest.TestCase): def setUp(self): self.con = sqlite.connect(":memory:") @@ -399,6 +408,7 @@ def setUp(self): self.con.create_aggregate("checkType", 2, AggrCheckType) self.con.create_aggregate("checkTypes", -1, AggrCheckTypes) self.con.create_aggregate("mysum", 1, AggrSum) + self.con.create_aggregate("aggtxt", 1, AggrText) def tearDown(self): #self.cur.close() @@ -495,6 +505,15 @@ def test_aggr_no_match(self): val = cur.fetchone()[0] self.assertIsNone(val) + def test_aggr_text(self): + cur = self.con.cursor() + for txt in ["foo", "1\x002"]: + with self.subTest(txt=txt): + cur.execute("select aggtxt(?) from test", (txt,)) + val = cur.fetchone()[0] + self.assertEqual(val, txt) + + class AuthorizerTests(unittest.TestCase): @staticmethod def authorizer_cb(action, arg1, arg2, dbname, source): From 55963ca0da7069238fcf0f5146a3ae5b0ac98793 Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Thu, 5 Aug 2021 00:13:13 +0200 Subject: [PATCH 7/8] Address review: reword NEWS entry --- .../next/Library/2021-08-04-12-29-00.bpo-44822.zePNXA.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Misc/NEWS.d/next/Library/2021-08-04-12-29-00.bpo-44822.zePNXA.rst b/Misc/NEWS.d/next/Library/2021-08-04-12-29-00.bpo-44822.zePNXA.rst index 98a35445942e7c..c6abbf87b11e3d 100644 --- a/Misc/NEWS.d/next/Library/2021-08-04-12-29-00.bpo-44822.zePNXA.rst +++ b/Misc/NEWS.d/next/Library/2021-08-04-12-29-00.bpo-44822.zePNXA.rst @@ -1,3 +1,3 @@ -:mod:`sqlite3` now raises :exc:`OverflowError` if an user-defined function -returns a :class:`str` larger than ``INT_MAX`` bytes. Patch by Erlend E. +:mod:`sqlite3` user-defined functions returning :class:`strings ` with +embedded ``NULL`` characters are no longer truncated. Patch by Erlend E. Aasland. From 7a2c279583482b0c5ac0a9c5600db7bd5d9cd0ec Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Thu, 5 Aug 2021 08:35:26 +0300 Subject: [PATCH 8/8] Apply suggestions from code review --- .../next/Library/2021-08-04-12-29-00.bpo-44822.zePNXA.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Misc/NEWS.d/next/Library/2021-08-04-12-29-00.bpo-44822.zePNXA.rst b/Misc/NEWS.d/next/Library/2021-08-04-12-29-00.bpo-44822.zePNXA.rst index c6abbf87b11e3d..d078142886d2e0 100644 --- a/Misc/NEWS.d/next/Library/2021-08-04-12-29-00.bpo-44822.zePNXA.rst +++ b/Misc/NEWS.d/next/Library/2021-08-04-12-29-00.bpo-44822.zePNXA.rst @@ -1,3 +1,3 @@ -:mod:`sqlite3` user-defined functions returning :class:`strings ` with -embedded ``NULL`` characters are no longer truncated. Patch by Erlend E. -Aasland. +:mod:`sqlite3` user-defined functions and aggregators returning +:class:`strings ` with embedded NUL characters are no longer +truncated. Patch by Erlend E. Aasland.