diff --git a/Doc/library/csv.rst b/Doc/library/csv.rst index 81949261c563d04..f855eecab6a8c9b 100644 --- a/Doc/library/csv.rst +++ b/Doc/library/csv.rst @@ -51,7 +51,7 @@ The :mod:`!csv` module defines the following functions: .. index:: single: universal newlines; csv.reader function -.. function:: reader(csvfile, /, dialect='excel', **fmtparams) +.. function:: reader(csvfile, /, dialect='excel', *, converter=None, **fmtparams) Return a :ref:`reader object ` that will process lines from the given *csvfile*. A csvfile must be an iterable of @@ -69,7 +69,11 @@ The :mod:`!csv` module defines the following functions: Each row read from the csv file is returned as a list of strings. No automatic data type conversion is performed unless the :data:`QUOTE_NONNUMERIC` format - option is specified (in which case unquoted fields are transformed into floats). + option is specified, + in which case unquoted fields are transformed with the optional *converter* argument, + or into floats if it is not given. + *converter* is called as ``converter(index, field)``, + where *index* is the 0-based position of the field in the row. A short usage example:: @@ -88,8 +92,11 @@ The :mod:`!csv` module defines the following functions: Spam Spam Spam Spam Spam |Baked Beans| Spam |Lovely Spam| |Wonderful Spam| + .. versionadded:: next + The *converter* parameter. -.. function:: writer(csvfile, /, dialect='excel', **fmtparams) + +.. function:: writer(csvfile, /, dialect='excel', *, formatter=None, **fmtparams) Return a writer object responsible for converting the user's data into delimited strings on the given file-like object. *csvfile* can be any object with a @@ -101,12 +108,20 @@ The :mod:`!csv` module defines the following functions: :func:`list_dialects` function. The other optional *fmtparams* keyword arguments can be given to override individual formatting parameters in the current dialect. For full details about dialects and formatting parameters, see - the :ref:`csv-fmt-params` section. To make it - as easy as possible to interface with modules which implement the DB API, the - value :const:`None` is written as the empty string. While this isn't a - reversible transformation, it makes it easier to dump SQL NULL data values to - CSV files without preprocessing the data returned from a ``cursor.fetch*`` call. - All other non-string data are stringified with :func:`str` before being written. + the :ref:`csv-fmt-params` section. + + To make it as easy as possible to interface with modules which implement the DB API, + the value :const:`None` is written as the empty string. + While this isn't a reversible transformation, + it makes it easier to dump SQL NULL data values to CSV files + without preprocessing the data returned from a ``cursor.fetch*`` call. + All other non-string data are stringified before being written + with the optional *formatter* argument, + or with :func:`str` if it is not given. + *formatter* is called as ``formatter(index, value)``, + where *index* is the 0-based position of the field in the row, + and must return a string. + Quoting is still determined by the original value. A short usage example:: @@ -124,6 +139,9 @@ The :mod:`!csv` module defines the following functions: Spam Spam Spam Spam Spam |Baked Beans| Spam |Lovely Spam| |Wonderful Spam| + .. versionadded:: next + The *formatter* parameter. + .. function:: register_dialect(name, /, dialect='excel', **fmtparams) diff --git a/Doc/whatsnew/3.16.rst b/Doc/whatsnew/3.16.rst index 6e69737768d5e15..51fb4aaa2a37c38 100644 --- a/Doc/whatsnew/3.16.rst +++ b/Doc/whatsnew/3.16.rst @@ -120,6 +120,13 @@ csv The results may differ from those of earlier Python versions. (Contributed by Serhiy Storchaka in :gh:`83273`.) +* Add the *converter* parameter in :func:`csv.reader` + and the *formatter* parameter in :func:`csv.writer`. + They are used instead of :func:`float` and :func:`str` + for converting between fields and values, + and allow to convert and format the fields depending on the column. + (Contributed by Serhiy Storchaka in :gh:`155097`.) + curses ------ diff --git a/Lib/test/test_csv.py b/Lib/test/test_csv.py index 91170cc16b3ac95..b583b5cef1590e5 100644 --- a/Lib/test/test_csv.py +++ b/Lib/test/test_csv.py @@ -460,6 +460,88 @@ def test_read_quoting(self): self._read_test(['1\\.5,\\.5,"\\.5"'], [[1.5, 0.5, ".5"]], quoting=csv.QUOTE_STRINGS, escapechar='\\') + def test_read_converter(self): + def converter(index, field): + calls.append((index, field)) + return types[index](field) + + types = [str, int, complex] + calls = [] + self._read_test(['spam,42,1j'], [['spam', 42, 1j]], + quoting=csv.QUOTE_NONNUMERIC, converter=converter) + self.assertEqual(calls, [(0, 'spam'), (1, '42'), (2, '1j')]) + + # The index is the position in the record and is reset for each record. + types = [int] * 3 + calls = [] + self._read_test(['1,2,3', '4,5', '6'], [[1, 2, 3], [4, 5], [6]], + quoting=csv.QUOTE_STRINGS, converter=converter) + self.assertEqual([index for index, field in calls], + [0, 1, 2, 0, 1, 0]) + + # Quoted and empty fields are not converted. + types = [str] * 3 + calls = [] + self._read_test(['"spam",,42'], [['spam', '', '42']], + quoting=csv.QUOTE_NONNUMERIC, converter=converter) + self.assertEqual(calls, [(2, '42')]) + + # Other quoting modes do not convert at all. + self._read_test(['1,2'], [['1', '2']], + converter=lambda index, field: int(field)) + self._read_test(['1,2'], [['1', '2']], + quoting=csv.QUOTE_ALL, + converter=lambda index, field: int(field)) + + # None means the default conversion. + self._read_test(['1,2'], [[1.0, 2.0]], + quoting=csv.QUOTE_NONNUMERIC, converter=None) + + def test_read_converter_errors(self): + with self.assertRaisesRegex(TypeError, 'must be callable or None'): + csv.reader([], converter='int') + with self.assertRaises(ZeroDivisionError): + self._read_test(['1,2'], [], quoting=csv.QUOTE_NONNUMERIC, + converter=lambda index, field: 1/0) + # A one-argument callable does not fit. + with self.assertRaises(TypeError): + self._read_test(['1,2'], [], quoting=csv.QUOTE_NONNUMERIC, + converter=float) + + def test_write_formatter(self): + def formatter(index, value): + calls.append((index, value)) + return format(value, '.2f') if index == 2 else str(value) + + calls = [] + self._write_test(['a', 1, 0.0, 3.14159], 'a,1,0.00,3.14159', + formatter=formatter) + self.assertEqual(calls, [(1, 1), (2, 0.0), (3, 3.14159)]) + + # Strings and None are not passed to the formatter. + calls = [] + self._write_test([0, 'a', None, 3], '<0>,a,,<3>', + formatter=lambda index, value: + calls.append(value) or f'<{index}>') + self.assertEqual(calls, [0, 3]) + + # Quoting is decided by the original value, not by the result. + self._write_test([1.5, 'a'], '1.50,"a"', quoting=csv.QUOTE_NONNUMERIC, + formatter=lambda index, value: format(value, '.2f')) + + # None means str(). + self._write_test([1, 2], '1,2', formatter=None) + + def test_write_formatter_errors(self): + with self.assertRaisesRegex(TypeError, 'must be callable or None'): + csv.writer(StringIO(), formatter='str') + with self.assertRaisesRegex(csv.Error, 'must return a string'): + self._write_test([1], '', formatter=lambda index, value: index) + self._write_error_test(ZeroDivisionError, [1], + formatter=lambda index, value: 1/0) + # A one-argument callable does not fit. + self._write_error_test(TypeError, [1], formatter=repr) + def test_read_skipinitialspace(self): self._read_test(['no space, space, spaces,\ttab'], [['no space', 'space', 'spaces', '\ttab']], diff --git a/Misc/NEWS.d/next/Library/2026-08-02-21-41-01.gh-issue-155097.bvED9Y.rst b/Misc/NEWS.d/next/Library/2026-08-02-21-41-01.gh-issue-155097.bvED9Y.rst new file mode 100644 index 000000000000000..2af705c1ce4e12f --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-08-02-21-41-01.gh-issue-155097.bvED9Y.rst @@ -0,0 +1,3 @@ +Add the *converter* parameter in :func:`csv.reader` and the *formatter* +parameter in :func:`csv.writer`. They are used instead of :func:`float` and +:func:`str` for converting between fields and values. diff --git a/Modules/_csv.c b/Modules/_csv.c index a7fcc78e058f058..2570735957217dd 100644 --- a/Modules/_csv.c +++ b/Modules/_csv.c @@ -127,6 +127,8 @@ typedef struct { DialectObj *dialect; /* parsing dialect */ + PyObject *converter; /* called to convert an unquoted field, or NULL */ + PyObject *fields; /* field list for current record */ ParserState state; /* current CSV parse state */ Py_UCS4 *field; /* temporary buffer */ @@ -143,6 +145,8 @@ typedef struct { DialectObj *dialect; /* parsing dialect */ + PyObject *formatter; /* called to convert a value to a string, or NULL */ + Py_UCS4 *rec; /* buffer for parser.join */ Py_ssize_t rec_size; /* size of allocated record */ Py_ssize_t rec_len; /* length of record */ @@ -651,6 +655,40 @@ _call_dialect(_csvstate *module_state, PyObject *dialect_inst, PyObject *kwargs) } } +/* Pop the callable *name* out of *kwargs, replacing it with a copy. + *result is set to NULL if it is not given or is None. */ +static int +pop_callable_kwarg(const char *name, PyObject **kwargs, PyObject **result) +{ + PyObject *value; + *result = NULL; + if (*kwargs == NULL) { + return 0; + } + int rc = PyDict_GetItemStringRef(*kwargs, name, &value); + if (rc <= 0) { /* not found or error */ + return rc; + } + if (value == Py_None) { + Py_CLEAR(value); + } + else if (!PyCallable_Check(value)) { + PyErr_Format(PyExc_TypeError, + "\"%s\" must be callable or None, not %T", name, value); + Py_DECREF(value); + return -1; + } + PyObject *copy = PyDict_Copy(*kwargs); + if (copy == NULL || PyDict_DelItemString(copy, name) < 0) { + Py_XDECREF(copy); + Py_XDECREF(value); + return -1; + } + *kwargs = copy; + *result = value; + return 0; +} + /* * READER */ @@ -676,7 +714,14 @@ parse_save_field(ReaderObj *self) self->field_len != 0 && (quoting == QUOTE_NONNUMERIC || quoting == QUOTE_STRINGS)) { - PyObject *tmp = PyNumber_Float(field); + PyObject *tmp; + if (self->converter != NULL) { + tmp = PyObject_CallFunction(self->converter, "nO", + PyList_GET_SIZE(self->fields), field); + } + else { + tmp = PyNumber_Float(field); + } Py_DECREF(field); if (tmp == NULL) { return -1; @@ -1025,6 +1070,7 @@ Reader_traverse(PyObject *op, visitproc visit, void *arg) { ReaderObj *self = _ReaderObj_CAST(op); Py_VISIT(self->dialect); + Py_VISIT(self->converter); Py_VISIT(self->input_iter); Py_VISIT(self->fields); Py_VISIT(Py_TYPE(self)); @@ -1036,6 +1082,7 @@ Reader_clear(PyObject *op) { ReaderObj *self = _ReaderObj_CAST(op); Py_CLEAR(self->dialect); + Py_CLEAR(self->converter); Py_CLEAR(self->input_iter); Py_CLEAR(self->fields); return 0; @@ -1096,6 +1143,7 @@ csv_reader(PyObject *module, PyObject *args, PyObject *keyword_args) return NULL; self->dialect = NULL; + self->converter = NULL; self->fields = NULL; self->input_iter = NULL; self->field = NULL; @@ -1116,8 +1164,15 @@ csv_reader(PyObject *module, PyObject *args, PyObject *keyword_args) Py_DECREF(self); return NULL; } - self->dialect = (DialectObj *)_call_dialect(module_state, dialect, - keyword_args); + PyObject *kwargs = keyword_args; + if (pop_callable_kwarg("converter", &kwargs, &self->converter) < 0) { + Py_DECREF(self); + return NULL; + } + self->dialect = (DialectObj *)_call_dialect(module_state, dialect, kwargs); + if (kwargs != keyword_args) { + Py_DECREF(kwargs); + } if (self->dialect == NULL) { Py_DECREF(self); return NULL; @@ -1344,6 +1399,7 @@ csv_writerow_lock_held(PyObject *op, PyObject *seq) /* Join all fields in internal buffer. */ join_reset(self); + Py_ssize_t field_index = 0; while ((field = PyIter_Next(iter))) { int append_ok; int quoted; @@ -1378,7 +1434,18 @@ csv_writerow_lock_held(PyObject *op, PyObject *seq) else { PyObject *str; - str = PyObject_Str(field); + if (self->formatter != NULL) { + str = PyObject_CallFunction(self->formatter, "nO", + field_index, field); + if (str != NULL && !PyUnicode_Check(str)) { + PyErr_Format(self->error_obj, + "formatter must return a string, not %T", str); + Py_CLEAR(str); + } + } + else { + str = PyObject_Str(field); + } Py_DECREF(field); if (str == NULL) { Py_DECREF(iter); @@ -1391,6 +1458,7 @@ csv_writerow_lock_held(PyObject *op, PyObject *seq) Py_DECREF(iter); return NULL; } + field_index++; } Py_DECREF(iter); if (PyErr_Occurred()) @@ -1496,6 +1564,7 @@ Writer_traverse(PyObject *op, visitproc visit, void *arg) { WriterObj *self = _WriterObj_CAST(op); Py_VISIT(self->dialect); + Py_VISIT(self->formatter); Py_VISIT(self->write); Py_VISIT(self->error_obj); Py_VISIT(Py_TYPE(self)); @@ -1507,6 +1576,7 @@ Writer_clear(PyObject *op) { WriterObj *self = _WriterObj_CAST(op); Py_CLEAR(self->dialect); + Py_CLEAR(self->formatter); Py_CLEAR(self->write); Py_CLEAR(self->error_obj); return 0; @@ -1564,6 +1634,7 @@ csv_writer(PyObject *module, PyObject *args, PyObject *keyword_args) self->dialect = NULL; self->write = NULL; + self->formatter = NULL; self->rec = NULL; self->rec_size = 0; @@ -1588,8 +1659,15 @@ csv_writer(PyObject *module, PyObject *args, PyObject *keyword_args) Py_DECREF(self); return NULL; } - self->dialect = (DialectObj *)_call_dialect(module_state, dialect, - keyword_args); + PyObject *kwargs = keyword_args; + if (pop_callable_kwarg("formatter", &kwargs, &self->formatter) < 0) { + Py_DECREF(self); + return NULL; + } + self->dialect = (DialectObj *)_call_dialect(module_state, dialect, kwargs); + if (kwargs != keyword_args) { + Py_DECREF(kwargs); + } if (self->dialect == NULL) { Py_DECREF(self); return NULL;