diff --git a/AUTHORS b/AUTHORS index cd73e657..2fceac0e 100644 --- a/AUTHORS +++ b/AUTHORS @@ -30,6 +30,7 @@ Here is a list of past and present much-appreciated contributors: Mathias Loesch Matthew Hegarty Matthias Dellweg + mika (mikamikasuki) Mike Waldner Nicholas Pilon Peyman Salehi diff --git a/HISTORY.md b/HISTORY.md index 77f3b6b3..abe709a9 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -2,6 +2,11 @@ ## 3.9.0 and newer +### Bugfixes + +- Keep generated values in their intended columns after inserting columns into + a dataset with dynamic columns. + See GitHub Releases: - https://github.com/jazzband/tablib/releases diff --git a/docs/tutorial.rst b/docs/tutorial.rst index e4c2ae6f..e5283069 100644 --- a/docs/tutorial.rst +++ b/docs/tutorial.rst @@ -281,6 +281,16 @@ either provide all values in the row, or only the non-dynamic values and then the dynamic values will be automatically generated using the function initially provided for the column calculation. +Callable columns can also be inserted at a chosen position with +:meth:`Dataset.insert_col`. Generated values retain that position when adding +rows containing only the non-dynamic values. For example:: + + >>> names = tablib.Dataset(['ada'], headers=['name']) + >>> names.insert_col(0, lambda row: row[0].upper(), header='uppercase') + >>> names.append(['grace']) + >>> names[-1] + ('GRACE', 'grace') + ..versionchanged:: 3.6.0 In older versions, you could only add new rows with fully-populated rows, diff --git a/src/tablib/core.py b/src/tablib/core.py index b0b56fd2..90781f61 100644 --- a/src/tablib/core.py +++ b/src/tablib/core.py @@ -11,11 +11,14 @@ __lazy_modules__ = { "copy", "operator", + "sys", "tablib.exceptions", "tablib.utils", } +import sys from copy import copy +from operator import index as operator_index from operator import itemgetter from .exceptions import ( @@ -548,9 +551,14 @@ def insert_col(self, index, col=None, header=None): if col is None: col = [] + index = operator_index(index) + if index > sys.maxsize or index < -sys.maxsize - 1: + raise OverflowError("Python int too large to convert to C ssize_t") + index = max(0, min(self.width, index if index >= 0 else self.width + index)) + # Callable Columns... - if callable(col): - self._dynamic_columns[self.width] = col + dynamic_col = col if callable(col) else None + if dynamic_col is not None: col = list(map(col, self._data)) col = self._clean_col(col) @@ -576,6 +584,14 @@ def insert_col(self, index, col=None, header=None): else: self._data = [Row([row]) for row in col] + dynamic_columns = { + pos + 1 if pos >= index else pos: func + for pos, func in self._dynamic_columns.items() + } + if dynamic_col is not None: + dynamic_columns[index] = dynamic_col + self._dynamic_columns = dict(sorted(dynamic_columns.items())) + def rpush_col(self, col, header=None): """Adds a column to the end of the :class:`Dataset`. See :method:`Dataset.insert` for additional documentation. diff --git a/tests/test_tablib.py b/tests/test_tablib.py index ca8ac05e..be2d80dc 100755 --- a/tests/test_tablib.py +++ b/tests/test_tablib.py @@ -6,6 +6,7 @@ import json import pickle import re +import sys import tempfile import unittest from decimal import Decimal @@ -229,6 +230,68 @@ def initials(row): ['JA', 'GW', 'TJ', 'SO', 'OS', 'AD'] ) + def test_insert_callable_column_positions(self): + for headers in (None, ['name', 'city']): + for index in (0, 1, -1, -3, 8): + with self.subTest(headers=headers, index=index): + dataset = tablib.Dataset(['Ada', 'London'], headers=headers) + dataset.insert_col(index, lambda row: row[0].upper(), header='uppercase') + dataset.append(['Grace', 'New York']) + expected = ['Grace', 'New York'] + expected.insert(index, 'GRACE') + self.assertEqual(dataset[-1], tuple(expected)) + if headers: + self.assertEqual(dataset['uppercase'], ['ADA', 'GRACE']) + + def test_insert_column_index_bounds(self): + class Index: + def __index__(self): + return -1 + + for index in (sys.maxsize, -sys.maxsize - 1, False, True, Index()): + with self.subTest(index=index): + dataset = tablib.Dataset(['ada'], headers=['name']) + dataset.insert_col(index, lambda row: row[0].upper(), header='uppercase') + dataset.append(['grace']) + expected = ['grace'] + expected.insert(index, 'GRACE') + self.assertEqual(dataset[-1], tuple(expected)) + + for index in (sys.maxsize + 1, -sys.maxsize - 2): + with self.subTest(index=index): + dataset = tablib.Dataset(['ada'], headers=['name']) + dataset.append_col(lambda row: row[0].upper(), header='uppercase') + with self.assertRaises(OverflowError): + dataset.insert_col(index, ['active'], header='status') + self.assertEqual(dataset.headers, ['name', 'uppercase']) + self.assertEqual(dataset[0], ('ada', 'ADA')) + dataset.append(['grace']) + self.assertEqual(dataset[-1], ('grace', 'GRACE')) + + def test_insert_columns_before_dynamic_column(self): + dataset = tablib.Dataset(['ada'], headers=['name']) + dataset.append_col(lambda row: row[-1].upper(), header='uppercase') + dataset.insert_col(0, ['active'], header='status') + dataset.append(['pending', 'grace']) + self.assertEqual(dataset[-1], ('pending', 'grace', 'GRACE')) + + def test_insert_callable_column_without_rows(self): + for index in (0, 8): + with self.subTest(index=index): + dataset = tablib.Dataset(headers=['name']) + dataset.insert_col(index, lambda row: row[0].upper(), header='uppercase') + dataset.append(['grace']) + expected = ['grace'] + expected.insert(index, 'GRACE') + self.assertEqual(dataset[0], tuple(expected)) + + def test_insert_multiple_callable_columns(self): + dataset = tablib.Dataset(['ada', 'London'], headers=['name', 'city']) + dataset.insert_col(1, lambda row: row[0].upper(), header='uppercase') + dataset.insert_col(0, lambda row: row[0].title(), header='display_name') + dataset.append(['linus', 'Helsinki']) + self.assertEqual(dataset[-1], ('Linus', 'linus', 'LINUS', 'Helsinki')) + def test_header_slicing(self): """Verify slicing by headers."""