Testing code for An+B microsyntax parsing library

Question

I have a small package that provides some API for playing around with the <An+B> CSS microsyntax. It is 100% type hinted and tested, but doesn't actually have any real/useful features and this question is not about the main code anyway.

I was introduced to the Hypothesis testing library in a previous question and have taken a liking to it, so this package is a way for me to practice using it. Therefore, I'm asking for a review of my testing code. Only test files are presented here; other files can be found at the package's repository.

Here's the project structure (irrelevant files removed):

.
├── pyproject.toml
├── src
│ └── a_n_plus_b
│ ├── __init__.py
│ ├── _grammar.py
│ └── py.typed
├── tests
│ ├── __init__.py
│ ├── test_alternate_constructors.py
│ ├── test_indices.py
│ └── test_other_methods.py
└── tox.ini

tests/__init__.py

from collections.abc import Callable, Iterable
from typing import Any, ParamSpec, TypeVar
from _pytest.mark import ParameterSet
from hypothesis import example
from hypothesis.strategies import composite, DrawFn, integers, SearchStrategy
from a_n_plus_b import ANPlusB
_T = TypeVar('_T')
_P = ParamSpec('_P')
_Decorator = Callable[[Callable[_P, _T]], Callable[_P, _T]]
newlines = ['\r\n', '\r', '\n', '\f']
whitespace = ['\t', '\x20'] + newlines
blank = [''] + whitespace
def join(whatever: Iterable[Any]) -> str:
 return ''.join(map(str, whatever))
# Originally from https://stackoverflow.com/a/70312417
def examples(
 parameter_sets: Iterable[ParameterSet | tuple[Any, ...] | Any]
) -> _Decorator[_P, _T]:
 parameter_sets = list(parameter_sets)
 
 def inner(test_case: Callable[_P, _T]) -> Callable[_P, _T]:
 for parameter_set in reversed(parameter_sets):
 if isinstance(parameter_set, ParameterSet):
 parameter_set = parameter_set.values
 
 if not isinstance(parameter_set, tuple):
 parameter_set = tuple([parameter_set])
 
 test_case = example(*parameter_set)(test_case)
 
 return test_case
 
 return inner
@composite
def a_n_plus_b_instances(
 draw: DrawFn,
 steps: SearchStrategy[int] = integers(),
 offsets: SearchStrategy[int] = integers()
) -> SearchStrategy[ANPlusB]:
 return ANPlusB(draw(steps), draw(offsets))

tests/test_alternate_constructors.py

from collections.abc import Callable
from typing import Any, TypeVar
import pytest
from hypothesis import given
from hypothesis.strategies import (
 composite, DrawFn, floats, integers, just, lists,
 one_of, sampled_from, SearchStrategy, tuples
)
from a_n_plus_b import (
 ANPlusB, ComplexWithNonIntegerPart,
 EmptyInput, InputIsNotParsable
)
from . import examples, join, whitespace
_E = TypeVar('_E')
_T = TypeVar('_T')
def _make_complex(example: tuple[int | float, int | float]) -> complex:
 return complex(*example)
def _starts_with_sign(example: tuple[str, Any]) -> bool:
 text, _ = example
 
 return text[0] in ('+', '-')
def _casing_scrambled(text: str) -> SearchStrategy[str]:
 lowercase = text.lower()
 uppercase = text.upper()
 
 substrategies = (sampled_from(chars) for chars in zip(lowercase, uppercase))
 
 return tuples(*substrategies).map(''.join)
@composite
def _variations(draw: DrawFn, text: str) -> str:
 scrambled_text = draw(_casing_scrambled(text))
 
 return draw(with_surrounding_whitespace(scrambled_text))
def _tupled_with(value: _T) -> Callable[[_E], tuple[_E, _T]]:
 def tupled_with_given_value(example: _E) -> tuple[_E, _T]:
 return example, value
 
 return tupled_with_given_value
def _whitespace_sequences() -> SearchStrategy[str]:
 return lists(
 sampled_from(whitespace),
 min_size = 1, max_size = 10
 ).map(''.join)
def whitespace_sequences_or_empty() -> SearchStrategy[str]:
 return _whitespace_sequences() | just('')
def with_surrounding_whitespace(text: str) -> SearchStrategy[str]:
 return tuples(
 whitespace_sequences_or_empty(),
 just(text),
 whitespace_sequences_or_empty()
 ) \
 .map(join)
def _non_integral_floats() -> SearchStrategy[float]:
 return floats().filter(lambda example: not example.is_integer())
def _operators() -> SearchStrategy[str]:
 return sampled_from(['+', '-'])
def _signs() -> SearchStrategy[str]:
 return just('') | _operators()
def _digits() -> SearchStrategy[int]:
 return integers(min_value = 0, max_value = 9)
@composite
def _integers_with_potentially_superfluous_sign(draw: DrawFn) -> str:
 sign = draw(_signs())
 digits = draw(lists(_digits(), min_size = 1, max_size = 10).map(join))
 
 return draw(with_surrounding_whitespace(sign + digits))
@composite
def _a_values(draw: DrawFn) -> str:
 sign = draw(_signs())
 digits = draw(lists(_digits(), max_size = 10).map(join))
 
 return sign + digits
def _b_values() -> SearchStrategy[str]:
 return lists(_digits(), min_size = 1, max_size = 10).map(join)
class ParseANPlusBTestCases:
 
 @staticmethod
 @composite
 def valid(draw: DrawFn) -> tuple[str, tuple[int, int]]:
 a = draw(_a_values())
 n = draw(sampled_from(['n', 'N']))
 operator = draw(_signs())
 
 if not a:
 step = 0
 elif a == '+':
 step = 1
 elif a == '-':
 step = -1
 else:
 step = int(a)
 
 if operator:
 b = draw(_b_values())
 offset = int(f'{operator}{b}')
 else:
 b = ''
 offset = 0
 
 operator = draw(with_surrounding_whitespace(operator))
 text = draw(with_surrounding_whitespace(f'{a}{n}{operator}{b}'))
 
 return text, (step, offset)
 
 @staticmethod
 @composite
 def whitespace_after_a_sign(draw: DrawFn) -> str:
 valid_cases = ParseANPlusBTestCases.valid().filter(_starts_with_sign)
 valid_input = draw(valid_cases)[0]
 
 a_sign, rest = valid_input[0], valid_input[1:]
 invalid_whitespace = draw(_whitespace_sequences())
 
 return f'{a_sign}{invalid_whitespace}{rest}'
 
 @staticmethod
 @composite
 def missing_b(draw: DrawFn) -> str:
 a = draw(_a_values())
 n = draw(sampled_from(['n', 'N']))
 operator = draw(_operators())
 
 operator = draw(with_surrounding_whitespace(operator))
 
 return draw(with_surrounding_whitespace(f'{a}{n}{operator}'))
 
 @staticmethod
 @composite
 def missing_operator(draw: DrawFn) -> str:
 a = draw(_a_values())
 n = draw(sampled_from(['n', 'N']))
 b = draw(_b_values())
 
 b = draw(with_surrounding_whitespace(b))
 
 return draw(with_surrounding_whitespace(f'{a}{n}{b}'))
@given(
 one_of([
 _variations('odd').map(_tupled_with((2, 1))),
 _variations('even').map(_tupled_with((2, 0)))
 ])
)
def test_parse_odd_even(text_and_expected):
 text, expected = text_and_expected
 instance = ANPlusB.parse(text)
 
 assert (instance.step, instance.offset) == expected
@given(_integers_with_potentially_superfluous_sign())
def test_parse_integer(text):
 expected = int(text)
 
 instance = ANPlusB.parse(text)
 
 assert instance.step == 0
 assert instance.offset == expected
@given(whitespace_sequences_or_empty())
def test_parse_empty(text):
 with pytest.raises(EmptyInput):
 ANPlusB.parse(text)
@given(ParseANPlusBTestCases.valid())
@examples([
 (['+3n+2', (3, 2)]),
 (['+4n+0', (4, 0)]),
 (['+6n', (6, 0)]),
 (['+5n-0', (5, 0)]),
 (['+7n-1', (7, -1)]),
 
 (['3n+2', (3, 2)]),
 (['4n+0', (4, 0)]),
 (['6n', (6, 0)]),
 (['5n-0', (5, 0)]),
 (['7n-1', (7, -1)]),
 
 (['+0n+2', (0, 2)]),
 (['+0n+0', (0, 0)]),
 (['+0n', (0, 0)]),
 (['+0n-0', (0, 0)]),
 (['+0n-1', (0, -1)]),
 
 (['0n+2', (0, 2)]),
 (['0n+0', (0, 0)]),
 (['0n', (0, 0)]),
 (['0n-0', (0, 0)]),
 (['0n-1', (0, -1)]),
 
 (['-0n+2', (0, 2)]),
 (['-0n+0', (0, 0)]),
 (['-0n', (0, 0)]),
 (['-0n-0', (0, 0)]),
 (['-0n-1', (0, -1)]),
 
 (['-3n+2', (-3, 2)]),
 (['-4n+0', (-4, 0)]),
 (['-6n', (-6, 0)]),
 (['-5n-0', (-5, 0)]),
 (['-7n-1', (-7, -1)]),
])
def test_parse_a_n_plus_b(text_and_expected):
 text, expected = text_and_expected
 instance = ANPlusB.parse(text)
 
 assert (instance.step, instance.offset) == expected
@given(
 one_of([
 ParseANPlusBTestCases.whitespace_after_a_sign(),
 ParseANPlusBTestCases.missing_b(),
 ParseANPlusBTestCases.missing_operator()
 ])
)
def test_parse_invalid(text):
 with pytest.raises(InputIsNotParsable):
 ANPlusB.parse(text)
@given(
 one_of([
 tuples(integers(), integers()).map(_make_complex),
 integers(),
 integers().map(float)
 ])
)
def test_from_complex(value):
 instance = ANPlusB.from_complex(value)
 expected = (int(value.imag), int(value.real))
 
 assert (instance.step, instance.offset) == expected
@given(
 one_of([
 tuples(_non_integral_floats(), integers()),
 tuples(integers(), _non_integral_floats()),
 tuples(_non_integral_floats(), _non_integral_floats())
 ]) \
 .map(_make_complex)
)
def test_from_complex_invalid(value):
 with pytest.raises(ComplexWithNonIntegerPart):
 ANPlusB.from_complex(value)

tests/test_indices.py

from collections.abc import Iterable
from itertools import product
from typing import cast, Literal
import pytest
from _pytest.mark import ParameterSet
from hypothesis import assume, given
from hypothesis.strategies import (
 booleans, composite, DrawFn, from_type, integers, just,
 one_of, sampled_from, SearchStrategy, tuples
)
from a_n_plus_b import ANPlusB, InvalidNumberOfChildren, InvalidOrder
from . import a_n_plus_b_instances, examples
_Order = Literal['ascending', 'descending', 'default']
_TestCase = tuple[ANPlusB, tuple[int, bool, _Order], Iterable[int]]
def _ascending(values: Iterable[int]) -> Iterable[int]:
 return sorted(values)
def _descending(indices: Iterable[int]) -> Iterable[int]:
 return sorted(indices, reverse = True)
def _from_last(indices: Iterable[int], population: int) -> Iterable[int]:
 return [population - index + 1 for index in indices]
def _sign(value: int, /) -> Literal['negative', 'positive', 'zero']:
 return 'negative' if value < 0 else 'positive' if value > 0 else 'zero'
def _describe(test_case: _TestCase) -> str:
 instance, (population, from_last, order), _ = test_case
 step, offset = instance.step, instance.offset
 
 descriptions = [
 f'{_sign(step)} step',
 f'{_sign(offset)} offset',
 f'from last' if from_last else 'from first',
 f'{order}'
 ]
 
 return ', '.join(descriptions)
def _human_integers(
 min_value: int = -(2 ** 16),
 max_value: int = 2 ** 16,
) -> SearchStrategy[int]:
 return integers(
 min_value = max(-(2 ** 16), min_value),
 max_value = min(2 ** 16, max_value)
 )
def _orders() -> SearchStrategy[_Order]:
 return cast(
 SearchStrategy[_Order],
 sampled_from(['ascending', 'descending', 'default'])
 )
def _non_positive_step_and_offset_test_cases() -> SearchStrategy[_TestCase]:
 return tuples(
 a_n_plus_b_instances(
 integers(max_value = 0),
 integers(max_value = 0)
 ),
 tuples(integers(min_value = 0), booleans(), _orders()),
 just(list[int]())
 )
@composite
def _zero_step_test_cases(draw: DrawFn) -> _TestCase:
 argument_sets = tuples(integers(min_value = 0), booleans(), _orders())
 
 instance = draw(a_n_plus_b_instances(just(0), integers()))
 population, from_last, order = draw(argument_sets)
 
 b = instance.offset
 
 index = population - b + 1 if from_last else b
 expected = [index] if 1 <= b <= population else list[int]()
 
 return instance, (population, from_last, order), expected
@composite
def _non_positive_step_zero_offset_test_cases(draw: DrawFn) -> _TestCase:
 argument_sets = tuples(integers(min_value = 0), booleans(), _orders())
 
 instance = draw(a_n_plus_b_instances(integers(max_value = -1), just(0)))
 arguments = draw(argument_sets)
 
 return instance, arguments, list[int]()
def _test_case_group(
 instance: ANPlusB,
 population: int,
 base_case_expected: Iterable[int]
) -> list[ParameterSet]:
 test_cases = []
 from_last_and_order_matrix = product(
 [False, True],
 ['default', 'ascending', 'descending']
 )
 
 for from_last, order in from_last_and_order_matrix:
 expected = base_case_expected
 
 if from_last:
 expected = _from_last(base_case_expected, population)
 
 if order == 'ascending':
 expected = _ascending(expected)
 elif order == 'descending':
 expected = _descending(expected)
 
 test_case = (instance, (population, from_last, order), expected)
 test_cases.append(
 pytest.param(test_case, id = _describe(test_case))
 )
 
 return test_cases
@given(
 one_of([
 _non_positive_step_and_offset_test_cases(),
 _zero_step_test_cases(),
 _non_positive_step_zero_offset_test_cases()
 ])
)
@examples([
 *_test_case_group(ANPlusB(3, 0), 100, list(range(3, 101, 3))),
 
 *_test_case_group(ANPlusB(-2, 6), 10, [6, 4, 2]),
 *_test_case_group(ANPlusB(-1, 4), 8, [4, 3, 2, 1]),
 *_test_case_group(ANPlusB(-3, 8), 18, [8, 5, 2]),
 
 *_test_case_group(ANPlusB(4, -5), 20, [3, 7, 11, 15, 19]),
 *_test_case_group(ANPlusB(5, -2), 12, [3, 8]),
 
 *_test_case_group(ANPlusB(2, 1), 15, [1, 3, 5, 7, 9, 11, 13, 15]),
 *_test_case_group(ANPlusB(3, 1), 10, [1, 4, 7, 10]),
 *_test_case_group(ANPlusB(1, 4), 11, [4, 5, 6, 7, 8, 9, 10, 11])
])
def test_indices(instance_arguments_expected: _TestCase):
 instance, arguments, expected = instance_arguments_expected
 population, from_last, order = arguments
 
 indices = instance.indices(population, from_last = from_last, order = order)
 
 assert list(indices) == expected
@given(
 a_n_plus_b_instances(),
 integers(), booleans(), from_type(str)
)
def test_indices_invalid_order(instance, population, from_last, order):
 assume(order not in ('ascending', 'descending', 'default'))
 
 with pytest.raises(InvalidOrder):
 instance.indices(population, from_last = from_last, order = order)
@given(
 a_n_plus_b_instances(),
 integers(max_value = -1), booleans(), _orders()
)
def test_indices_invalid_number_of_children(
 instance, population, from_last, order
):
 with pytest.raises(InvalidNumberOfChildren):
 instance.indices(population, from_last = from_last, order = order)

tests/test_other_methods.py

import pytest
from hypothesis import given, infer
from hypothesis.strategies import from_type, integers
from a_n_plus_b import ANPlusB, IncorrectUseOfConstructor
from . import a_n_plus_b_instances
_EqTestCase = tuple[ANPlusB, '_ANPlusBSubclass', bool]
class _ANPlusBSubclass(ANPlusB):
 pass
@given(integers(), integers())
def test_construction(step, offset):
 instance = ANPlusB(step, offset)
 
 assert instance.step == step
 assert instance.offset == offset
@given(integers())
def test_construction_single_argument(offset):
 instance = ANPlusB(offset)
 
 assert instance.step == 0
 assert instance.offset == offset
@given(infer)
def test_construction_invalid(text: str):
 with pytest.raises(IncorrectUseOfConstructor):
 ANPlusB(text) # noqa
@pytest.mark.parametrize(('instance', 'expected'), [
 (ANPlusB(0, -2), '-2'),
 (ANPlusB(0, 0), '0'),
 (ANPlusB(0, 2), '2'),
 
 (ANPlusB(1, -3), 'n-3'),
 (ANPlusB(1, 0), 'n'),
 (ANPlusB(1, 3), 'n+3'),
 
 (ANPlusB(-1, -4), '-n-4'),
 (ANPlusB(-1, 0), '-n'),
 (ANPlusB(-1, 4), '-n+4'),
 
 (ANPlusB(3, 4), '3n+4'),
 (ANPlusB(3, 0), '3n'),
 (ANPlusB(3, -5), '3n-5'),
 
 (ANPlusB(-4, 5), '-4n+5'),
 (ANPlusB(-4, 0), '-4n'),
 (ANPlusB(-4, -6), '-4n-6')
])
def test_str(instance, expected):
 assert str(instance) == expected
 assert repr(instance) == f'{ANPlusB.__name__}({expected})'
@given(a_n_plus_b_instances())
def test_values(instance):
 a, b = instance.step, instance.offset
 
 for index, value in zip(range(10), instance.values()):
 assert value == a * index + b
@given(a_n_plus_b_instances(), integers(min_value = 0))
def test_values_contain_getitem(instance, index):
 a, b = instance.step, instance.offset
 value = instance.values()[index]
 
 assert value == a * index + b
 assert value in instance.values()
@given(
 a_n_plus_b_instances(),
 from_type(object).filter(lambda o: not isinstance(o, int))
)
def test_values_not_contain(instance, item):
 assert item not in instance.values()
@given(a_n_plus_b_instances(), integers(max_value = -1))
def test_getitem_invalid(instance, index):
 with pytest.raises(IndexError):
 _ = instance.values()[index]
@given(a_n_plus_b_instances())
def test_eq(this):
 a, b = this.step, this.offset
 that = ANPlusB(a, b)
 
 assert this == that
 assert hash(this) == hash(that) == hash((a, b))
@pytest.mark.parametrize(('this', 'that', 'expected'), [
 (ANPlusB(-2, -3), ANPlusB(-2, -3), True),
 (ANPlusB(-1, 0), ANPlusB(-1, 0), True),
 (ANPlusB(0, -5), ANPlusB(0, -5), True),
 (_ANPlusBSubclass(-4, -3), _ANPlusBSubclass(-4, -3), True),
 (_ANPlusBSubclass(-1, -2), _ANPlusBSubclass(-1, -2), True),
 (_ANPlusBSubclass(0, 0), _ANPlusBSubclass(0, 0), True),
 (ANPlusB(0, 0), _ANPlusBSubclass(0, 0), True),
 (_ANPlusBSubclass(0, 0), ANPlusB(0, 0), True),
 (ANPlusB(3, 4), ANPlusB(4, 3), False),
 (_ANPlusBSubclass(3, 4), _ANPlusBSubclass(4, 3), False),
 (ANPlusB(-2, -3), ANPlusB(-4, -5), False),
 (_ANPlusBSubclass(-1, -2), _ANPlusBSubclass(-3, -4), False),
])
def test_eq_subclass(this, that, expected):
 assert (this == that) is expected

Answer 1

Kudos for embarking on this adventure! Stochastic testing can certainly reveal surprising things about the target code.

VT

No idea what your requirements are. But apparently they mention form-feed and horizontal-tab, and are well aligned with isspace(). That is, string.whitespace seems relevant.

newlines = ['\r\n', '\r', '\n', '\f']
whitespace = ['\t', '\x20'] + newlines

Vertical-tab \v is missing. Just sayin'.

And thank goodness the requirements are simple enough that we needn't worry about decimal 160 NBSP and its ilk, such as ZWSP. (Which answers the question: When is a space not a space?)

numeric tower

def _make_complex(example: tuple[int | float, int | float]) -> complex:
 return complex(*example)

Well, we needn't make the signature quite so complex.

The PEP explains that

when an argument is annotated as having type float, an argument of type int is acceptable

That is, int is subsumed by float, despite the lack of an inheritance relationship for these primitives, and the fact that python ints can certainly represent quantities of far greater magnitude than an IEEE double can.

This gives us the more readable:

def _make_complex(example: tuple[float, float]) -> complex:
 ...

edge cases

I like _casing_scrambled. Given a long string, it can certainly generate a great many possibilities. It would take an exponentially long time to generate all of them.

I wonder if there is value in offering a variant which simply downcases the "middle" bulk of the input string, then picks a random index and length to upcase. The idea would be to more rapidly explore the space in order to provoke target code bugs. I am thinking of the several implementations I've encountered for converting between snake_case & camelCase, and the subtle bugs they exhibit near non-alphabetics like _ underscore or digits.

magic number

def _whitespace_sequences() -> ... :
 return ...
 min_size = 1, max_size = 10 ...

Consider promoting 10 up into a keyword default signature parameter.

Or make it a global? I see at least three subsequent uses of it, e.g. in _b_values. Maybe we want to admit of 33-bit numbers greater than four billion?

lines of code

I agree with each function individually, they're doing the right thing.

Overall, it seems like a lot of verbiage, though.

For example, when I look at the test_parse_a_n_plus_b @examples I like and agree with them. But I can't help but think a regex would have solved 90% of extracting expected from each input string.

The subsequent @given @one_of's really do validate the power of the technique, concisely describing edge cases to be handled.

positional only

def _sign(value: int, /) -> ...

Maybe delete the / slash? It's not obvious to me why it's needed.

---

This is an impressive effort.

Maybe it was worth it? Or maybe it ran into diminishing returns, compared to the effort of how a less exhaustive approach would still manage to exercise an interesting fraction of the target code's state space.

I would love to see you write up "mistakes uncovered", that is, what the test code taught you about evolving target code. And if the testing approach suggested API changes that improve accessibility / testability of more of the state space.

• Toby Speight: I second your final paragraph - it would be very valuable (and perhaps motivational) to learn how this approach was helpful to the asker.
• InSync: Regarding the whitespace requirements, I didn't make that up. It's defined by the specs.
• J_H: @InSync, excellent, thank you for the URL citation. Feel free to add a one-line citation to the source code, so future maintainers will know what it's intending to implement.