Property-Based Testing: Let Your Computer Find Bugs You Can't Imagine

1
def test_sort_examples():
2
    assert sort([3, 1, 2]) == [1, 2, 3]
3
    assert sort([]) == []
4
    assert sort([1]) == [1]
5
    assert sort([2, 2, 1]) == [1, 2, 2]

1
from hypothesis import given, strategies as st
2

3
@given(st.lists(st.integers()))
4
def test_sort_properties(lst):
5
    sorted_list = sort(lst)
6
    
7
    # Property 1: Output length equals input length
8
    assert len(sorted_list) == len(lst)
9
    
10
    # Property 2: Output is ordered
11
    for i in range(len(sorted_list) - 1):
12
        assert sorted_list[i] <= sorted_list[i + 1]
13
    
14
    # Property 3: Output contains same elements as input
15
    assert sorted(lst) == sorted_list

1
def reverse_string(s: str) -> str:
2
    """Reverse a string."""
3
    return s[::-1]
4

5
# Traditional test
6
def test_reverse_examples():
7
    assert reverse_string("hello") == "olleh"
8
    assert reverse_string("") == ""
9
    assert reverse_string("a") == "a"
10

11
# Property-based test
12
from hypothesis import given
13
from hypothesis import strategies as st
14

15
@given(st.text())
16
def test_reverse_properties(s):
17
    reversed_s = reverse_string(s)
18
    
19
    # Property: Reversing twice gives original
20
    assert reverse_string(reversed_s) == s
21
    
22
    # Property: Length is preserved
23
    assert len(reversed_s) == len(s)
24
    
25
    # Property: First char becomes last (if non-empty)
26
    if s:
27
        assert reversed_s[-1] == s[0]
28
        assert reversed_s[0] == s[-1]

1
@given(st.dictionaries(st.text(), st.integers()))
2
def test_cache_size_invariant(initial_data):
3
    cache = LRUCache(capacity=100)
4
    
5
    for key, value in initial_data.items():
6
        cache.put(key, value)
7
        # Invariant: size never exceeds capacity
8
        assert len(cache) <= 100

1
@given(st.text())
2
def test_json_roundtrip(data):
3
    # Skip if the string contains invalid JSON characters
4
    try:
5
        json_str = json.dumps(data)
6
        assert json.loads(json_str) == data
7
    except (UnicodeDecodeError, UnicodeEncodeError):
8
        # Some strings can't be JSON encoded
9
        pass
10

11
@given(st.binary())
12
def test_compression_roundtrip(data):
13
    compressed = zlib.compress(data)
14
    decompressed = zlib.decompress(compressed)
15
    assert decompressed == data

1
@given(st.lists(st.floats(allow_nan=False, allow_infinity=False)))
2
def test_average_scaling(numbers):
3
    if not numbers:
4
        return
5
    
6
    avg1 = average(numbers)
7
    scaled = [x * 2 for x in numbers]
8
    avg2 = average(scaled)
9
    
10
    # Property: Scaling all inputs scales the average
11
    assert abs(avg2 - (avg1 * 2)) < 0.0001

1
@given(st.lists(st.integers()))
2
def test_custom_sort_matches_builtin(lst):
3
    custom_sorted = my_custom_sort(lst.copy())
4
    builtin_sorted = sorted(lst)
5
    assert custom_sorted == builtin_sorted

1
from hypothesis import strategies as st
2
from datetime import datetime
3

4
# Basic types
5
integers = st.integers(min_value=0, max_value=100)
6
floats = st.floats(allow_nan=False, allow_infinity=False)
7
text = st.text(alphabet="abcdefghijklmnopqrstuvwxyz", min_size=1)
8

9
# Collections
10
lists_of_ints = st.lists(st.integers(), min_size=1, max_size=10)
11
dict_str_to_int = st.dictionaries(st.text(), st.integers())
12

13
# Complex objects
14
@st.composite
15
def user_profiles(draw):
16
    # The `draw` function is the magic of composite strategies.
17
    # It takes a strategy and "draws" a single value from it,
18
    # allowing you to combine multiple strategies into one complex object.
19
    return {
20
        "username": draw(st.text(min_size=3, max_size=20)),
21
        "age": draw(st.integers(min_value=13, max_value=120)),
22
        "email": draw(st.emails()),
23
        "joined": draw(st.datetimes(
24
            min_value=datetime(2020, 1, 1),
25
            max_value=datetime(2025, 1, 1)
26
        )),
27
        "premium": draw(st.booleans())
28
    }
29

30
@given(user_profiles())
31
def test_user_serialization(user):
32
    serialized = serialize_user(user)
33
    deserialized = deserialize_user(serialized)
34
    assert deserialized == user

1
def find_median(numbers):
2
    """Find the median of a list of numbers."""
3
    if not numbers:
4
        raise ValueError("Cannot find median of empty list")
5
    
6
    sorted_nums = sorted(numbers)
7
    n = len(sorted_nums)
8
    
9
    if n % 2 == 1:
10
        return sorted_nums[n // 2]
11
    else:
12
        # Bug: integer division when we need float division
13
        return (sorted_nums[n // 2 - 1] + sorted_nums[n // 2]) // 2
14

15
# Property-based test
16
@given(st.lists(st.integers(), min_size=1))
17
def test_median_properties(numbers):
18
    median = find_median(numbers)
19
    
20
    # Property 1: Median is within the range
21
    assert min(numbers) <= median <= max(numbers)
22
    
23
    # Property 2: At least half elements are >= median
24
    greater_equal = sum(1 for n in numbers if n >= median)
25
    assert greater_equal >= len(numbers) // 2
26
    
27
    # Property 3: At least half elements are <= median
28
    less_equal = sum(1 for n in numbers if n <= median)
29
    assert less_equal >= len(numbers) // 2

1
def remove_duplicates(items):
2
    """Remove duplicates while preserving order."""
3
    seen = set()
4
    result = []
5
    for item in items:
6
        if item not in seen:
7
            seen.add(item)
8
            result.append(item)
9
    # Bug: returning the set of seen items, which is unordered
10
    return seen
11

12
@given(st.lists(st.integers()))
13
def test_remove_duplicates_properties(items):
14
    result = remove_duplicates(items)
15
    
16
    # Property 1: All items in the result are unique
17
    assert len(result) == len(set(result))
18
    
19
    # Property 2: The result contains only items from the original list
20
    assert set(result).issubset(set(items))
21
    
22
    # Property 3 (the one that fails): Order is preserved
23
    # We can build the expected list and compare
24
    expected = []
25
    seen = set()
26
    for item in items:
27
        if item not in seen:
28
            seen.add(item)
29
            expected.append(item)
30
    
31
    # This assertion will fail because `result` is an unordered set
32
    assert list(result) == expected

1
# conftest.py
2
from hypothesis import settings
3

4
# Configure Hypothesis for CI
5
settings.register_profile("ci", max_examples=1000)
6
settings.register_profile("dev", max_examples=100)
7
settings.register_profile("debug", max_examples=10, verbosity=Verbosity.verbose)
8

9
# Run with: pytest --hypothesis-profile=ci

1
class TestUserRegistration:
2
    # Traditional edge case tests
3
    def test_empty_username_rejected(self):
4
        with pytest.raises(ValueError):
5
            register_user("", "email@example.com")
6
    
7
    def test_duplicate_email_rejected(self):
8
        register_user("user1", "test@example.com")
9
        with pytest.raises(ValueError):
10
            register_user("user2", "test@example.com")
11
    
12
    # Property-based tests for deeper coverage
13
    @given(st.text(), st.emails())
14
    def test_registration_properties(self, username, email):
15
        # `assume` tells Hypothesis to discard test cases that don't meet a condition.
16
        # It's different from an `assert` because it doesn't cause a failure;
17
        # it just skips uninteresting or invalid examples.
18
        # Here, we're not interested in testing empty usernames with this property.
19
        assume(username)
20
        
21
        user = register_user(username, email)
22
        
23
        # Properties that should hold
24
        assert user.username == username
25
        assert user.email == email.lower()
26
        assert user.id is not None
27
        assert user.created_at <= datetime.now()

1
from hypothesis.stateful import RuleBasedStateMachine, rule, invariant
2

3
class ShoppingCartMachine(RuleBasedStateMachine):
4
    def __init__(self):
5
        super().__init__()
6
        self.cart = ShoppingCart()
7
        self.model_items = {}
8
    
9
    @rule(item_id=st.integers(), quantity=st.integers(min_value=1, max_value=10))
10
    def add_item(self, item_id, quantity):
11
        self.cart.add_item(item_id, quantity)
12
        self.model_items[item_id] = self.model_items.get(item_id, 0) + quantity
13
    
14
    @rule(item_id=st.integers())
15
    def remove_item(self, item_id):
16
        if item_id in self.model_items:
17
            self.cart.remove_item(item_id)
18
            del self.model_items[item_id]
19
    
20
    @invariant()
21
    def quantities_match(self):
22
        for item_id, quantity in self.model_items.items():
23
            assert self.cart.get_quantity(item_id) == quantity
24

25
# Run the state machine test
26
TestCart = ShoppingCartMachine.TestCase

1
@st.composite
2
def sorted_lists(draw, elements=st.integers()):
3
    """Generate sorted lists."""
4
    lst = draw(st.lists(elements))
5
    return sorted(lst)
6

7
@st.composite  
8
def balanced_trees(draw):
9
    """Generate balanced binary trees."""
10
    size = draw(st.integers(min_value=0, max_value=4))
11
    if size == 0:
12
        return None
13
    
14
    left_size = size // 2
15
    right_size = size - left_size - 1
16
    
17
    return {
18
        'value': draw(st.integers()),
19
        'left': draw(balanced_trees()) if left_size > 0 else None,
20
        'right': draw(balanced_trees()) if right_size > 0 else None
21
    }