Split deterministically regardless of test order Fix #23

src/pytest_split/algorithms.py

@@ -41,14 +41,19 @@ def least_duration(
         (*tup, i) for i, tup in enumerate(items_with_durations)
     ]
 
+    # Sort by name to ensure it's always the same order
+    items_with_durations_indexed = sorted(
+        items_with_durations_indexed, key=lambda tup: str(tup[0])
+    )
+
     # sort in ascending order
     sorted_items_with_durations = sorted(
         items_with_durations_indexed, key=lambda tup: tup[1], reverse=True
     )
 
-    selected: "List[List[Tuple[nodes.Item, int]]]" = [[] for i in range(splits)]
-    deselected: "List[List[nodes.Item]]" = [[] for i in range(splits)]
-    duration: "List[float]" = [0 for i in range(splits)]
+    selected: "List[List[Tuple[nodes.Item, int]]]" = [[] for _ in range(splits)]
+    deselected: "List[List[nodes.Item]]" = [[] for _ in range(splits)]
+    duration: "List[float]" = [0 for _ in range(splits)]
 
     # create a heap of the form (summed_durations, group_index)
     heap: "List[Tuple[float, int]]" = [(0, i) for i in range(splits)]

tests/test_algorithms.py

@@ -1,7 +1,13 @@
+import itertools
 from collections import namedtuple
+from typing import TYPE_CHECKING
 
 import pytest
 
+if TYPE_CHECKING:
+    from typing import List, Set
+    from _pytest.nodes import Item
+
 from pytest_split.algorithms import Algorithms
 
 item = namedtuple("item", "nodeid")
@@ -110,3 +116,18 @@ def test__split_tests_maintains_relative_order_of_tests(self, algo_name, expecte
         expected_first, expected_second = expected
         assert first.selected == expected_first
         assert second.selected == expected_second
+
+    def test__split_tests_same_set_regardless_of_order(self):
+        """NOTE: only least_duration does this correctly"""
+        tests = ["a", "b", "c", "d", "e", "f", "g"]
+        durations = {t: 1 for t in tests}
+        items = [item(t) for t in tests]
+        algo = Algorithms["least_duration"].value
+        for n in (2, 3, 4):
+            selected_each: "List[Set[Item]]" = [set() for _ in range(n)]
+            for order in itertools.permutations(items):
+                splits = algo(splits=n, items=order, durations=durations)
+                for i, group in enumerate(splits):
+                    if not selected_each[i]:
+                        selected_each[i] = set(group.selected)
+                    assert selected_each[i] == set(group.selected)

tests/test_plugin.py

@@ -141,8 +141,8 @@ class TestSplitToSuites:
             ["test_1", "test_2", "test_3", "test_4", "test_5", "test_6", "test_7"],
         ),
         (2, 2, "duration_based_chunks", ["test_8", "test_9", "test_10"]),
-        (2, 1, "least_duration", ["test_3", "test_5", "test_6", "test_8", "test_10"]),
-        (2, 2, "least_duration", ["test_1", "test_2", "test_4", "test_7", "test_9"]),
+        (2, 1, "least_duration", ["test_3", "test_5", "test_7", "test_9", "test_10"]),
+        (2, 2, "least_duration", ["test_1", "test_2", "test_4", "test_6", "test_8"]),
         (
             3,
             1,
@@ -151,17 +151,17 @@ class TestSplitToSuites:
         ),
         (3, 2, "duration_based_chunks", ["test_6", "test_7", "test_8"]),
         (3, 3, "duration_based_chunks", ["test_9", "test_10"]),
-        (3, 1, "least_duration", ["test_3", "test_6", "test_9"]),
-        (3, 2, "least_duration", ["test_4", "test_7", "test_10"]),
-        (3, 3, "least_duration", ["test_1", "test_2", "test_5", "test_8"]),
+        (3, 1, "least_duration", ["test_3", "test_8", "test_10"]),
+        (3, 2, "least_duration", ["test_4", "test_6", "test_9"]),
+        (3, 3, "least_duration", ["test_1", "test_2", "test_5", "test_7"]),
         (4, 1, "duration_based_chunks", ["test_1", "test_2", "test_3", "test_4"]),
         (4, 2, "duration_based_chunks", ["test_5", "test_6", "test_7"]),
         (4, 3, "duration_based_chunks", ["test_8", "test_9"]),
         (4, 4, "duration_based_chunks", ["test_10"]),
-        (4, 1, "least_duration", ["test_6", "test_10"]),
-        (4, 2, "least_duration", ["test_1", "test_4", "test_7"]),
-        (4, 3, "least_duration", ["test_2", "test_5", "test_8"]),
-        (4, 4, "least_duration", ["test_3", "test_9"]),
+        (4, 1, "least_duration", ["test_9", "test_10"]),
+        (4, 2, "least_duration", ["test_1", "test_4", "test_6"]),
+        (4, 3, "least_duration", ["test_2", "test_5", "test_7"]),
+        (4, 4, "least_duration", ["test_3", "test_8"]),
     ]
     legacy_duration = [True, False]
     all_params = [