[mypyc] Make compilation order with multiple files consistent (#21419)

The `mypyc` plugin sets all modules inside a group as dependent on each
other so they are part of a single SCC. When we compile the SCC, we
compile its modules according to the order in `scc.mod_ids`. `mod_ids`
is a `set` so the order is non-deterministic.

This might result in random failures when multiple files are compiled
together and we have a bug in mypyc caused by relying on data that is
dependent on the compilation order. For example if we are compiling two
files `a.py` and `b.py` each with a single class and an attribute of
`ClassIR` representing the class `A` that is not set during the
preparation phase is used when compiling the class `B`, the result will
depend on whether `a.py` was compiled before `b.py`.

To fix, make the order deterministic based on module names by sorting
them first. Also sort the source files and groups passed to `mypycify`
so that invoking `mypyc a.py b.py` and `mypyc b.py a.py` produces
identical results.

Author: p-sawicki

mypyc/build.py

@@ -485,15 +485,19 @@ def construct_groups(
     else:
         groups = [(sources, None)]
 
-    # Generate missing names
+    # Generate missing names.
+    # Sort the modules to make the compilation results consistent regardless of
+    # the source file order passed to mypycify.
     for i, (group, name) in enumerate(groups):
+        group = sorted(group, key=lambda source: source.module)
         if use_shared_lib and not name:
             if group_name_override is not None:
                 name = group_name_override
             else:
                 name = group_name([source.module for source in group])
         groups[i] = (group, name)
 
+    groups = sorted(groups, key=lambda g: (g[1] or "", [s.module for s in g[0]]))
     return groups
 
 

mypyc/codegen/emitmodule.py

@@ -305,7 +305,7 @@ def compile_modules_to_ir(
 
     # Process the graph by SCC in topological order, like we do in mypy.build
     for scc in sorted_components(result.graph):
-        scc_states = [result.graph[id] for id in scc.mod_ids]
+        scc_states = [result.graph[id] for id in sorted(scc.mod_ids)]
         trees = [st.tree for st in scc_states if st.id in mapper.group_map and st.tree]
 
         if not trees:
@@ -1441,7 +1441,7 @@ def _toposort_visit(name: str) -> None:
             if decl.mark:
                 return
 
-            for child in decl.declaration.dependencies:
+            for child in sorted(decl.declaration.dependencies):
                 _toposort_visit(child)
 
             result.append(decl.declaration)

mypyc/irbuild/builder.py

@@ -1076,11 +1076,11 @@ def get_sequence_type_from_type(self, target_type: Type) -> RType:
             items = target_type.items
             assert items, "This function does not support empty tuples"
             # Tuple might have elements of different types.
-            rtypes = set(map(self.mapper.type_to_rtype, items))
+            rtypes = list(dict.fromkeys(self.mapper.type_to_rtype(item) for item in items))
             if len(rtypes) == 1:
                 return rtypes.pop()
             else:
-                return RUnion.make_simplified_union(list(rtypes))
+                return RUnion.make_simplified_union(rtypes)
         assert False, target_type
 
     def get_dict_base_type(self, expr: Expression) -> list[Instance]:

mypyc/test/test_emitmodule.py

@@ -0,0 +1,76 @@
+from __future__ import annotations
+
+import tempfile
+import unittest
+from pathlib import Path
+
+import pytest
+
+from mypy import build
+from mypy.options import Options
+from mypyc.build import construct_groups
+from mypyc.codegen import emitmodule
+from mypyc.errors import Errors
+from mypyc.irbuild.mapper import Mapper
+from mypyc.options import CompilerOptions
+
+
+class FakeSCC:
+    def __init__(self, mod_ids: list[str]) -> None:
+        self.mod_ids = mod_ids
+
+
+class TestEmitModule(unittest.TestCase):
+    def test_compile_modules_to_ir_orders_scc_members_deterministically(self) -> None:
+        with tempfile.TemporaryDirectory() as tmp_dir, pytest.MonkeyPatch.context() as monkeypatch:
+            tmp_path = Path(tmp_dir)
+            a_py = tmp_path / "a.py"
+            b_py = tmp_path / "b.py"
+            a_py.write_text("import b\n\nclass A: pass\nclass C(A): pass\n", encoding="utf-8")
+            b_py.write_text(
+                "import a\n\nclass B(a.A): pass\nclass D(a.A): pass\n", encoding="utf-8"
+            )
+
+            sources = [
+                build.BuildSource(str(a_py), "a", None),
+                build.BuildSource(str(b_py), "b", None),
+            ]
+            options = Options()
+            options.preserve_asts = True
+            options.mypy_path = [str(tmp_path)]
+            options.cache_dir = str(tmp_path / ".mypy_cache")
+            for source in sources:
+                options.per_module_options.setdefault(source.module, {})["mypyc"] = True
+
+            compiler_options = CompilerOptions(strict_traceback_checks=True)
+            groups = construct_groups(
+                sources, False, use_shared_lib=True, group_name_override=None
+            )
+            result = emitmodule.parse_and_typecheck(sources, options, compiler_options, groups)
+            try:
+                group_map = {
+                    source.module: lib_name for group, lib_name in groups for source in group
+                }
+                children_by_order = []
+                for order in (["a", "b"], ["b", "a"]):
+                    monkeypatch.setattr(
+                        emitmodule,
+                        "sorted_components",
+                        lambda graph, order=order: [FakeSCC(order)],
+                    )
+                    mapper = Mapper(group_map)
+                    errors = Errors(options)
+                    modules = emitmodule.compile_modules_to_ir(
+                        result, mapper, compiler_options, errors
+                    )
+                    assert errors.num_errors == 0, errors.new_messages()
+                    classes = {
+                        cl.fullname: cl for module in modules.values() for cl in module.classes
+                    }
+                    children = classes["a.A"].children
+                    assert children is not None
+                    children_by_order.append([child.fullname for child in children])
+
+                assert children_by_order[1] == children_by_order[0]
+            finally:
+                result.manager.metastore.close()