[AUTOGENERATED] [release/2.9] [Inductor] Fix flaky epilogue fusion tests by adding missing tearDown…#3250
Draft
rocm-repo-management-api-6[bot] wants to merge 1 commit into
ROCm Repo Management API / Tests / Tests / Test Inductor / Run pytorch_inductor_1
failed
May 21, 2026 in 0s
failed: 4, skipped: 23617, passed: 20439
failed: 4, skipped: 23617, passed: 20439
Details
AOTInductorTestABICompatibleGpu.test_aoti_debug_printer_sym_inputs_cuda
AttributeError: 'int' object has no attribute 'is_Add'
To execute this test, run the following from the base repo dir:
PYTORCH_TEST_WITH_ROCM=1 python test/inductor/test_aot_inductor.py AOTInductorTestABICompatibleGpu.test_aoti_debug_printer_sym_inputs_cuda
This message can be suppressed by setting PYTORCH_PRINT_REPRO_ON_FAILURE=0
Stack trace
Traceback (most recent call last):
File "/var/lib/jenkins/pytorch/test/inductor/test_torchinductor.py", line 14068, in new_test
return value(self)
^^^^^^^^^^^
File "/var/lib/jenkins/pytorch/test/inductor/test_aot_inductor.py", line 5185, in test_aoti_debug_printer_sym_inputs
result, code = run_and_get_cpp_code(
^^^^^^^^^^^^^^^^^^^^^
File "/opt/conda/envs/py_3.12/lib/python3.12/site-packages/torch/_inductor/utils.py", line 2978, in run_and_get_cpp_code
result = fn(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^
File "/var/lib/jenkins/pytorch/test/inductor/test_aot_inductor_utils.py", line 168, in compile
package_path = torch._inductor.aoti_compile_and_package(
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/opt/conda/envs/py_3.12/lib/python3.12/site-packages/torch/_inductor/__init__.py", line 151, in aoti_compile_and_package
return aot_inductor_minifier_wrapper(
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/opt/conda/envs/py_3.12/lib/python3.12/site-packages/torch/_inductor/debug.py", line 1320, in aot_inductor_minifier_wrapper
raise e
File "/opt/conda/envs/py_3.12/lib/python3.12/site-packages/torch/_inductor/debug.py", line 1290, in aot_inductor_minifier_wrapper
return func(
^^^^^
File "/opt/conda/envs/py_3.12/lib/python3.12/site-packages/torch/_inductor/__init__.py", line 194, in _aoti_compile_and_package_inner
aoti_files = aot_compile(gm, args, kwargs, options=inductor_configs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/opt/conda/envs/py_3.12/lib/python3.12/site-packages/torch/_inductor/__init__.py", line 310, in aot_compile
return compile_fx_aot(
^^^^^^^^^^^^^^^
File "/opt/conda/envs/py_3.12/lib/python3.12/site-packages/torch/_inductor/compile_fx.py", line 1941, in compile_fx_aot
compiled_artifacts = compile_fx(
^^^^^^^^^^^
File "/opt/conda/envs/py_3.12/lib/python3.12/site-packages/torch/_inductor/compile_fx.py", line 2413, in compile_fx
return compile_fx(
^^^^^^^^^^^
File "/opt/conda/envs/py_3.12/lib/python3.12/site-packages/torch/_inductor/compile_fx.py", line 2474, in compile_fx
return compile_fx(
^^^^^^^^^^^
File "/opt/conda/envs/py_3.12/lib/python3.12/site-packages/torch/_inductor/compile_fx.py", line 2672, in compile_fx
return inference_compiler(unlifted_gm, example_inputs_)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/opt/conda/envs/py_3.12/lib/python3.12/site-packages/torch/_functorch/_aot_autograd/schemas.py", line 1251, in __call__
return self.compiler_fn(gm, example_inputs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/opt/conda/envs/py_3.12/lib/python3.12/site-packages/torch/_inductor/compile_fx.py", line 2558, in fw_compiler_base
return compile_fx_forward(
^^^^^^^^^^^^^^^^^^^
File "/opt/conda/envs/py_3.12/lib/python3.12/site-packages/torch/_inductor/compile_fx.py", line 2275, in compile_fx_forward
return inner_compile(
^^^^^^^^^^^^^^
File "/opt/conda/envs/py_3.12/lib/python3.12/contextlib.py", line 81, in inner
return func(*args, **kwds)
^^^^^^^^^^^^^^^^^^^
File "/opt/conda/envs/py_3.12/lib/python3.12/site-packages/torch/_inductor/compile_fx.py", line 782, in compile_fx_inner
return wrap_compiler_debug(_compile_fx_inner, compiler_name="inductor")(
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/opt/conda/envs/py_3.12/lib/python3.12/site-packages/torch/_dynamo/repro/after_aot.py", line 144, in debug_wrapper
inner_compiled_fn = compiler_fn(gm, example_inputs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/opt/conda/envs/py_3.12/lib/python3.12/site-packages/torch/_inductor/compile_fx.py", line 963, in _compile_fx_inner
mb_compiled_graph = fx_codegen_and_compile(
^^^^^^^^^^^^^^^^^^^^^^^
File "/opt/conda/envs/py_3.12/lib/python3.12/site-packages/torch/_inductor/compile_fx.py", line 1695, in fx_codegen_and_compile
return scheme.codegen_and_compile(gm, example_inputs, inputs_to_check, graph_kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/opt/conda/envs/py_3.12/lib/python3.12/site-packages/torch/_inductor/compile_fx.py", line 1464, in codegen_and_compile
wrapper_code, kernel_code = graph.codegen_with_cpp_wrapper()
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/opt/conda/envs/py_3.12/lib/python3.12/site-packages/torch/_inductor/graph.py", line 2220, in codegen_with_cpp_wrapper
return self.codegen()
^^^^^^^^^^^^^^
File "/opt/conda/envs/py_3.12/lib/python3.12/site-packages/torch/_inductor/graph.py", line 2260, in codegen
self._update_scheduler()
File "/opt/conda/envs/py_3.12/lib/python3.12/site-packages/torch/_inductor/graph.py", line 2254, in _update_scheduler
self.scheduler = Scheduler(self.operations)
^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/opt/conda/envs/py_3.12/lib/python3.12/site-packages/torch/_inductor/scheduler.py", line 2226, in __init__
self._init(nodes)
File "/opt/conda/envs/py_3.12/lib/python3.12/site-packages/torch/_inductor/scheduler.py", line 2327, in _init
self.nodes = reorder_for_peak_memory(
^^^^^^^^^^^^^^^^^^^^^^^^
File "/opt/conda/envs/py_3.12/lib/python3.12/site-packages/torch/_inductor/memory.py", line 882, in reorder_for_peak_memory
estimated_peak_memory, name_to_freeable_input_buf = prepare_planning_info(
^^^^^^^^^^^^^^^^^^^^^^
File "/opt/conda/envs/py_3.12/lib/python3.12/site-packages/torch/_inductor/memory.py", line 850, in prepare_planning_info
assign_memory_planning_info_for_scheduler_buffers(nodes, name_to_buf)
File "/opt/conda/envs/py_3.12/lib/python3.12/site-packages/torch/_inductor/memory.py", line 215, in assign_memory_planning_info_for_scheduler_buffers
sched_buf_to_size = compute_size_for_scheduler_buffer(name_to_buf)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/opt/conda/envs/py_3.12/lib/python3.12/site-packages/torch/_inductor/memory.py", line 201, in compute_size_for_scheduler_buffer
_compute_and_update_buf_size(sched_buf)
File "/opt/conda/envs/py_3.12/lib/python3.12/site-packages/torch/_inductor/memory.py", line 188, in _compute_and_update_buf_size
buf_size = V.graph.sizevars.size_hint(
^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/opt/conda/envs/py_3.12/lib/python3.12/site-packages/torch/_inductor/sizevars.py", line 605, in size_hint
substituted = sympy_subs(out, unbacked_hints)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/opt/conda/envs/py_3.12/lib/python3.12/site-packages/torch/_inductor/utils.py", line 1087, in sympy_subs
return sympy.sympify(expr).xreplace(
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/opt/conda/envs/py_3.12/lib/python3.12/site-packages/sympy/core/basic.py", line 1313, in xreplace
value, _ = self._xreplace(rule)
^^^^^^^^^^^^^^^^^^^^
File "/opt/conda/envs/py_3.12/lib/python3.12/site-packages/sympy/core/basic.py", line 1335, in _xreplace
return self.func(*args), True
^^^^^^^^^^^^^^^^
File "/opt/conda/envs/py_3.12/lib/python3.12/site-packages/torch/utils/_sympy/functions.py", line 608, in __new__
else cls._satisfy_unique_summations_symbols(original_args)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/opt/conda/envs/py_3.12/lib/python3.12/site-packages/torch/utils/_sympy/functions.py", line 682, in _satisfy_unique_summations_symbols
if not _is_symbols_binary_summation(rhs):
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/opt/conda/envs/py_3.12/lib/python3.12/site-packages/torch/utils/_sympy/functions.py", line 93, in _is_symbols_binary_summation
expr.is_Add
AttributeError: 'int' object has no attribute 'is_Add'
To execute this test, run the following from the base repo dir:
PYTORCH_TEST_WITH_ROCM=1 python test/inductor/test_aot_inductor.py AOTInductorTestABICompatibleGpu.test_aoti_debug_printer_sym_inputs_cuda
This message can be suppressed by setting PYTORCH_PRINT_REPRO_ON_FAILURE=0
Standard out
stats [('calls_captured', 6), ('unique_graphs', 1)]
inductor [('pattern_matcher_count', 1), ('pattern_matcher_nodes', 1)]
graph_break []
stats [('calls_captured', 6), ('unique_graphs', 1)]
inductor [('pattern_matcher_count', 1), ('pattern_matcher_nodes', 1)]
graph_break []
stats [('calls_captured', 6), ('unique_graphs', 1)]
inductor [('pattern_matcher_count', 1), ('pattern_matcher_nodes', 1)]
graph_break []
AOTInductorTestABICompatibleGpu.test_aoti_debug_printer_sym_inputs_cuda
AttributeError: 'int' object has no attribute 'is_Add'
To execute this test, run the following from the base repo dir:
PYTORCH_TEST_WITH_ROCM=1 python test/inductor/test_aot_inductor.py AOTInductorTestABICompatibleGpu.test_aoti_debug_printer_sym_inputs_cuda
This message can be suppressed by setting PYTORCH_PRINT_REPRO_ON_FAILURE=0
Stack trace
Traceback (most recent call last):
File "/var/lib/jenkins/pytorch/test/inductor/test_torchinductor.py", line 14068, in new_test
return value(self)
^^^^^^^^^^^
File "/var/lib/jenkins/pytorch/test/inductor/test_aot_inductor.py", line 5185, in test_aoti_debug_printer_sym_inputs
result, code = run_and_get_cpp_code(
^^^^^^^^^^^^^^^^^^^^^
File "/opt/conda/envs/py_3.12/lib/python3.12/site-packages/torch/_inductor/utils.py", line 2978, in run_and_get_cpp_code
result = fn(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^
File "/var/lib/jenkins/pytorch/test/inductor/test_aot_inductor_utils.py", line 168, in compile
package_path = torch._inductor.aoti_compile_and_package(
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/opt/conda/envs/py_3.12/lib/python3.12/site-packages/torch/_inductor/__init__.py", line 151, in aoti_compile_and_package
return aot_inductor_minifier_wrapper(
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/opt/conda/envs/py_3.12/lib/python3.12/site-packages/torch/_inductor/debug.py", line 1320, in aot_inductor_minifier_wrapper
raise e
File "/opt/conda/envs/py_3.12/lib/python3.12/site-packages/torch/_inductor/debug.py", line 1290, in aot_inductor_minifier_wrapper
return func(
^^^^^
File "/opt/conda/envs/py_3.12/lib/python3.12/site-packages/torch/_inductor/__init__.py", line 194, in _aoti_compile_and_package_inner
aoti_files = aot_compile(gm, args, kwargs, options=inductor_configs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/opt/conda/envs/py_3.12/lib/python3.12/site-packages/torch/_inductor/__init__.py", line 310, in aot_compile
return compile_fx_aot(
^^^^^^^^^^^^^^^
File "/opt/conda/envs/py_3.12/lib/python3.12/site-packages/torch/_inductor/compile_fx.py", line 1941, in compile_fx_aot
compiled_artifacts = compile_fx(
^^^^^^^^^^^
File "/opt/conda/envs/py_3.12/lib/python3.12/site-packages/torch/_inductor/compile_fx.py", line 2413, in compile_fx
return compile_fx(
^^^^^^^^^^^
File "/opt/conda/envs/py_3.12/lib/python3.12/site-packages/torch/_inductor/compile_fx.py", line 2474, in compile_fx
return compile_fx(
^^^^^^^^^^^
File "/opt/conda/envs/py_3.12/lib/python3.12/site-packages/torch/_inductor/compile_fx.py", line 2672, in compile_fx
return inference_compiler(unlifted_gm, example_inputs_)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/opt/conda/envs/py_3.12/lib/python3.12/site-packages/torch/_functorch/_aot_autograd/schemas.py", line 1251, in __call__
return self.compiler_fn(gm, example_inputs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/opt/conda/envs/py_3.12/lib/python3.12/site-packages/torch/_inductor/compile_fx.py", line 2558, in fw_compiler_base
return compile_fx_forward(
^^^^^^^^^^^^^^^^^^^
File "/opt/conda/envs/py_3.12/lib/python3.12/site-packages/torch/_inductor/compile_fx.py", line 2275, in compile_fx_forward
return inner_compile(
^^^^^^^^^^^^^^
File "/opt/conda/envs/py_3.12/lib/python3.12/contextlib.py", line 81, in inner
return func(*args, **kwds)
^^^^^^^^^^^^^^^^^^^
File "/opt/conda/envs/py_3.12/lib/python3.12/site-packages/torch/_inductor/compile_fx.py", line 782, in compile_fx_inner
return wrap_compiler_debug(_compile_fx_inner, compiler_name="inductor")(
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/opt/conda/envs/py_3.12/lib/python3.12/site-packages/torch/_dynamo/repro/after_aot.py", line 144, in debug_wrapper
inner_compiled_fn = compiler_fn(gm, example_inputs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/opt/conda/envs/py_3.12/lib/python3.12/site-packages/torch/_inductor/compile_fx.py", line 963, in _compile_fx_inner
mb_compiled_graph = fx_codegen_and_compile(
^^^^^^^^^^^^^^^^^^^^^^^
File "/opt/conda/envs/py_3.12/lib/python3.12/site-packages/torch/_inductor/compile_fx.py", line 1695, in fx_codegen_and_compile
return scheme.codegen_and_compile(gm, example_inputs, inputs_to_check, graph_kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/opt/conda/envs/py_3.12/lib/python3.12/site-packages/torch/_inductor/compile_fx.py", line 1464, in codegen_and_compile
wrapper_code, kernel_code = graph.codegen_with_cpp_wrapper()
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/opt/conda/envs/py_3.12/lib/python3.12/site-packages/torch/_inductor/graph.py", line 2220, in codegen_with_cpp_wrapper
return self.codegen()
^^^^^^^^^^^^^^
File "/opt/conda/envs/py_3.12/lib/python3.12/site-packages/torch/_inductor/graph.py", line 2260, in codegen
self._update_scheduler()
File "/opt/conda/envs/py_3.12/lib/python3.12/site-packages/torch/_inductor/graph.py", line 2254, in _update_scheduler
self.scheduler = Scheduler(self.operations)
^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/opt/conda/envs/py_3.12/lib/python3.12/site-packages/torch/_inductor/scheduler.py", line 2226, in __init__
self._init(nodes)
File "/opt/conda/envs/py_3.12/lib/python3.12/site-packages/torch/_inductor/scheduler.py", line 2327, in _init
self.nodes = reorder_for_peak_memory(
^^^^^^^^^^^^^^^^^^^^^^^^
File "/opt/conda/envs/py_3.12/lib/python3.12/site-packages/torch/_inductor/memory.py", line 882, in reorder_for_peak_memory
estimated_peak_memory, name_to_freeable_input_buf = prepare_planning_info(
^^^^^^^^^^^^^^^^^^^^^^
File "/opt/conda/envs/py_3.12/lib/python3.12/site-packages/torch/_inductor/memory.py", line 850, in prepare_planning_info
assign_memory_planning_info_for_scheduler_buffers(nodes, name_to_buf)
File "/opt/conda/envs/py_3.12/lib/python3.12/site-packages/torch/_inductor/memory.py", line 215, in assign_memory_planning_info_for_scheduler_buffers
sched_buf_to_size = compute_size_for_scheduler_buffer(name_to_buf)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/opt/conda/envs/py_3.12/lib/python3.12/site-packages/torch/_inductor/memory.py", line 201, in compute_size_for_scheduler_buffer
_compute_and_update_buf_size(sched_buf)
File "/opt/conda/envs/py_3.12/lib/python3.12/site-packages/torch/_inductor/memory.py", line 188, in _compute_and_update_buf_size
buf_size = V.graph.sizevars.size_hint(
^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/opt/conda/envs/py_3.12/lib/python3.12/site-packages/torch/_inductor/sizevars.py", line 605, in size_hint
substituted = sympy_subs(out, unbacked_hints)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/opt/conda/envs/py_3.12/lib/python3.12/site-packages/torch/_inductor/utils.py", line 1087, in sympy_subs
return sympy.sympify(expr).xreplace(
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/opt/conda/envs/py_3.12/lib/python3.12/site-packages/sympy/core/basic.py", line 1313, in xreplace
value, _ = self._xreplace(rule)
^^^^^^^^^^^^^^^^^^^^
File "/opt/conda/envs/py_3.12/lib/python3.12/site-packages/sympy/core/basic.py", line 1335, in _xreplace
return self.func(*args), True
^^^^^^^^^^^^^^^^
File "/opt/conda/envs/py_3.12/lib/python3.12/site-packages/torch/utils/_sympy/functions.py", line 608, in __new__
else cls._satisfy_unique_summations_symbols(original_args)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/opt/conda/envs/py_3.12/lib/python3.12/site-packages/torch/utils/_sympy/functions.py", line 682, in _satisfy_unique_summations_symbols
if not _is_symbols_binary_summation(rhs):
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/opt/conda/envs/py_3.12/lib/python3.12/site-packages/torch/utils/_sympy/functions.py", line 93, in _is_symbols_binary_summation
expr.is_Add
AttributeError: 'int' object has no attribute 'is_Add'
To execute this test, run the following from the base repo dir:
PYTORCH_TEST_WITH_ROCM=1 python test/inductor/test_aot_inductor.py AOTInductorTestABICompatibleGpu.test_aoti_debug_printer_sym_inputs_cuda
This message can be suppressed by setting PYTORCH_PRINT_REPRO_ON_FAILURE=0
Standard out
stats [('calls_captured', 6), ('unique_graphs', 1)]
inductor [('pattern_matcher_count', 1), ('pattern_matcher_nodes', 1)]
graph_break []
stats [('calls_captured', 6), ('unique_graphs', 1)]
inductor [('pattern_matcher_count', 1), ('pattern_matcher_nodes', 1)]
graph_break []
stats [('calls_captured', 6), ('unique_graphs', 1)]
inductor [('pattern_matcher_count', 1), ('pattern_matcher_nodes', 1)]
graph_break []
AOTInductorTestABICompatibleGpu.test_aoti_debug_printer_sym_inputs_cuda
AttributeError: 'int' object has no attribute 'is_Add'
To execute this test, run the following from the base repo dir:
PYTORCH_TEST_WITH_ROCM=1 python test/inductor/test_aot_inductor.py AOTInductorTestABICompatibleGpu.test_aoti_debug_printer_sym_inputs_cuda
This message can be suppressed by setting PYTORCH_PRINT_REPRO_ON_FAILURE=0
Stack trace
Traceback (most recent call last):
File "/var/lib/jenkins/pytorch/test/inductor/test_torchinductor.py", line 14068, in new_test
return value(self)
^^^^^^^^^^^
File "/var/lib/jenkins/pytorch/test/inductor/test_aot_inductor.py", line 5185, in test_aoti_debug_printer_sym_inputs
result, code = run_and_get_cpp_code(
^^^^^^^^^^^^^^^^^^^^^
File "/opt/conda/envs/py_3.12/lib/python3.12/site-packages/torch/_inductor/utils.py", line 2978, in run_and_get_cpp_code
result = fn(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^
File "/var/lib/jenkins/pytorch/test/inductor/test_aot_inductor_utils.py", line 168, in compile
package_path = torch._inductor.aoti_compile_and_package(
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/opt/conda/envs/py_3.12/lib/python3.12/site-packages/torch/_inductor/__init__.py", line 151, in aoti_compile_and_package
return aot_inductor_minifier_wrapper(
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/opt/conda/envs/py_3.12/lib/python3.12/site-packages/torch/_inductor/debug.py", line 1320, in aot_inductor_minifier_wrapper
raise e
File "/opt/conda/envs/py_3.12/lib/python3.12/site-packages/torch/_inductor/debug.py", line 1290, in aot_inductor_minifier_wrapper
return func(
^^^^^
File "/opt/conda/envs/py_3.12/lib/python3.12/site-packages/torch/_inductor/__init__.py", line 194, in _aoti_compile_and_package_inner
aoti_files = aot_compile(gm, args, kwargs, options=inductor_configs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/opt/conda/envs/py_3.12/lib/python3.12/site-packages/torch/_inductor/__init__.py", line 310, in aot_compile
return compile_fx_aot(
^^^^^^^^^^^^^^^
File "/opt/conda/envs/py_3.12/lib/python3.12/site-packages/torch/_inductor/compile_fx.py", line 1941, in compile_fx_aot
compiled_artifacts = compile_fx(
^^^^^^^^^^^
File "/opt/conda/envs/py_3.12/lib/python3.12/site-packages/torch/_inductor/compile_fx.py", line 2413, in compile_fx
return compile_fx(
^^^^^^^^^^^
File "/opt/conda/envs/py_3.12/lib/python3.12/site-packages/torch/_inductor/compile_fx.py", line 2474, in compile_fx
return compile_fx(
^^^^^^^^^^^
File "/opt/conda/envs/py_3.12/lib/python3.12/site-packages/torch/_inductor/compile_fx.py", line 2672, in compile_fx
return inference_compiler(unlifted_gm, example_inputs_)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/opt/conda/envs/py_3.12/lib/python3.12/site-packages/torch/_functorch/_aot_autograd/schemas.py", line 1251, in __call__
return self.compiler_fn(gm, example_inputs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/opt/conda/envs/py_3.12/lib/python3.12/site-packages/torch/_inductor/compile_fx.py", line 2558, in fw_compiler_base
return compile_fx_forward(
^^^^^^^^^^^^^^^^^^^
File "/opt/conda/envs/py_3.12/lib/python3.12/site-packages/torch/_inductor/compile_fx.py", line 2275, in compile_fx_forward
return inner_compile(
^^^^^^^^^^^^^^
File "/opt/conda/envs/py_3.12/lib/python3.12/contextlib.py", line 81, in inner
return func(*args, **kwds)
^^^^^^^^^^^^^^^^^^^
File "/opt/conda/envs/py_3.12/lib/python3.12/site-packages/torch/_inductor/compile_fx.py", line 782, in compile_fx_inner
return wrap_compiler_debug(_compile_fx_inner, compiler_name="inductor")(
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/opt/conda/envs/py_3.12/lib/python3.12/site-packages/torch/_dynamo/repro/after_aot.py", line 144, in debug_wrapper
inner_compiled_fn = compiler_fn(gm, example_inputs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/opt/conda/envs/py_3.12/lib/python3.12/site-packages/torch/_inductor/compile_fx.py", line 963, in _compile_fx_inner
mb_compiled_graph = fx_codegen_and_compile(
^^^^^^^^^^^^^^^^^^^^^^^
File "/opt/conda/envs/py_3.12/lib/python3.12/site-packages/torch/_inductor/compile_fx.py", line 1695, in fx_codegen_and_compile
return scheme.codegen_and_compile(gm, example_inputs, inputs_to_check, graph_kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/opt/conda/envs/py_3.12/lib/python3.12/site-packages/torch/_inductor/compile_fx.py", line 1464, in codegen_and_compile
wrapper_code, kernel_code = graph.codegen_with_cpp_wrapper()
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/opt/conda/envs/py_3.12/lib/python3.12/site-packages/torch/_inductor/graph.py", line 2220, in codegen_with_cpp_wrapper
return self.codegen()
^^^^^^^^^^^^^^
File "/opt/conda/envs/py_3.12/lib/python3.12/site-packages/torch/_inductor/graph.py", line 2260, in codegen
self._update_scheduler()
File "/opt/conda/envs/py_3.12/lib/python3.12/site-packages/torch/_inductor/graph.py", line 2254, in _update_scheduler
self.scheduler = Scheduler(self.operations)
^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/opt/conda/envs/py_3.12/lib/python3.12/site-packages/torch/_inductor/scheduler.py", line 2226, in __init__
self._init(nodes)
File "/opt/conda/envs/py_3.12/lib/python3.12/site-packages/torch/_inductor/scheduler.py", line 2327, in _init
self.nodes = reorder_for_peak_memory(
^^^^^^^^^^^^^^^^^^^^^^^^
File "/opt/conda/envs/py_3.12/lib/python3.12/site-packages/torch/_inductor/memory.py", line 882, in reorder_for_peak_memory
estimated_peak_memory, name_to_freeable_input_buf = prepare_planning_info(
^^^^^^^^^^^^^^^^^^^^^^
File "/opt/conda/envs/py_3.12/lib/python3.12/site-packages/torch/_inductor/memory.py", line 850, in prepare_planning_info
assign_memory_planning_info_for_scheduler_buffers(nodes, name_to_buf)
File "/opt/conda/envs/py_3.12/lib/python3.12/site-packages/torch/_inductor/memory.py", line 215, in assign_memory_planning_info_for_scheduler_buffers
sched_buf_to_size = compute_size_for_scheduler_buffer(name_to_buf)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/opt/conda/envs/py_3.12/lib/python3.12/site-packages/torch/_inductor/memory.py", line 201, in compute_size_for_scheduler_buffer
_compute_and_update_buf_size(sched_buf)
File "/opt/conda/envs/py_3.12/lib/python3.12/site-packages/torch/_inductor/memory.py", line 188, in _compute_and_update_buf_size
buf_size = V.graph.sizevars.size_hint(
^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/opt/conda/envs/py_3.12/lib/python3.12/site-packages/torch/_inductor/sizevars.py", line 605, in size_hint
substituted = sympy_subs(out, unbacked_hints)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/opt/conda/envs/py_3.12/lib/python3.12/site-packages/torch/_inductor/utils.py", line 1087, in sympy_subs
return sympy.sympify(expr).xreplace(
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/opt/conda/envs/py_3.12/lib/python3.12/site-packages/sympy/core/basic.py", line 1313, in xreplace
value, _ = self._xreplace(rule)
^^^^^^^^^^^^^^^^^^^^
File "/opt/conda/envs/py_3.12/lib/python3.12/site-packages/sympy/core/basic.py", line 1335, in _xreplace
return self.func(*args), True
^^^^^^^^^^^^^^^^
File "/opt/conda/envs/py_3.12/lib/python3.12/site-packages/torch/utils/_sympy/functions.py", line 608, in __new__
else cls._satisfy_unique_summations_symbols(original_args)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/opt/conda/envs/py_3.12/lib/python3.12/site-packages/torch/utils/_sympy/functions.py", line 682, in _satisfy_unique_summations_symbols
if not _is_symbols_binary_summation(rhs):
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/opt/conda/envs/py_3.12/lib/python3.12/site-packages/torch/utils/_sympy/functions.py", line 93, in _is_symbols_binary_summation
expr.is_Add
AttributeError: 'int' object has no attribute 'is_Add'
To execute this test, run the following from the base repo dir:
PYTORCH_TEST_WITH_ROCM=1 python test/inductor/test_aot_inductor.py AOTInductorTestABICompatibleGpu.test_aoti_debug_printer_sym_inputs_cuda
This message can be suppressed by setting PYTORCH_PRINT_REPRO_ON_FAILURE=0
Standard out
stats [('calls_captured', 6), ('unique_graphs', 1)]
inductor [('pattern_matcher_count', 1), ('pattern_matcher_nodes', 1)]
graph_break []
stats [('calls_captured', 6), ('unique_graphs', 1)]
inductor [('pattern_matcher_count', 1), ('pattern_matcher_nodes', 1)]
graph_break []
stats [('calls_captured', 6), ('unique_graphs', 1)]
inductor [('pattern_matcher_count', 1), ('pattern_matcher_nodes', 1)]
graph_break []
GPUTests.test_split_cumprod_cuda
AssertionError: Tensor-likes are not close!
Mismatched elements: 23616 / 100000 (23.6%)
Greatest absolute difference: nan at index (6, 4096) (up to 1e-05 allowed)
Greatest relative difference: nan at index (6, 4096) (up to 0.0001 allowed)
To execute this test, run the following from the base repo dir:
PYTORCH_TEST_WITH_ROCM=1 python test/inductor/test_torchinductor.py GPUTests.test_split_cumprod_cuda
This message can be suppressed by setting PYTORCH_PRINT_REPRO_ON_FAILURE=0
Stack trace
Traceback (most recent call last):
File "/var/lib/jenkins/pytorch/test/inductor/test_torchinductor.py", line 14068, in new_test
return value(self)
^^^^^^^^^^^
File "/var/lib/jenkins/pytorch/test/inductor/test_torchinductor.py", line 2237, in test_split_cumprod
self.common(fn, (inp,), atol=1e-5, rtol=1e-4, check_lowp=False)
File "/opt/conda/envs/py_3.12/lib/python3.12/contextlib.py", line 81, in inner
return func(*args, **kwds)
^^^^^^^^^^^^^^^^^^^
File "/var/lib/jenkins/pytorch/test/inductor/test_torchinductor.py", line 686, in check_model_gpu
check_model(
File "/var/lib/jenkins/pytorch/test/inductor/test_torchinductor.py", line 560, in check_model
assert_equal_fn(
File "/opt/conda/envs/py_3.12/lib/python3.12/site-packages/torch/_dynamo/test_case.py", line 111, in assertEqual
return super().assertEqual(x, y, *args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/opt/conda/envs/py_3.12/lib/python3.12/site-packages/torch/testing/_internal/common_utils.py", line 4196, in assertEqual
raise error_metas.pop()[0].to_error( # type: ignore[index]
AssertionError: Tensor-likes are not close!
Mismatched elements: 23616 / 100000 (23.6%)
Greatest absolute difference: nan at index (6, 4096) (up to 1e-05 allowed)
Greatest relative difference: nan at index (6, 4096) (up to 0.0001 allowed)
To execute this test, run the following from the base repo dir:
PYTORCH_TEST_WITH_ROCM=1 python test/inductor/test_torchinductor.py GPUTests.test_split_cumprod_cuda
This message can be suppressed by setting PYTORCH_PRINT_REPRO_ON_FAILURE=0
Standard out
inline_call []
stats [('calls_captured', 1), ('unique_graphs', 1)]
aot_autograd [('total', 1), ('autograd_cache_miss', 1), ('autograd_cache_saved', 1), ('ok', 1)]
inductor [('triton_bundler_save_kernel', 8), ('async_compile_cache_miss', 2), ('fxgraph_cache_miss', 1), ('async_compile_cache_hit', 1)]
graph_break []
inline_call []
stats [('calls_captured', 1), ('unique_graphs', 1)]
aot_autograd [('total', 1), ('autograd_cache_miss', 1), ('autograd_cache_saved', 1), ('ok', 1)]
inductor [('triton_bundler_save_kernel', 8), ('async_compile_cache_miss', 2), ('fxgraph_cache_miss', 1), ('async_compile_cache_hit', 1)]
graph_break []
inline_call []
stats [('calls_captured', 1), ('unique_graphs', 1)]
aot_autograd [('total', 1), ('autograd_cache_miss', 1), ('autograd_cache_saved', 1), ('ok', 1)]
inductor [('triton_bundler_save_kernel', 8), ('async_compile_cache_miss', 2), ('fxgraph_cache_miss', 1), ('async_compile_cache_hit', 1)]
graph_break []
Loading