From c7e6fcec04262f3bdf341b0a8ce5d067322702b4 Mon Sep 17 00:00:00 2001 From: Jatin Bhateja Date: Wed, 22 Apr 2026 09:27:11 +0000 Subject: [PATCH 01/10] 8382713: [VectorAPI] Perform late inlining of failed vector intrinsics --- src/hotspot/share/opto/callGenerator.cpp | 59 ++++++++++++++++++++++++ src/hotspot/share/opto/callGenerator.hpp | 4 ++ src/hotspot/share/opto/compile.cpp | 45 ++++++++++++++++++ src/hotspot/share/opto/compile.hpp | 7 +++ src/hotspot/share/opto/doCall.cpp | 4 +- 5 files changed, 118 insertions(+), 1 deletion(-) diff --git a/src/hotspot/share/opto/callGenerator.cpp b/src/hotspot/share/opto/callGenerator.cpp index 49897ca3c1763..f05e9487b38d7 100644 --- a/src/hotspot/share/opto/callGenerator.cpp +++ b/src/hotspot/share/opto/callGenerator.cpp @@ -437,6 +437,65 @@ CallGenerator* CallGenerator::for_mh_late_inline(ciMethod* caller, ciMethod* cal return cg; } +class LateInlineVectorCallGenerator : public LateInlineCallGenerator { + protected: + bool _use_fallback_generator; + CallGenerator* _inline_cg2; + + public: + LateInlineVectorCallGenerator(ciMethod* method, CallGenerator* intrinsic_cg, CallGenerator* inline_cg) : + LateInlineCallGenerator(method, intrinsic_cg) , _inline_cg2(inline_cg) { + _use_fallback_generator = false; + } + + virtual CallGenerator* inline_cg() const { + return _use_fallback_generator ? _inline_cg2 : _inline_cg; + } + virtual bool inline_fallback() const; + virtual bool is_vector_late_inline() const { return true; } + virtual void enable_fallback_generation() { + _use_fallback_generator = true; + } +}; + +bool LateInlineVectorCallGenerator::inline_fallback() const { + switch (method()->intrinsic_id()) { + case vmIntrinsics::_VectorUnaryOp: + case vmIntrinsics::_VectorBinaryOp: + case vmIntrinsics::_VectorUnaryLibOp: + case vmIntrinsics::_VectorBinaryLibOp: + case vmIntrinsics::_VectorTernaryOp: + case vmIntrinsics::_VectorSelectFromTwoVectorOp: + case vmIntrinsics::_VectorFromBitsCoerced: + case vmIntrinsics::_VectorLoadOp: + case vmIntrinsics::_VectorLoadMaskedOp: + case vmIntrinsics::_VectorStoreOp: + case vmIntrinsics::_VectorStoreMaskedOp: + case vmIntrinsics::_VectorGatherOp: + case vmIntrinsics::_VectorScatterOp: + case vmIntrinsics::_VectorReductionCoerced: + case vmIntrinsics::_VectorTest: + case vmIntrinsics::_VectorBlend: + case vmIntrinsics::_VectorCompare: + case vmIntrinsics::_VectorRearrange: + case vmIntrinsics::_VectorSelectFrom: + case vmIntrinsics::_VectorExtract: + case vmIntrinsics::_VectorInsert: + case vmIntrinsics::_VectorBroadcastInt: + case vmIntrinsics::_VectorConvert: + case vmIntrinsics::_VectorMaskOp: + case vmIntrinsics::_VectorCompressExpand: + return true; + default: + return false; + } +} + +CallGenerator* CallGenerator::for_vector_late_inline(ciMethod* m, CallGenerator* intrinsic_cg, CallGenerator* inline_cg) { + return new LateInlineVectorCallGenerator(m, intrinsic_cg, inline_cg); +} + + // Allow inlining decisions to be delayed class LateInlineVirtualCallGenerator : public VirtualCallGenerator { private: diff --git a/src/hotspot/share/opto/callGenerator.hpp b/src/hotspot/share/opto/callGenerator.hpp index 75ba6f709c07c..dc6a52c097122 100644 --- a/src/hotspot/share/opto/callGenerator.hpp +++ b/src/hotspot/share/opto/callGenerator.hpp @@ -75,6 +75,9 @@ class CallGenerator : public ArenaObj { // same but for method handle calls virtual bool is_mh_late_inline() const { return false; } virtual bool is_string_late_inline() const { return false; } + virtual bool is_vector_late_inline() const { return false; } + virtual bool inline_fallback() const { return false; } + virtual void enable_fallback_generation() { return; } virtual bool is_boxing_late_inline() const { return false; } virtual bool is_vector_reboxing_late_inline() const { return false; } virtual bool is_virtual_late_inline() const { return false; } @@ -142,6 +145,7 @@ class CallGenerator : public ArenaObj { static CallGenerator* for_late_inline(ciMethod* m, CallGenerator* inline_cg); static CallGenerator* for_mh_late_inline(ciMethod* caller, ciMethod* callee, bool input_not_const); static CallGenerator* for_string_late_inline(ciMethod* m, CallGenerator* inline_cg); + static CallGenerator* for_vector_late_inline(ciMethod* m, CallGenerator* intrinsic_cg, CallGenerator* inline_cg); static CallGenerator* for_boxing_late_inline(ciMethod* m, CallGenerator* inline_cg); static CallGenerator* for_vector_reboxing_late_inline(ciMethod* m, CallGenerator* inline_cg); static CallGenerator* for_late_inline_virtual(ciMethod* m, int vtable_index, float expected_uses); diff --git a/src/hotspot/share/opto/compile.cpp b/src/hotspot/share/opto/compile.cpp index 382c8f89a5fcb..1a9f1417b8423 100644 --- a/src/hotspot/share/opto/compile.cpp +++ b/src/hotspot/share/opto/compile.cpp @@ -416,6 +416,7 @@ void Compile::remove_useless_node(Node* dead) { remove_useless_late_inlines( &_late_inlines, dead); remove_useless_late_inlines( &_string_late_inlines, dead); remove_useless_late_inlines( &_boxing_late_inlines, dead); + remove_useless_late_inlines( &_vector_late_inlines, dead); remove_useless_late_inlines(&_vector_reboxing_late_inlines, dead); if (dead->is_CallStaticJava()) { @@ -480,6 +481,7 @@ void Compile::disconnect_useless_nodes(Unique_Node_List& useful, Unique_Node_Lis remove_useless_late_inlines( &_late_inlines, useful); remove_useless_late_inlines( &_string_late_inlines, useful); remove_useless_late_inlines( &_boxing_late_inlines, useful); + remove_useless_late_inlines( &_vector_late_inlines, useful); remove_useless_late_inlines(&_vector_reboxing_late_inlines, useful); DEBUG_ONLY(verify_graph_edges(true /*check for no_dead_code*/, root_and_safepoints);) } @@ -693,6 +695,7 @@ Compile::Compile(ciEnv* ci_env, ciMethod* target, int osr_bci, _string_late_inlines(comp_arena(), 2, 0, nullptr), _boxing_late_inlines(comp_arena(), 2, 0, nullptr), _vector_reboxing_late_inlines(comp_arena(), 2, 0, nullptr), + _vector_late_inlines(comp_arena(), 2, 0, nullptr), _late_inlines_pos(0), _has_mh_late_inlines(false), _oom(false), @@ -2016,6 +2019,32 @@ void Compile::process_for_unstable_if_traps(PhaseIterGVN& igvn) { igvn.optimize(); } +void Compile::inline_vector_fallback(PhaseIterGVN& igvn) { + while (_vector_late_inlines.length() > 0) { + CallGenerator* cg = _vector_late_inlines.pop(); + assert(cg->is_vector_late_inline(), ""); + if (cg->inline_fallback()) { + cg->enable_fallback_generation(); + cg->do_late_inline(); + } + if (failing()) return; + } + _vector_late_inlines.trunc_to(0); + + inline_incrementally_cleanup(igvn); + + while (_late_inlines.length() > 0) { + igvn_worklist()->ensure_empty(); // should be done with igvn + + while (inline_incrementally_one()) { + assert(!failing_internal() || failure_is_artificial(), "inconsistent"); + } + if (failing()) return; + + inline_incrementally_cleanup(igvn); + } +} + // StringOpts and late inlining of string methods void Compile::inline_string_calls(bool parse_time) { { @@ -2096,6 +2125,9 @@ bool Compile::inline_incrementally_one() { } else if (inlining_progress()) { _late_inlines_pos = i+1; // restore the position in case new elements were inserted print_method(PHASE_INCREMENTAL_INLINE_STEP, 3, cg->call_node()); + if (cg->is_vector_late_inline()) { + C->vector_late_inlines()->remove_if_existing(cg); + } break; // process one call site at a time } else { bool is_scheduled_for_igvn_after = C->igvn_worklist()->member(cg->call_node()); @@ -2106,6 +2138,9 @@ bool Compile::inline_incrementally_one() { // Ensure call node has not disappeared from IGVN worklist during a failed inlining attempt assert(!is_scheduled_for_igvn_before || is_scheduled_for_igvn_after, "call node removed from IGVN list during inlining pass"); cg->call_node()->set_generator(cg); + if (cg->is_vector_late_inline()) { + C->vector_late_inlines()->append_if_missing(cg); + } } } } else { @@ -2229,6 +2264,16 @@ void Compile::inline_incrementally(PhaseIterGVN& igvn) { inline_incrementally_cleanup(igvn); } + if (_vector_late_inlines.length() > 0) { + inline_vector_fallback(igvn); + + if (failing()) { + return; + } + + inline_incrementally_cleanup(igvn); + } + set_inlining_incrementally(false); } diff --git a/src/hotspot/share/opto/compile.hpp b/src/hotspot/share/opto/compile.hpp index ff0085d79deac..a6495f9bc2b13 100644 --- a/src/hotspot/share/opto/compile.hpp +++ b/src/hotspot/share/opto/compile.hpp @@ -480,6 +480,7 @@ class Compile : public Phase { GrowableArray _boxing_late_inlines; // same but for boxing operations GrowableArray _vector_reboxing_late_inlines; // same but for vector reboxing operations + GrowableArray _vector_late_inlines; // inline fallback implementation for failed intrinsics int _late_inlines_pos; // Where in the queue should the next late inlining candidate go (emulate depth first inlining) bool _has_mh_late_inlines; // Can there still be a method handle late inlining pending? @@ -508,6 +509,11 @@ class Compile : public Phase { InlinePrinter _inline_printer; public: + + GrowableArray* vector_late_inlines() { + return &_vector_late_inlines; + } + void* barrier_set_state() const { return _barrier_set_state; } InlinePrinter* inline_printer() { return &_inline_printer; } @@ -1130,6 +1136,7 @@ class Compile : public Phase { bool should_stress_inlining() { return StressIncrementalInlining && (random() % 2) == 0; } bool should_delay_inlining() { return AlwaysIncrementalInline || should_stress_inlining(); } void inline_string_calls(bool parse_time); + void inline_vector_fallback(PhaseIterGVN& igvn); void inline_boxing_calls(PhaseIterGVN& igvn); bool optimize_loops(PhaseIterGVN& igvn, LoopOptsMode mode); void remove_root_to_sfpts_edges(PhaseIterGVN& igvn); diff --git a/src/hotspot/share/opto/doCall.cpp b/src/hotspot/share/opto/doCall.cpp index d6e75f17f5012..4513c524c043d 100644 --- a/src/hotspot/share/opto/doCall.cpp +++ b/src/hotspot/share/opto/doCall.cpp @@ -166,7 +166,9 @@ CallGenerator* Compile::call_generator(ciMethod* callee, int vtable_index, bool cg_intrinsic = cg; cg = nullptr; } else if (IncrementalInline && should_delay_vector_inlining(callee, jvms)) { - return CallGenerator::for_late_inline(callee, cg); + float expected_uses = jvms->method()->scale_count(site_count, prof_factor); + CallGenerator* inline_cg = CallGenerator::for_inline(callee, expected_uses); + return CallGenerator::for_vector_late_inline(callee, cg, inline_cg); } else { return cg; } From 931d45e9b81ee938229ac997a53d45aea7ac36c3 Mon Sep 17 00:00:00 2001 From: Jatin Bhateja Date: Mon, 27 Apr 2026 05:32:56 +0000 Subject: [PATCH 02/10] Review comments resolutions --- src/hotspot/share/opto/callGenerator.cpp | 59 +++++------------------- src/hotspot/share/opto/callGenerator.hpp | 3 +- src/hotspot/share/opto/compile.cpp | 52 ++++++--------------- src/hotspot/share/opto/compile.hpp | 1 - 4 files changed, 27 insertions(+), 88 deletions(-) diff --git a/src/hotspot/share/opto/callGenerator.cpp b/src/hotspot/share/opto/callGenerator.cpp index f05e9487b38d7..cf2fa325f0f04 100644 --- a/src/hotspot/share/opto/callGenerator.cpp +++ b/src/hotspot/share/opto/callGenerator.cpp @@ -438,61 +438,24 @@ CallGenerator* CallGenerator::for_mh_late_inline(ciMethod* caller, ciMethod* cal } class LateInlineVectorCallGenerator : public LateInlineCallGenerator { - protected: - bool _use_fallback_generator; - CallGenerator* _inline_cg2; + private: + CallGenerator* _fallback_cg; public: - LateInlineVectorCallGenerator(ciMethod* method, CallGenerator* intrinsic_cg, CallGenerator* inline_cg) : - LateInlineCallGenerator(method, intrinsic_cg) , _inline_cg2(inline_cg) { - _use_fallback_generator = false; - } + LateInlineVectorCallGenerator(ciMethod* method, CallGenerator* intrinsic_cg, CallGenerator* fallback_cg) : + LateInlineCallGenerator(method, intrinsic_cg), _fallback_cg(fallback_cg) {} - virtual CallGenerator* inline_cg() const { - return _use_fallback_generator ? _inline_cg2 : _inline_cg; - } - virtual bool inline_fallback() const; virtual bool is_vector_late_inline() const { return true; } - virtual void enable_fallback_generation() { - _use_fallback_generator = true; - } -}; + virtual CallGenerator* fallback_inline_cg() const { return _fallback_cg; } -bool LateInlineVectorCallGenerator::inline_fallback() const { - switch (method()->intrinsic_id()) { - case vmIntrinsics::_VectorUnaryOp: - case vmIntrinsics::_VectorBinaryOp: - case vmIntrinsics::_VectorUnaryLibOp: - case vmIntrinsics::_VectorBinaryLibOp: - case vmIntrinsics::_VectorTernaryOp: - case vmIntrinsics::_VectorSelectFromTwoVectorOp: - case vmIntrinsics::_VectorFromBitsCoerced: - case vmIntrinsics::_VectorLoadOp: - case vmIntrinsics::_VectorLoadMaskedOp: - case vmIntrinsics::_VectorStoreOp: - case vmIntrinsics::_VectorStoreMaskedOp: - case vmIntrinsics::_VectorGatherOp: - case vmIntrinsics::_VectorScatterOp: - case vmIntrinsics::_VectorReductionCoerced: - case vmIntrinsics::_VectorTest: - case vmIntrinsics::_VectorBlend: - case vmIntrinsics::_VectorCompare: - case vmIntrinsics::_VectorRearrange: - case vmIntrinsics::_VectorSelectFrom: - case vmIntrinsics::_VectorExtract: - case vmIntrinsics::_VectorInsert: - case vmIntrinsics::_VectorBroadcastInt: - case vmIntrinsics::_VectorConvert: - case vmIntrinsics::_VectorMaskOp: - case vmIntrinsics::_VectorCompressExpand: - return true; - default: - return false; + virtual JVMState* generate(JVMState* jvms) { + Compile::current()->vector_late_inlines()->append(this); + return LateInlineCallGenerator::generate(jvms); } -} +}; -CallGenerator* CallGenerator::for_vector_late_inline(ciMethod* m, CallGenerator* intrinsic_cg, CallGenerator* inline_cg) { - return new LateInlineVectorCallGenerator(m, intrinsic_cg, inline_cg); +CallGenerator* CallGenerator::for_vector_late_inline(ciMethod* m, CallGenerator* intrinsic_cg, CallGenerator* fallback_cg) { + return new LateInlineVectorCallGenerator(m, intrinsic_cg, fallback_cg); } diff --git a/src/hotspot/share/opto/callGenerator.hpp b/src/hotspot/share/opto/callGenerator.hpp index dc6a52c097122..6f5bd8c33b5c3 100644 --- a/src/hotspot/share/opto/callGenerator.hpp +++ b/src/hotspot/share/opto/callGenerator.hpp @@ -76,8 +76,7 @@ class CallGenerator : public ArenaObj { virtual bool is_mh_late_inline() const { return false; } virtual bool is_string_late_inline() const { return false; } virtual bool is_vector_late_inline() const { return false; } - virtual bool inline_fallback() const { return false; } - virtual void enable_fallback_generation() { return; } + virtual CallGenerator* fallback_inline_cg() const { return nullptr; } virtual bool is_boxing_late_inline() const { return false; } virtual bool is_vector_reboxing_late_inline() const { return false; } virtual bool is_virtual_late_inline() const { return false; } diff --git a/src/hotspot/share/opto/compile.cpp b/src/hotspot/share/opto/compile.cpp index 1a9f1417b8423..8c218cff69ecc 100644 --- a/src/hotspot/share/opto/compile.cpp +++ b/src/hotspot/share/opto/compile.cpp @@ -2019,32 +2019,6 @@ void Compile::process_for_unstable_if_traps(PhaseIterGVN& igvn) { igvn.optimize(); } -void Compile::inline_vector_fallback(PhaseIterGVN& igvn) { - while (_vector_late_inlines.length() > 0) { - CallGenerator* cg = _vector_late_inlines.pop(); - assert(cg->is_vector_late_inline(), ""); - if (cg->inline_fallback()) { - cg->enable_fallback_generation(); - cg->do_late_inline(); - } - if (failing()) return; - } - _vector_late_inlines.trunc_to(0); - - inline_incrementally_cleanup(igvn); - - while (_late_inlines.length() > 0) { - igvn_worklist()->ensure_empty(); // should be done with igvn - - while (inline_incrementally_one()) { - assert(!failing_internal() || failure_is_artificial(), "inconsistent"); - } - if (failing()) return; - - inline_incrementally_cleanup(igvn); - } -} - // StringOpts and late inlining of string methods void Compile::inline_string_calls(bool parse_time) { { @@ -2125,9 +2099,6 @@ bool Compile::inline_incrementally_one() { } else if (inlining_progress()) { _late_inlines_pos = i+1; // restore the position in case new elements were inserted print_method(PHASE_INCREMENTAL_INLINE_STEP, 3, cg->call_node()); - if (cg->is_vector_late_inline()) { - C->vector_late_inlines()->remove_if_existing(cg); - } break; // process one call site at a time } else { bool is_scheduled_for_igvn_after = C->igvn_worklist()->member(cg->call_node()); @@ -2138,9 +2109,6 @@ bool Compile::inline_incrementally_one() { // Ensure call node has not disappeared from IGVN worklist during a failed inlining attempt assert(!is_scheduled_for_igvn_before || is_scheduled_for_igvn_after, "call node removed from IGVN list during inlining pass"); cg->call_node()->set_generator(cg); - if (cg->is_vector_late_inline()) { - C->vector_late_inlines()->append_if_missing(cg); - } } } } else { @@ -2265,13 +2233,23 @@ void Compile::inline_incrementally(PhaseIterGVN& igvn) { } if (_vector_late_inlines.length() > 0) { - inline_vector_fallback(igvn); - - if (failing()) { - return; + while (_vector_late_inlines.length() > 0) { + CallGenerator* cg = _vector_late_inlines.pop(); + CallGenerator* fallback = CallGenerator::for_late_inline(cg->method(), cg->fallback_inline_cg()); + fallback = fallback->with_call_node(cg->call_node()); + add_late_inline(fallback); } - inline_incrementally_cleanup(igvn); + while (_late_inlines.length() > 0) { + igvn_worklist()->ensure_empty(); + + while (inline_incrementally_one()) { + assert(!failing_internal() || failure_is_artificial(), "inconsistent"); + } + if (failing()) return; + + inline_incrementally_cleanup(igvn); + } } set_inlining_incrementally(false); diff --git a/src/hotspot/share/opto/compile.hpp b/src/hotspot/share/opto/compile.hpp index a6495f9bc2b13..e008461bc2480 100644 --- a/src/hotspot/share/opto/compile.hpp +++ b/src/hotspot/share/opto/compile.hpp @@ -1136,7 +1136,6 @@ class Compile : public Phase { bool should_stress_inlining() { return StressIncrementalInlining && (random() % 2) == 0; } bool should_delay_inlining() { return AlwaysIncrementalInline || should_stress_inlining(); } void inline_string_calls(bool parse_time); - void inline_vector_fallback(PhaseIterGVN& igvn); void inline_boxing_calls(PhaseIterGVN& igvn); bool optimize_loops(PhaseIterGVN& igvn, LoopOptsMode mode); void remove_root_to_sfpts_edges(PhaseIterGVN& igvn); From e779a2fef4900baa170841a848c7d89a05317dfa Mon Sep 17 00:00:00 2001 From: Jatin Bhateja Date: Tue, 28 Apr 2026 06:12:12 +0000 Subject: [PATCH 03/10] Review comments resolutions --- src/hotspot/share/opto/c2_globals.hpp | 3 +++ src/hotspot/share/opto/callGenerator.cpp | 10 ++++++--- src/hotspot/share/opto/callGenerator.hpp | 1 - src/hotspot/share/opto/compile.cpp | 26 ++++++------------------ src/hotspot/share/opto/compile.hpp | 4 ++-- 5 files changed, 18 insertions(+), 26 deletions(-) diff --git a/src/hotspot/share/opto/c2_globals.hpp b/src/hotspot/share/opto/c2_globals.hpp index dacc8ce9c261c..1ac450f1a32ab 100644 --- a/src/hotspot/share/opto/c2_globals.hpp +++ b/src/hotspot/share/opto/c2_globals.hpp @@ -837,6 +837,9 @@ product(bool, EnableVectorAggressiveReboxing, false, EXPERIMENTAL, \ "Enables aggressive reboxing of vectors") \ \ + product(bool, InlineVectorFallback, true, DIAGNOSTIC, \ + "Inline fallback implementation of failed vector intrinsics") \ + \ product(bool, UseTypeSpeculation, true, \ "Speculatively propagate types from profiles") \ \ diff --git a/src/hotspot/share/opto/callGenerator.cpp b/src/hotspot/share/opto/callGenerator.cpp index cf2fa325f0f04..e0f2bf01b52d4 100644 --- a/src/hotspot/share/opto/callGenerator.cpp +++ b/src/hotspot/share/opto/callGenerator.cpp @@ -446,11 +446,15 @@ class LateInlineVectorCallGenerator : public LateInlineCallGenerator { LateInlineCallGenerator(method, intrinsic_cg), _fallback_cg(fallback_cg) {} virtual bool is_vector_late_inline() const { return true; } - virtual CallGenerator* fallback_inline_cg() const { return _fallback_cg; } virtual JVMState* generate(JVMState* jvms) { - Compile::current()->vector_late_inlines()->append(this); - return LateInlineCallGenerator::generate(jvms); + JVMState* new_jvms = LateInlineCallGenerator::generate(jvms); + if (InlineVectorFallback) { + CallGenerator* fallback = CallGenerator::for_late_inline(method(), _fallback_cg); + fallback = fallback->with_call_node(call_node()); + Compile::current()->add_vector_late_inline(fallback); + } + return new_jvms; } }; diff --git a/src/hotspot/share/opto/callGenerator.hpp b/src/hotspot/share/opto/callGenerator.hpp index 6f5bd8c33b5c3..8db6fb83c99d8 100644 --- a/src/hotspot/share/opto/callGenerator.hpp +++ b/src/hotspot/share/opto/callGenerator.hpp @@ -76,7 +76,6 @@ class CallGenerator : public ArenaObj { virtual bool is_mh_late_inline() const { return false; } virtual bool is_string_late_inline() const { return false; } virtual bool is_vector_late_inline() const { return false; } - virtual CallGenerator* fallback_inline_cg() const { return nullptr; } virtual bool is_boxing_late_inline() const { return false; } virtual bool is_vector_reboxing_late_inline() const { return false; } virtual bool is_virtual_late_inline() const { return false; } diff --git a/src/hotspot/share/opto/compile.cpp b/src/hotspot/share/opto/compile.cpp index 8c218cff69ecc..a0385691b5cfa 100644 --- a/src/hotspot/share/opto/compile.cpp +++ b/src/hotspot/share/opto/compile.cpp @@ -2218,6 +2218,12 @@ void Compile::inline_incrementally(PhaseIterGVN& igvn) { print_method(PHASE_INCREMENTAL_INLINE_STEP, 3); if (failing()) return; + + if (_late_inlines.length() == 0 && _vector_late_inlines.length() > 0) { + while (_vector_late_inlines.length() > 0) { + add_late_inline(_vector_late_inlines.pop()); + } + } } igvn_worklist()->ensure_empty(); // should be done with igvn @@ -2232,26 +2238,6 @@ void Compile::inline_incrementally(PhaseIterGVN& igvn) { inline_incrementally_cleanup(igvn); } - if (_vector_late_inlines.length() > 0) { - while (_vector_late_inlines.length() > 0) { - CallGenerator* cg = _vector_late_inlines.pop(); - CallGenerator* fallback = CallGenerator::for_late_inline(cg->method(), cg->fallback_inline_cg()); - fallback = fallback->with_call_node(cg->call_node()); - add_late_inline(fallback); - } - - while (_late_inlines.length() > 0) { - igvn_worklist()->ensure_empty(); - - while (inline_incrementally_one()) { - assert(!failing_internal() || failure_is_artificial(), "inconsistent"); - } - if (failing()) return; - - inline_incrementally_cleanup(igvn); - } - } - set_inlining_incrementally(false); } diff --git a/src/hotspot/share/opto/compile.hpp b/src/hotspot/share/opto/compile.hpp index e008461bc2480..d201686ae1e8a 100644 --- a/src/hotspot/share/opto/compile.hpp +++ b/src/hotspot/share/opto/compile.hpp @@ -510,8 +510,8 @@ class Compile : public Phase { public: - GrowableArray* vector_late_inlines() { - return &_vector_late_inlines; + void add_vector_late_inline(CallGenerator* cg) { + _vector_late_inlines.push(cg); } void* barrier_set_state() const { return _barrier_set_state; } From 5f46f5bb76592424998c46b6361174e90a261eb0 Mon Sep 17 00:00:00 2001 From: Jatin Bhateja Date: Tue, 5 May 2026 10:39:10 +0000 Subject: [PATCH 04/10] Review comments resolutions --- src/hotspot/share/opto/c2_globals.hpp | 6 +++--- src/hotspot/share/opto/callGenerator.cpp | 17 +++++++---------- src/hotspot/share/opto/callGenerator.hpp | 2 +- src/hotspot/share/opto/compile.cpp | 16 ++++++++++++++-- src/hotspot/share/opto/doCall.cpp | 4 +--- 5 files changed, 26 insertions(+), 19 deletions(-) diff --git a/src/hotspot/share/opto/c2_globals.hpp b/src/hotspot/share/opto/c2_globals.hpp index 1ac450f1a32ab..eec979efbc0d1 100644 --- a/src/hotspot/share/opto/c2_globals.hpp +++ b/src/hotspot/share/opto/c2_globals.hpp @@ -800,6 +800,9 @@ product(bool, IncrementalInlineForceCleanup, false, DIAGNOSTIC, \ "do cleanup after every iteration of incremental inlining") \ \ + product(bool, IncrementalInlineVector, true, DIAGNOSTIC, \ + "Inline fallback implementation of failed vector intrinsics") \ + \ product(intx, LiveNodeCountInliningCutoff, 40000, \ "max number of live nodes in a method") \ range(0, max_juint / 8) \ @@ -837,9 +840,6 @@ product(bool, EnableVectorAggressiveReboxing, false, EXPERIMENTAL, \ "Enables aggressive reboxing of vectors") \ \ - product(bool, InlineVectorFallback, true, DIAGNOSTIC, \ - "Inline fallback implementation of failed vector intrinsics") \ - \ product(bool, UseTypeSpeculation, true, \ "Speculatively propagate types from profiles") \ \ diff --git a/src/hotspot/share/opto/callGenerator.cpp b/src/hotspot/share/opto/callGenerator.cpp index e0f2bf01b52d4..d67af873460f2 100644 --- a/src/hotspot/share/opto/callGenerator.cpp +++ b/src/hotspot/share/opto/callGenerator.cpp @@ -438,28 +438,25 @@ CallGenerator* CallGenerator::for_mh_late_inline(ciMethod* caller, ciMethod* cal } class LateInlineVectorCallGenerator : public LateInlineCallGenerator { - private: - CallGenerator* _fallback_cg; - public: - LateInlineVectorCallGenerator(ciMethod* method, CallGenerator* intrinsic_cg, CallGenerator* fallback_cg) : - LateInlineCallGenerator(method, intrinsic_cg), _fallback_cg(fallback_cg) {} + LateInlineVectorCallGenerator(ciMethod* method, CallGenerator* intrinsic_cg) : + LateInlineCallGenerator(method, intrinsic_cg) {} virtual bool is_vector_late_inline() const { return true; } virtual JVMState* generate(JVMState* jvms) { JVMState* new_jvms = LateInlineCallGenerator::generate(jvms); - if (InlineVectorFallback) { - CallGenerator* fallback = CallGenerator::for_late_inline(method(), _fallback_cg); - fallback = fallback->with_call_node(call_node()); + if (IncrementalInlineVector) { + CallGenerator* inline_cg = CallGenerator::for_inline(method()); + CallGenerator* fallback = CallGenerator::for_late_inline(method(), inline_cg)->with_call_node(call_node()); Compile::current()->add_vector_late_inline(fallback); } return new_jvms; } }; -CallGenerator* CallGenerator::for_vector_late_inline(ciMethod* m, CallGenerator* intrinsic_cg, CallGenerator* fallback_cg) { - return new LateInlineVectorCallGenerator(m, intrinsic_cg, fallback_cg); +CallGenerator* CallGenerator::for_vector_late_inline(ciMethod* m, CallGenerator* intrinsic_cg) { + return new LateInlineVectorCallGenerator(m, intrinsic_cg); } diff --git a/src/hotspot/share/opto/callGenerator.hpp b/src/hotspot/share/opto/callGenerator.hpp index 8db6fb83c99d8..b786f392b8a19 100644 --- a/src/hotspot/share/opto/callGenerator.hpp +++ b/src/hotspot/share/opto/callGenerator.hpp @@ -143,7 +143,7 @@ class CallGenerator : public ArenaObj { static CallGenerator* for_late_inline(ciMethod* m, CallGenerator* inline_cg); static CallGenerator* for_mh_late_inline(ciMethod* caller, ciMethod* callee, bool input_not_const); static CallGenerator* for_string_late_inline(ciMethod* m, CallGenerator* inline_cg); - static CallGenerator* for_vector_late_inline(ciMethod* m, CallGenerator* intrinsic_cg, CallGenerator* inline_cg); + static CallGenerator* for_vector_late_inline(ciMethod* m, CallGenerator* intrinsic_cg); static CallGenerator* for_boxing_late_inline(ciMethod* m, CallGenerator* inline_cg); static CallGenerator* for_vector_reboxing_late_inline(ciMethod* m, CallGenerator* inline_cg); static CallGenerator* for_late_inline_virtual(ciMethod* m, int vtable_index, float expected_uses); diff --git a/src/hotspot/share/opto/compile.cpp b/src/hotspot/share/opto/compile.cpp index a0385691b5cfa..6cca8a259e3ff 100644 --- a/src/hotspot/share/opto/compile.cpp +++ b/src/hotspot/share/opto/compile.cpp @@ -2220,9 +2220,21 @@ void Compile::inline_incrementally(PhaseIterGVN& igvn) { if (failing()) return; if (_late_inlines.length() == 0 && _vector_late_inlines.length() > 0) { - while (_vector_late_inlines.length() > 0) { - add_late_inline(_vector_late_inlines.pop()); + for (int i = 0; i < _vector_late_inlines.length(); i++) { + CallGenerator* cg = _vector_late_inlines.at(i); + // When a vector intrinsic fails, set_generator(cg) caches the + // LateInlineVectorCallGenerator on the call node to allow retries + // if IGVN optimizes the call node's inputs. If the call node is not + // on the IGVN worklist when cleanup runs, CallStaticJavaNode::Ideal + // does not fire and the cached generator persists. Once _late_inlines + // drains and we commit to the fallback here, clear the stale generator + // to prevent a subsequent IGVN pass from re-registering the intrinsic + // attempt into _late_inlines alongside the fallback, which would create + // duplicate call_node entries. + cg->call_node()->as_CallJava()->set_generator(nullptr); + add_late_inline(cg); } + _vector_late_inlines.clear(); } } diff --git a/src/hotspot/share/opto/doCall.cpp b/src/hotspot/share/opto/doCall.cpp index 4513c524c043d..af517a4a2aa8f 100644 --- a/src/hotspot/share/opto/doCall.cpp +++ b/src/hotspot/share/opto/doCall.cpp @@ -166,9 +166,7 @@ CallGenerator* Compile::call_generator(ciMethod* callee, int vtable_index, bool cg_intrinsic = cg; cg = nullptr; } else if (IncrementalInline && should_delay_vector_inlining(callee, jvms)) { - float expected_uses = jvms->method()->scale_count(site_count, prof_factor); - CallGenerator* inline_cg = CallGenerator::for_inline(callee, expected_uses); - return CallGenerator::for_vector_late_inline(callee, cg, inline_cg); + return CallGenerator::for_vector_late_inline(callee, cg); } else { return cg; } From d18bd2a3a24041843bb3bc7e914a1b9ce46b4da2 Mon Sep 17 00:00:00 2001 From: Jatin Bhateja Date: Wed, 6 May 2026 07:41:47 +0000 Subject: [PATCH 05/10] Review comments resolution --- src/hotspot/share/opto/compile.cpp | 34 +++++++++++++++++------------- src/hotspot/share/opto/compile.hpp | 1 + 2 files changed, 20 insertions(+), 15 deletions(-) diff --git a/src/hotspot/share/opto/compile.cpp b/src/hotspot/share/opto/compile.cpp index 6cca8a259e3ff..49b1e985e2fdc 100644 --- a/src/hotspot/share/opto/compile.cpp +++ b/src/hotspot/share/opto/compile.cpp @@ -2161,6 +2161,24 @@ void Compile::shuffle_late_inlines() { shuffle_array(*C, _late_inlines); } +void Compile::transfer_vector_late_inlines() { + for (int i = 0; i < _vector_late_inlines.length(); i++) { + CallGenerator* cg = _vector_late_inlines.at(i); + // When a vector intrinsic fails, set_generator(cg) caches the + // LateInlineVectorCallGenerator on the call node to allow retries + // if IGVN optimizes the call node's inputs. If the call node is not + // on the IGVN worklist when cleanup runs, CallStaticJavaNode::Ideal + // does not fire and the cached generator persists. Once _late_inlines + // drains and we commit to the fallback here, clear the stale generator + // to prevent a subsequent IGVN pass from re-registering the intrinsic + // attempt into _late_inlines alongside the fallback, which would create + // duplicate call_node entries. + cg->call_node()->as_CallJava()->set_generator(nullptr); + add_late_inline(cg); + } + _vector_late_inlines.clear(); +} + // Perform incremental inlining until bound on number of live nodes is reached void Compile::inline_incrementally(PhaseIterGVN& igvn) { TracePhase tp(_t_incrInline); @@ -2220,21 +2238,7 @@ void Compile::inline_incrementally(PhaseIterGVN& igvn) { if (failing()) return; if (_late_inlines.length() == 0 && _vector_late_inlines.length() > 0) { - for (int i = 0; i < _vector_late_inlines.length(); i++) { - CallGenerator* cg = _vector_late_inlines.at(i); - // When a vector intrinsic fails, set_generator(cg) caches the - // LateInlineVectorCallGenerator on the call node to allow retries - // if IGVN optimizes the call node's inputs. If the call node is not - // on the IGVN worklist when cleanup runs, CallStaticJavaNode::Ideal - // does not fire and the cached generator persists. Once _late_inlines - // drains and we commit to the fallback here, clear the stale generator - // to prevent a subsequent IGVN pass from re-registering the intrinsic - // attempt into _late_inlines alongside the fallback, which would create - // duplicate call_node entries. - cg->call_node()->as_CallJava()->set_generator(nullptr); - add_late_inline(cg); - } - _vector_late_inlines.clear(); + transfer_vector_late_inlines(); } } diff --git a/src/hotspot/share/opto/compile.hpp b/src/hotspot/share/opto/compile.hpp index d201686ae1e8a..8043dc4f1625e 100644 --- a/src/hotspot/share/opto/compile.hpp +++ b/src/hotspot/share/opto/compile.hpp @@ -513,6 +513,7 @@ class Compile : public Phase { void add_vector_late_inline(CallGenerator* cg) { _vector_late_inlines.push(cg); } + void transfer_vector_late_inlines(); void* barrier_set_state() const { return _barrier_set_state; } From 8171911d28371854284bf525365f959a85819952 Mon Sep 17 00:00:00 2001 From: Jatin Bhateja Date: Fri, 8 May 2026 11:03:52 +0000 Subject: [PATCH 06/10] Review comment resolution --- src/hotspot/share/opto/compile.cpp | 6 +++--- src/hotspot/share/opto/compile.hpp | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/hotspot/share/opto/compile.cpp b/src/hotspot/share/opto/compile.cpp index 49b1e985e2fdc..40ca188b668f8 100644 --- a/src/hotspot/share/opto/compile.cpp +++ b/src/hotspot/share/opto/compile.cpp @@ -2161,7 +2161,7 @@ void Compile::shuffle_late_inlines() { shuffle_array(*C, _late_inlines); } -void Compile::transfer_vector_late_inlines() { +void Compile::process_vector_late_inlines() { for (int i = 0; i < _vector_late_inlines.length(); i++) { CallGenerator* cg = _vector_late_inlines.at(i); // When a vector intrinsic fails, set_generator(cg) caches the @@ -2237,8 +2237,8 @@ void Compile::inline_incrementally(PhaseIterGVN& igvn) { if (failing()) return; - if (_late_inlines.length() == 0 && _vector_late_inlines.length() > 0) { - transfer_vector_late_inlines(); + if (_late_inlines.length() == 0) { + process_vector_late_inlines(); } } diff --git a/src/hotspot/share/opto/compile.hpp b/src/hotspot/share/opto/compile.hpp index 8043dc4f1625e..a3ee4ed6f8ef7 100644 --- a/src/hotspot/share/opto/compile.hpp +++ b/src/hotspot/share/opto/compile.hpp @@ -513,7 +513,7 @@ class Compile : public Phase { void add_vector_late_inline(CallGenerator* cg) { _vector_late_inlines.push(cg); } - void transfer_vector_late_inlines(); + void process_vector_late_inlines(); void* barrier_set_state() const { return _barrier_set_state; } From 679e444c1a3e21a076d8b258c9ced3a1046387bb Mon Sep 17 00:00:00 2001 From: Jatin Bhateja Date: Tue, 12 May 2026 03:32:01 +0000 Subject: [PATCH 07/10] Review comments resolution --- src/hotspot/share/opto/compile.cpp | 8 ++++++++ test/hotspot/jtreg/compiler/vectorapi/TestVectorTest.java | 6 +++--- .../compiler/vectorapi/VectorMaskCompareNotTest.java | 2 +- 3 files changed, 12 insertions(+), 4 deletions(-) diff --git a/src/hotspot/share/opto/compile.cpp b/src/hotspot/share/opto/compile.cpp index 40ca188b668f8..dc461f20f3bd9 100644 --- a/src/hotspot/share/opto/compile.cpp +++ b/src/hotspot/share/opto/compile.cpp @@ -2164,6 +2164,14 @@ void Compile::shuffle_late_inlines() { void Compile::process_vector_late_inlines() { for (int i = 0; i < _vector_late_inlines.length(); i++) { CallGenerator* cg = _vector_late_inlines.at(i); + ciMethod* callee = cg->method(); + + // Skip fallback inlining for callees already compiled into large nmethods. + if (callee->has_compiled_code() && + callee->inline_instructions_size() > InlineSmallCode) { + continue; + } + // When a vector intrinsic fails, set_generator(cg) caches the // LateInlineVectorCallGenerator on the call node to allow retries // if IGVN optimizes the call node's inputs. If the call node is not diff --git a/test/hotspot/jtreg/compiler/vectorapi/TestVectorTest.java b/test/hotspot/jtreg/compiler/vectorapi/TestVectorTest.java index c6329c70f6594..9cc88c32b6602 100644 --- a/test/hotspot/jtreg/compiler/vectorapi/TestVectorTest.java +++ b/test/hotspot/jtreg/compiler/vectorapi/TestVectorTest.java @@ -47,7 +47,7 @@ public static void main(String[] args) { public int call() { return 1; } @Test - @IR(failOn = {IRNode.CMP_I, IRNode.CMOVE_I}) + @IR(failOn = {IRNode.CMP_I, IRNode.CMOVE_I}, applyIf = {"IncrementalInlineVector", "false"}) @IR(counts = {IRNode.VECTOR_TEST, "1"}) public int branch(long maskLong) { var mask = VectorMask.fromLong(ByteVector.SPECIES_PREFERRED, maskLong); @@ -55,8 +55,8 @@ public int branch(long maskLong) { } @Test - @IR(failOn = {IRNode.CMP_I}) - @IR(counts = {IRNode.VECTOR_TEST, "1", IRNode.CMOVE_I, "1"}) + @IR(failOn = {IRNode.CMP_I}, applyIf = {"IncrementalInlineVector", "false"}) + @IR(counts = {IRNode.VECTOR_TEST, "1", IRNode.CMOVE_I, "1"}, applyIf = {"IncrementalInlineVector", "false"}) public int cmove(long maskLong) { var mask = VectorMask.fromLong(ByteVector.SPECIES_PREFERRED, maskLong); return mask.allTrue() ? 1 : 0; diff --git a/test/hotspot/jtreg/compiler/vectorapi/VectorMaskCompareNotTest.java b/test/hotspot/jtreg/compiler/vectorapi/VectorMaskCompareNotTest.java index 09185f63c6942..c21a6d86d7c01 100644 --- a/test/hotspot/jtreg/compiler/vectorapi/VectorMaskCompareNotTest.java +++ b/test/hotspot/jtreg/compiler/vectorapi/VectorMaskCompareNotTest.java @@ -1294,7 +1294,7 @@ public static void testCompareMaskNotDoubleNegative() { public static void main(String[] args) { TestFramework testFramework = new TestFramework(); testFramework.setDefaultWarmup(5000) - .addFlags("--add-modules=jdk.incubator.vector") + .addFlags("--add-modules=jdk.incubator.vector", "-XX:InlineSmallCode=100000") .start(); } } From dc3ffe82568265e68fc50f63b105234a4fdba04f Mon Sep 17 00:00:00 2001 From: Jatin Bhateja Date: Mon, 18 May 2026 05:00:23 +0000 Subject: [PATCH 08/10] Review comments resolution --- src/hotspot/share/opto/callGenerator.cpp | 16 +++++++++------- src/hotspot/share/opto/callGenerator.hpp | 2 +- src/hotspot/share/opto/compile.cpp | 7 ------- src/hotspot/share/opto/doCall.cpp | 7 ++++++- .../jtreg/compiler/vectorapi/TestVectorTest.java | 10 ++++++---- 5 files changed, 22 insertions(+), 20 deletions(-) diff --git a/src/hotspot/share/opto/callGenerator.cpp b/src/hotspot/share/opto/callGenerator.cpp index d67af873460f2..669ac9fa090ef 100644 --- a/src/hotspot/share/opto/callGenerator.cpp +++ b/src/hotspot/share/opto/callGenerator.cpp @@ -438,25 +438,27 @@ CallGenerator* CallGenerator::for_mh_late_inline(ciMethod* caller, ciMethod* cal } class LateInlineVectorCallGenerator : public LateInlineCallGenerator { + private: + CallGenerator* _fallback_cg; + public: - LateInlineVectorCallGenerator(ciMethod* method, CallGenerator* intrinsic_cg) : - LateInlineCallGenerator(method, intrinsic_cg) {} + LateInlineVectorCallGenerator(ciMethod* method, CallGenerator* intrinsic_cg, CallGenerator* fallback_cg) : + LateInlineCallGenerator(method, intrinsic_cg), _fallback_cg(fallback_cg) {} virtual bool is_vector_late_inline() const { return true; } virtual JVMState* generate(JVMState* jvms) { JVMState* new_jvms = LateInlineCallGenerator::generate(jvms); - if (IncrementalInlineVector) { - CallGenerator* inline_cg = CallGenerator::for_inline(method()); - CallGenerator* fallback = CallGenerator::for_late_inline(method(), inline_cg)->with_call_node(call_node()); + if (_fallback_cg != nullptr && _fallback_cg->is_parse()) { + CallGenerator* fallback = CallGenerator::for_late_inline(method(), _fallback_cg)->with_call_node(call_node()); Compile::current()->add_vector_late_inline(fallback); } return new_jvms; } }; -CallGenerator* CallGenerator::for_vector_late_inline(ciMethod* m, CallGenerator* intrinsic_cg) { - return new LateInlineVectorCallGenerator(m, intrinsic_cg); +CallGenerator* CallGenerator::for_vector_late_inline(ciMethod* m, CallGenerator* intrinsic_cg, CallGenerator* fallback_cg) { + return new LateInlineVectorCallGenerator(m, intrinsic_cg, fallback_cg); } diff --git a/src/hotspot/share/opto/callGenerator.hpp b/src/hotspot/share/opto/callGenerator.hpp index b786f392b8a19..234d2c29441c2 100644 --- a/src/hotspot/share/opto/callGenerator.hpp +++ b/src/hotspot/share/opto/callGenerator.hpp @@ -143,7 +143,7 @@ class CallGenerator : public ArenaObj { static CallGenerator* for_late_inline(ciMethod* m, CallGenerator* inline_cg); static CallGenerator* for_mh_late_inline(ciMethod* caller, ciMethod* callee, bool input_not_const); static CallGenerator* for_string_late_inline(ciMethod* m, CallGenerator* inline_cg); - static CallGenerator* for_vector_late_inline(ciMethod* m, CallGenerator* intrinsic_cg); + static CallGenerator* for_vector_late_inline(ciMethod* m, CallGenerator* intrinsic_cg, CallGenerator* fallback_cg = nullptr); static CallGenerator* for_boxing_late_inline(ciMethod* m, CallGenerator* inline_cg); static CallGenerator* for_vector_reboxing_late_inline(ciMethod* m, CallGenerator* inline_cg); static CallGenerator* for_late_inline_virtual(ciMethod* m, int vtable_index, float expected_uses); diff --git a/src/hotspot/share/opto/compile.cpp b/src/hotspot/share/opto/compile.cpp index dc461f20f3bd9..4658332b3ab3b 100644 --- a/src/hotspot/share/opto/compile.cpp +++ b/src/hotspot/share/opto/compile.cpp @@ -2164,13 +2164,6 @@ void Compile::shuffle_late_inlines() { void Compile::process_vector_late_inlines() { for (int i = 0; i < _vector_late_inlines.length(); i++) { CallGenerator* cg = _vector_late_inlines.at(i); - ciMethod* callee = cg->method(); - - // Skip fallback inlining for callees already compiled into large nmethods. - if (callee->has_compiled_code() && - callee->inline_instructions_size() > InlineSmallCode) { - continue; - } // When a vector intrinsic fails, set_generator(cg) caches the // LateInlineVectorCallGenerator on the call node to allow retries diff --git a/src/hotspot/share/opto/doCall.cpp b/src/hotspot/share/opto/doCall.cpp index af517a4a2aa8f..1564a3029bb77 100644 --- a/src/hotspot/share/opto/doCall.cpp +++ b/src/hotspot/share/opto/doCall.cpp @@ -166,7 +166,12 @@ CallGenerator* Compile::call_generator(ciMethod* callee, int vtable_index, bool cg_intrinsic = cg; cg = nullptr; } else if (IncrementalInline && should_delay_vector_inlining(callee, jvms)) { - return CallGenerator::for_vector_late_inline(callee, cg); + CallGenerator* fallback_cg = nullptr; + if (IncrementalInlineVector) { + fallback_cg = this->call_generator(callee, + vtable_index, call_does_dispatch, jvms, allow_inline, prof_factor, speculative_receiver_type, false); + } + return CallGenerator::for_vector_late_inline(callee, cg, fallback_cg); } else { return cg; } diff --git a/test/hotspot/jtreg/compiler/vectorapi/TestVectorTest.java b/test/hotspot/jtreg/compiler/vectorapi/TestVectorTest.java index 9cc88c32b6602..82f650af5a8a8 100644 --- a/test/hotspot/jtreg/compiler/vectorapi/TestVectorTest.java +++ b/test/hotspot/jtreg/compiler/vectorapi/TestVectorTest.java @@ -40,14 +40,16 @@ */ public class TestVectorTest { public static void main(String[] args) { - TestFramework.runWithFlags("--add-modules=jdk.incubator.vector"); + TestFramework.runWithFlags("--add-modules=jdk.incubator.vector", + "-XX:-IncrementalInlineVector", + "-XX:InlineSmallCode=1000000"); } @DontInline public int call() { return 1; } @Test - @IR(failOn = {IRNode.CMP_I, IRNode.CMOVE_I}, applyIf = {"IncrementalInlineVector", "false"}) + @IR(failOn = {IRNode.CMP_I, IRNode.CMOVE_I}) @IR(counts = {IRNode.VECTOR_TEST, "1"}) public int branch(long maskLong) { var mask = VectorMask.fromLong(ByteVector.SPECIES_PREFERRED, maskLong); @@ -55,8 +57,8 @@ public int branch(long maskLong) { } @Test - @IR(failOn = {IRNode.CMP_I}, applyIf = {"IncrementalInlineVector", "false"}) - @IR(counts = {IRNode.VECTOR_TEST, "1", IRNode.CMOVE_I, "1"}, applyIf = {"IncrementalInlineVector", "false"}) + @IR(failOn = {IRNode.CMP_I}) + @IR(counts = {IRNode.VECTOR_TEST, "1", IRNode.CMOVE_I, "1"}) public int cmove(long maskLong) { var mask = VectorMask.fromLong(ByteVector.SPECIES_PREFERRED, maskLong); return mask.allTrue() ? 1 : 0; From 77150d9bbb033228d147a992a053027856fd0c33 Mon Sep 17 00:00:00 2001 From: Jatin Bhateja Date: Tue, 19 May 2026 14:21:01 +0000 Subject: [PATCH 09/10] Review comments resolutions --- src/hotspot/share/opto/callGenerator.cpp | 3 ++- src/hotspot/share/opto/doCall.cpp | 16 +++++++++++----- .../jtreg/compiler/vectorapi/TestVectorTest.java | 3 +-- .../vectorapi/VectorMaskCompareNotTest.java | 3 ++- 4 files changed, 16 insertions(+), 9 deletions(-) diff --git a/src/hotspot/share/opto/callGenerator.cpp b/src/hotspot/share/opto/callGenerator.cpp index 669ac9fa090ef..9f3331ddf90d2 100644 --- a/src/hotspot/share/opto/callGenerator.cpp +++ b/src/hotspot/share/opto/callGenerator.cpp @@ -449,7 +449,8 @@ class LateInlineVectorCallGenerator : public LateInlineCallGenerator { virtual JVMState* generate(JVMState* jvms) { JVMState* new_jvms = LateInlineCallGenerator::generate(jvms); - if (_fallback_cg != nullptr && _fallback_cg->is_parse()) { + if (_fallback_cg != nullptr) { + assert(_fallback_cg->is_parse(), ""); CallGenerator* fallback = CallGenerator::for_late_inline(method(), _fallback_cg)->with_call_node(call_node()); Compile::current()->add_vector_late_inline(fallback); } diff --git a/src/hotspot/share/opto/doCall.cpp b/src/hotspot/share/opto/doCall.cpp index 1564a3029bb77..13c64ad8fce91 100644 --- a/src/hotspot/share/opto/doCall.cpp +++ b/src/hotspot/share/opto/doCall.cpp @@ -166,12 +166,18 @@ CallGenerator* Compile::call_generator(ciMethod* callee, int vtable_index, bool cg_intrinsic = cg; cg = nullptr; } else if (IncrementalInline && should_delay_vector_inlining(callee, jvms)) { - CallGenerator* fallback_cg = nullptr; - if (IncrementalInlineVector) { - fallback_cg = this->call_generator(callee, - vtable_index, call_does_dispatch, jvms, allow_inline, prof_factor, speculative_receiver_type, false); + if (IncrementalInlineVector && allow_inline) { + // Try to late inline fallback implementation if intrinsification attempt fails. + CallGenerator* fallback_cg = call_generator(callee, vtable_index, call_does_dispatch, jvms, + true /*allow_inline*/, prof_factor, + speculative_receiver_type, false /*allow_intrinsics*/); + if (fallback_cg != nullptr && fallback_cg->is_parse()) { + return CallGenerator::for_vector_late_inline(callee, cg, fallback_cg); + } + // Fallback not inlineable by regular heuristics; fall through. } - return CallGenerator::for_vector_late_inline(callee, cg, fallback_cg); + // Don't try to inline fallback implementation. + return CallGenerator::for_late_inline(callee, cg); } else { return cg; } diff --git a/test/hotspot/jtreg/compiler/vectorapi/TestVectorTest.java b/test/hotspot/jtreg/compiler/vectorapi/TestVectorTest.java index 82f650af5a8a8..9b1a95dfb1440 100644 --- a/test/hotspot/jtreg/compiler/vectorapi/TestVectorTest.java +++ b/test/hotspot/jtreg/compiler/vectorapi/TestVectorTest.java @@ -41,8 +41,7 @@ public class TestVectorTest { public static void main(String[] args) { TestFramework.runWithFlags("--add-modules=jdk.incubator.vector", - "-XX:-IncrementalInlineVector", - "-XX:InlineSmallCode=1000000"); + "-XX:-IncrementalInlineVector"); } @DontInline diff --git a/test/hotspot/jtreg/compiler/vectorapi/VectorMaskCompareNotTest.java b/test/hotspot/jtreg/compiler/vectorapi/VectorMaskCompareNotTest.java index c21a6d86d7c01..935363f8526a2 100644 --- a/test/hotspot/jtreg/compiler/vectorapi/VectorMaskCompareNotTest.java +++ b/test/hotspot/jtreg/compiler/vectorapi/VectorMaskCompareNotTest.java @@ -1294,7 +1294,8 @@ public static void testCompareMaskNotDoubleNegative() { public static void main(String[] args) { TestFramework testFramework = new TestFramework(); testFramework.setDefaultWarmup(5000) - .addFlags("--add-modules=jdk.incubator.vector", "-XX:InlineSmallCode=100000") + .addFlags("--add-modules=jdk.incubator.vector", + "-XX:InlineSmallCode=1000000") .start(); } } From ae4a37354003bb491df1f41d701119eb0c233e0d Mon Sep 17 00:00:00 2001 From: Jatin Bhateja Date: Thu, 21 May 2026 06:20:53 +0000 Subject: [PATCH 10/10] Review comments resolutions --- src/hotspot/share/opto/callGenerator.cpp | 11 +++++------ src/hotspot/share/opto/callGenerator.hpp | 2 +- .../compiler/vectorapi/VectorCompareWithZeroTest.java | 3 ++- .../compiler/vectorapi/VectorMaskCompareNotTest.java | 2 +- 4 files changed, 9 insertions(+), 9 deletions(-) diff --git a/src/hotspot/share/opto/callGenerator.cpp b/src/hotspot/share/opto/callGenerator.cpp index 9f3331ddf90d2..0e56eb67a1ad7 100644 --- a/src/hotspot/share/opto/callGenerator.cpp +++ b/src/hotspot/share/opto/callGenerator.cpp @@ -443,17 +443,16 @@ class LateInlineVectorCallGenerator : public LateInlineCallGenerator { public: LateInlineVectorCallGenerator(ciMethod* method, CallGenerator* intrinsic_cg, CallGenerator* fallback_cg) : - LateInlineCallGenerator(method, intrinsic_cg), _fallback_cg(fallback_cg) {} + LateInlineCallGenerator(method, intrinsic_cg), _fallback_cg(fallback_cg) { + assert(_fallback_cg != nullptr && _fallback_cg->is_parse(), ""); + } virtual bool is_vector_late_inline() const { return true; } virtual JVMState* generate(JVMState* jvms) { JVMState* new_jvms = LateInlineCallGenerator::generate(jvms); - if (_fallback_cg != nullptr) { - assert(_fallback_cg->is_parse(), ""); - CallGenerator* fallback = CallGenerator::for_late_inline(method(), _fallback_cg)->with_call_node(call_node()); - Compile::current()->add_vector_late_inline(fallback); - } + CallGenerator* fallback = CallGenerator::for_late_inline(method(), _fallback_cg)->with_call_node(call_node()); + Compile::current()->add_vector_late_inline(fallback); return new_jvms; } }; diff --git a/src/hotspot/share/opto/callGenerator.hpp b/src/hotspot/share/opto/callGenerator.hpp index 234d2c29441c2..2dd102b9e0601 100644 --- a/src/hotspot/share/opto/callGenerator.hpp +++ b/src/hotspot/share/opto/callGenerator.hpp @@ -143,7 +143,7 @@ class CallGenerator : public ArenaObj { static CallGenerator* for_late_inline(ciMethod* m, CallGenerator* inline_cg); static CallGenerator* for_mh_late_inline(ciMethod* caller, ciMethod* callee, bool input_not_const); static CallGenerator* for_string_late_inline(ciMethod* m, CallGenerator* inline_cg); - static CallGenerator* for_vector_late_inline(ciMethod* m, CallGenerator* intrinsic_cg, CallGenerator* fallback_cg = nullptr); + static CallGenerator* for_vector_late_inline(ciMethod* m, CallGenerator* intrinsic_cg, CallGenerator* fallback_cg); static CallGenerator* for_boxing_late_inline(ciMethod* m, CallGenerator* inline_cg); static CallGenerator* for_vector_reboxing_late_inline(ciMethod* m, CallGenerator* inline_cg); static CallGenerator* for_late_inline_virtual(ciMethod* m, int vtable_index, float expected_uses); diff --git a/test/hotspot/jtreg/compiler/vectorapi/VectorCompareWithZeroTest.java b/test/hotspot/jtreg/compiler/vectorapi/VectorCompareWithZeroTest.java index 26e159fb768d0..0b1bfd2c80209 100644 --- a/test/hotspot/jtreg/compiler/vectorapi/VectorCompareWithZeroTest.java +++ b/test/hotspot/jtreg/compiler/vectorapi/VectorCompareWithZeroTest.java @@ -253,7 +253,8 @@ public static void testLongVectorUnsignedCondition() { public static void main(String[] args) { TestFramework testFramework = new TestFramework(); testFramework.setDefaultWarmup(10000) - .addFlags("--add-modules=jdk.incubator.vector") + .addFlags("--add-modules=jdk.incubator.vector", + "-XX:-IncrementalInlineVector") .addFlags("-XX:UseSVE=0") .start(); } diff --git a/test/hotspot/jtreg/compiler/vectorapi/VectorMaskCompareNotTest.java b/test/hotspot/jtreg/compiler/vectorapi/VectorMaskCompareNotTest.java index 935363f8526a2..4aeb5ba36b08c 100644 --- a/test/hotspot/jtreg/compiler/vectorapi/VectorMaskCompareNotTest.java +++ b/test/hotspot/jtreg/compiler/vectorapi/VectorMaskCompareNotTest.java @@ -1295,7 +1295,7 @@ public static void main(String[] args) { TestFramework testFramework = new TestFramework(); testFramework.setDefaultWarmup(5000) .addFlags("--add-modules=jdk.incubator.vector", - "-XX:InlineSmallCode=1000000") + "-XX:-IncrementalInlineVector") .start(); } }