late dead stores removal

src/coreclr/jit/compiler.cpp

@@ -5131,6 +5131,9 @@ void Compiler::compCompile(void** methodCodePtr, uint32_t* methodCodeSize, JitFl
     // Expand stack allocated arrays
     DoPhase(this, PHASE_EXPAND_STACK_ARR, &Compiler::fgExpandStackArrayAllocations);
 
+    // Remove dead stores
+    DoPhase(this, PHASE_LATE_DEAD_STORES, &Compiler::fgLateDeadStoresRemoval);
+
     // Insert GC Polls
     DoPhase(this, PHASE_INSERT_GC_POLLS, &Compiler::fgInsertGCPolls);
 

src/coreclr/jit/compiler.h

@@ -1086,6 +1086,7 @@ class LclVarDsc
                                // appearance count (computed during address-exposed analysis)
                                // that fgMakeOutgoingStructArgCopy consults during global morph
                                // to determine if eliding its copy is legal.
+    unsigned short m_lvAddrTakenCnt;
 
     weight_t m_lvRefCntWtd; // weighted reference count
 
@@ -1094,6 +1095,9 @@ class LclVarDsc
     void           incLvRefCnt(unsigned short delta, RefCountState state = RCS_NORMAL);
     void           setLvRefCnt(unsigned short newValue, RefCountState state = RCS_NORMAL);
     void           incLvRefCntSaturating(unsigned short delta, RefCountState state = RCS_NORMAL);
+    unsigned short lvAddrTakenCnt() const;
+    void           setLvAddrTakenCnt(unsigned short newValue);
+    void           incLvAddrTakenCntSaturating(unsigned short delta);
 
     weight_t lvRefCntWtd(RefCountState state = RCS_NORMAL) const;
     void     incLvRefCntWtd(weight_t delta, RefCountState state = RCS_NORMAL);
@@ -6108,6 +6112,8 @@ class Compiler
 
     PhaseStatus fgExpandStackArrayAllocations();
     bool fgExpandStackArrayAllocation(BasicBlock* pBlock, Statement* stmt, GenTreeCall* call);
+
+    PhaseStatus fgLateDeadStoresRemoval();
 
     PhaseStatus fgVNBasedIntrinsicExpansion();
     bool fgVNBasedIntrinsicExpansionForCall(BasicBlock** pBlock, Statement* stmt, GenTreeCall* call);
@@ -10147,6 +10153,7 @@ class Compiler
 
     bool fgLocalVarLivenessDone = false; // Note that this one is used outside of debug.
     bool fgLocalVarLivenessChanged;
+    bool fgLocalVarTrackingSetChanged;
     bool fgIsDoingEarlyLiveness         = false;
     bool fgDidEarlyLiveness             = false;
     bool compPostImportationCleanupDone = false;

src/coreclr/jit/compiler.hpp

@@ -4856,6 +4856,40 @@ inline void LclVarDsc::setLvRefCntWtd(weight_t newValue, RefCountState state)
     m_lvRefCntWtd = newValue;
 }
 
+//------------------------------------------------------------------------------
+// incLvAddrTakenCntSaturating: increment address taken count for this local var (with saturating semantics)
+//
+// Arguments:
+//    delta: the amount of the address taken
+//
+inline void LclVarDsc::incLvAddrTakenCntSaturating(unsigned short delta)
+{
+    int newAddrTakenCnt = m_lvAddrTakenCnt + delta;
+    m_lvAddrTakenCnt    = static_cast<unsigned short>(min(USHRT_MAX, newAddrTakenCnt));
+}
+
+//------------------------------------------------------------------------------
+// setLvAddrTakenCnt: set address taken count for this local var
+//
+// Arguments:
+//    newValue: the desired new address taken count
+//
+inline void LclVarDsc::setLvAddrTakenCnt(unsigned short newValue)
+{
+    m_lvAddrTakenCnt = newValue;
+}
+
+//------------------------------------------------------------------------------
+// lvAddrTakenCnt: address taken count for this local var
+//
+// Return Value:
+//    Addr taken count for the local.
+
+inline unsigned short LclVarDsc::lvAddrTakenCnt() const
+{
+    return m_lvAddrTakenCnt;
+}
+
 //------------------------------------------------------------------------------
 // compCanHavePatchpoints: return true if patchpoints are supported in this
 //   method.

src/coreclr/jit/compphases.h

@@ -110,6 +110,7 @@ CompPhaseNameMacro(PHASE_EXPAND_STATIC_INIT,         "Expand static init",
 CompPhaseNameMacro(PHASE_EXPAND_CASTS,               "Expand casts",                   false, -1, true)
 CompPhaseNameMacro(PHASE_EXPAND_TLS,                 "Expand TLS access",              false, -1, true)
 CompPhaseNameMacro(PHASE_EXPAND_STACK_ARR,           "Expand stack array allocation",  false, -1, true)
+CompPhaseNameMacro(PHASE_LATE_DEAD_STORES,           "Late dead stores removal",       false, -1, true)
 CompPhaseNameMacro(PHASE_INSERT_GC_POLLS,            "Insert GC Polls",                false, -1, true)
 CompPhaseNameMacro(PHASE_CREATE_THROW_HELPERS,       "Create throw helper blocks",     false, -1, true)
 CompPhaseNameMacro(PHASE_DETERMINE_FIRST_COLD_BLOCK, "Determine first cold block",     false, -1, true)

src/coreclr/jit/liveness.cpp

@@ -139,7 +139,8 @@ void Compiler::fgLocalVarLiveness()
     // Initialize the per-block var sets.
     fgInitBlockVarSets();
 
-    fgLocalVarLivenessChanged = false;
+    fgLocalVarLivenessChanged    = false;
+    fgLocalVarTrackingSetChanged = false;
     do
     {
         /* Figure out use/def info for all basic blocks */
@@ -150,7 +151,13 @@ void Compiler::fgLocalVarLiveness()
 
         fgStmtRemoved = false;
         fgInterBlockLocalVarLiveness();
-    } while (fgStmtRemoved && fgLocalVarLivenessChanged);
+
+        if (fgStmtRemoved && fgLocalVarTrackingSetChanged)
+        {
+            fgLocalVarLivenessInit();
+            fgInitBlockVarSets();
+        }
+    } while (fgStmtRemoved && (fgLocalVarLivenessChanged || fgLocalVarTrackingSetChanged));
 
     EndPhase(PHASE_LCLVARLIVENESS_INTERBLOCK);
 }
@@ -231,6 +238,28 @@ void Compiler::fgPerNodeLocalVarLiveness(GenTree* tree)
 
                 fgMarkUseDef(tree->AsLclVarCommon());
             }
+            else
+            {
+                LclVarDsc* const varDsc = lvaGetDesc(tree->AsLclVarCommon());
+                varDsc->incLvAddrTakenCntSaturating(1);
+                if (varDsc->lvIsStructField)
+                {
+                    // Promoted field, increase count for the parent lclVar.
+                    //
+                    lvaGetDesc(varDsc->lvParentLcl)->incLvAddrTakenCntSaturating(1);
+                }
+
+                if (varDsc->lvPromoted)
+                {
+                    // Promoted struct, increase count for each promoted field.
+                    //
+                    for (unsigned childLclNum = varDsc->lvFieldLclStart;
+                         childLclNum < varDsc->lvFieldLclStart + varDsc->lvFieldCnt; ++childLclNum)
+                    {
+                        lvaGetDesc(childLclNum)->incLvAddrTakenCntSaturating(1);
+                    }
+                }
+            }
             break;
 
         case GT_IND:
@@ -372,6 +401,11 @@ void Compiler::fgPerBlockLocalVarLiveness()
     VarSetOps::AssignNoCopy(this, fgCurUseSet, VarSetOps::MakeEmpty(this));
     VarSetOps::AssignNoCopy(this, fgCurDefSet, VarSetOps::MakeEmpty(this));
 
+    for (unsigned lclNum = 0; lclNum < lvaCount; lclNum++)
+    {
+        lvaGetDesc(lclNum)->setLvAddrTakenCnt(0);
+    }
+
     // GC Heap and ByrefExposed can share states unless we see a def of byref-exposed
     // memory that is not a GC Heap def.
     byrefStatesMatchGcHeapStates = true;
@@ -1763,6 +1797,25 @@ bool Compiler::fgRemoveDeadStore(GenTree**           pTree,
             // No side effects - remove the whole statement from the block->bbStmtList.
             fgRemoveStmt(compCurBB, compCurStmt);
 
+            for (GenTree* removedTree = compCurStmt->GetRootNode(); removedTree != nullptr;
+                 removedTree          = removedTree->gtPrev)
+            {
+                if (removedTree->IsLclVarAddr())
+                {
+                    unsigned       lclNum    = removedTree->AsLclVarCommon()->GetLclNum();
+                    LclVarDsc*     lclVarDsc = lvaGetDesc(lclNum);
+                    unsigned short oldCnt    = lclVarDsc->lvAddrTakenCnt();
+                    assert(oldCnt > 0);
+                    lclVarDsc->setLvAddrTakenCnt(oldCnt - 1);
+                    if (lclVarDsc->lvAddrTakenCnt() == 0)
+                    {
+                        lclVarDsc->CleanAddressExposed();
+                        fgLocalVarTrackingSetChanged = true;
+                        JITDUMP("V%02u no longer address exposed\n", lclNum);
+                    }
+                }
+            }
+
             // Since we removed it do not process the rest (i.e. "data") of the statement
             // variables in "data" will not be marked as live, so we get the benefit of
             // propagating dead variables up the chain
@@ -2229,7 +2282,8 @@ PhaseStatus Compiler::fgEarlyLiveness()
     // Initialize the per-block var sets.
     fgInitBlockVarSets();
 
-    fgLocalVarLivenessChanged = false;
+    fgLocalVarLivenessChanged    = false;
+    fgLocalVarTrackingSetChanged = false;
     do
     {
         /* Figure out use/def info for all basic blocks */
@@ -2240,9 +2294,34 @@ PhaseStatus Compiler::fgEarlyLiveness()
 
         fgStmtRemoved = false;
         fgInterBlockLocalVarLiveness();
-    } while (fgStmtRemoved && fgLocalVarLivenessChanged);
+
+        if (fgStmtRemoved && fgLocalVarTrackingSetChanged)
+        {
+            lvaSortByRefCount();
+            fgInitBlockVarSets();
+        }
+
+    } while (fgStmtRemoved && (fgLocalVarLivenessChanged || fgLocalVarTrackingSetChanged));
 
     fgIsDoingEarlyLiveness = false;
     fgDidEarlyLiveness     = true;
     return PhaseStatus::MODIFIED_EVERYTHING;
 }
+
+PhaseStatus Compiler::fgLateDeadStoresRemoval()
+{
+    if (!opts.OptimizationEnabled())
+    {
+        return PhaseStatus::MODIFIED_NOTHING;
+    }
+
+    fgModified = false;
+    fgDfsBlocksAndRemove();
+    fgLocalVarLiveness();
+    if (fgModified)
+    {
+        fgUpdateFlowGraphPhase();
+    }
+    fgInvalidateDfsTree();
+    return PhaseStatus::MODIFIED_EVERYTHING;
+}