LLVM: lib/CodeGen/StaticDataSplitter.cpp Source File

LLVM 22.0.0git
StaticDataSplitter.cpp
Go to the documentation of this file.
1//===- StaticDataSplitter.cpp ---------------------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// The pass uses branch profile data to assign hotness based section qualifiers
10// for the following types of static data:
11// - Jump tables
12// - Module-internal global variables
13// - Constant pools
14//
15// For the original RFC of this pass please see
16// https://discourse.llvm.org/t/rfc-profile-guided-static-data-partitioning/83744
17
18#include "llvm/ADT/Statistic.h"
19#include "llvm/Analysis/ProfileSummaryInfo.h"
20#include "llvm/Analysis/StaticDataProfileInfo.h"
21#include "llvm/CodeGen/MBFIWrapper.h"
22#include "llvm/CodeGen/MachineBasicBlock.h"
23#include "llvm/CodeGen/MachineBlockFrequencyInfo.h"
24#include "llvm/CodeGen/MachineBranchProbabilityInfo.h"
25#include "llvm/CodeGen/MachineConstantPool.h"
26#include "llvm/CodeGen/MachineFunction.h"
27#include "llvm/CodeGen/MachineFunctionPass.h"
28#include "llvm/CodeGen/MachineJumpTableInfo.h"
29#include "llvm/CodeGen/Passes.h"
30#include "llvm/IR/GlobalVariable.h"
31#include "llvm/InitializePasses.h"
32#include "llvm/Pass.h"
33#include "llvm/Target/TargetLoweringObjectFile.h"
34
35using namespace llvm;
36
37 #define DEBUG_TYPE "static-data-splitter"
38
39 STATISTIC(NumHotJumpTables, "Number of hot jump tables seen.");
40 STATISTIC(NumColdJumpTables, "Number of cold jump tables seen.");
41 STATISTIC(NumUnknownJumpTables,
42 "Number of jump tables with unknown hotness. They are from functions "
43 "without profile information.");
44
46 const MachineBranchProbabilityInfo *MBPI = nullptr;
47 const MachineBlockFrequencyInfo *MBFI = nullptr;
48 const ProfileSummaryInfo *PSI = nullptr;
49 StaticDataProfileInfo *SDPI = nullptr;
50
51 // If the global value is a local linkage global variable, return it.
52 // Otherwise, return nullptr.
53 const GlobalVariable *getLocalLinkageGlobalVariable(const GlobalValue *GV);
54
55 // Returns true if the global variable is in one of {.rodata, .bss, .data,
56 // .data.rel.ro} sections.
57 bool inStaticDataSection(const GlobalVariable &GV, const TargetMachine &TM);
58
59 // Returns the constant if the operand refers to a global variable or constant
60 // that gets lowered to static data sections. Otherwise, return nullptr.
61 const Constant *getConstant(const MachineOperand &Op, const TargetMachine &TM,
62 const MachineConstantPool *MCP);
63
64 // Use profiles to partition static data.
65 bool partitionStaticDataWithProfiles(MachineFunction &MF);
66
67 // Update LLVM statistics for a machine function with profiles.
68 void updateStatsWithProfiles(const MachineFunction &MF);
69
70 // Update LLVM statistics for a machine function without profiles.
71 void updateStatsWithoutProfiles(const MachineFunction &MF);
72
73 void annotateStaticDataWithoutProfiles(const MachineFunction &MF);
74
75public:
76 static char ID;
77
81
82 StringRef getPassName() const override { return "Static Data Splitter"; }
83
84 void getAnalysisUsage(AnalysisUsage &AU) const override {
90 // This pass does not modify any required analysis results except
91 // StaticDataProfileInfoWrapperPass, but StaticDataProfileInfoWrapperPass
92 // is made an immutable pass that it won't be re-scheduled by pass manager
93 // anyway. So mark setPreservesAll() here for faster compile time.
94 AU.setPreservesAll();
95 }
96
97 bool runOnMachineFunction(MachineFunction &MF) override;
98};
99
104
106 .getStaticDataProfileInfo();
107
108 const bool ProfileAvailable = PSI && PSI->hasProfileSummary() && MBFI &&
110
111 if (!ProfileAvailable) {
112 annotateStaticDataWithoutProfiles(MF);
113 updateStatsWithoutProfiles(MF);
114 return false;
115 }
116
117 bool Changed = partitionStaticDataWithProfiles(MF);
118
119 updateStatsWithProfiles(MF);
120 return Changed;
121}
122
123const Constant *
124StaticDataSplitter::getConstant(const MachineOperand &Op,
125 const TargetMachine &TM,
126 const MachineConstantPool *MCP) {
127 if (!Op.isGlobal() && !Op.isCPI())
128 return nullptr;
129
130 if (Op.isGlobal()) {
131 // Find global variables with local linkage.
132 const GlobalVariable *GV = getLocalLinkageGlobalVariable(Op.getGlobal());
133 // Skip those not eligible for annotation or not in static data sections.
134 if (!GV || !llvm::memprof::IsAnnotationOK(*GV) ||
135 !inStaticDataSection(*GV, TM))
136 return nullptr;
137 return GV;
138 }
139 assert(Op.isCPI() && "Op must be constant pool index in this branch");
140 int CPI = Op.getIndex();
141 if (CPI == -1)
142 return nullptr;
143
144 assert(MCP != nullptr && "Constant pool info is not available.");
145 const MachineConstantPoolEntry &CPE = MCP->getConstants()[CPI];
146
147 if (CPE.isMachineConstantPoolEntry())
148 return nullptr;
149
150 return CPE.Val.ConstVal;
151}
152
153bool StaticDataSplitter::partitionStaticDataWithProfiles(MachineFunction &MF) {
154 // If any of the static data (jump tables, global variables, constant pools)
155 // are captured by the analysis, set `Changed` to true. Note this pass won't
156 // invalidate any analysis pass (see `getAnalysisUsage` above), so the main
157 // purpose of tracking and conveying the change (to pass manager) is
158 // informative as opposed to invalidating any analysis results. As an example
159 // of where this information is useful, `PMDataManager::dumpPassInfo` will
160 // only dump pass info if a local change happens, otherwise a pass appears as
161 // "skipped".
162 bool Changed = false;
163
164 MachineJumpTableInfo *MJTI = MF.getJumpTableInfo();
165
166 // Jump table could be used by either terminating instructions or
167 // non-terminating ones, so we walk all instructions and use
168 // `MachineOperand::isJTI()` to identify jump table operands.
169 // Similarly, `MachineOperand::isCPI()` is used to identify constant pool
170 // usages in the same loop.
171 for (const auto &MBB : MF) {
172 std::optional<uint64_t> Count = MBFI->getBlockProfileCount(&MBB);
173 for (const MachineInstr &I : MBB) {
174 for (const MachineOperand &Op : I.operands()) {
175 if (!Op.isJTI() && !Op.isGlobal() && !Op.isCPI())
176 continue;
177
178 if (Op.isJTI()) {
179 assert(MJTI != nullptr && "Jump table info is not available.");
180 const int JTI = Op.getIndex();
181 // This is not a source block of jump table.
182 if (JTI == -1)
183 continue;
184
185 auto Hotness = MachineFunctionDataHotness::Hot;
186
187 // Hotness is based on source basic block hotness.
188 // TODO: PSI APIs are about instruction hotness. Introduce API for
189 // data access hotness.
190 if (Count && PSI->isColdCount(*Count))
191 Hotness = MachineFunctionDataHotness::Cold;
192
193 Changed |= MJTI->updateJumpTableEntryHotness(JTI, Hotness);
194 } else if (const Constant *C =
195 getConstant(Op, MF.getTarget(), MF.getConstantPool())) {
196 SDPI->addConstantProfileCount(C, Count);
197 Changed = true;
198 }
199 }
200 }
201 }
202 return Changed;
203}
204
205const GlobalVariable *
206StaticDataSplitter::getLocalLinkageGlobalVariable(const GlobalValue *GV) {
207 // LLVM IR Verifier requires that a declaration must have valid declaration
208 // linkage, and local linkages are not among the valid ones. So there is no
209 // need to check GV is not a declaration here.
210 return (GV && GV->hasLocalLinkage()) ? dyn_cast<GlobalVariable>(GV) : nullptr;
211}
212
213bool StaticDataSplitter::inStaticDataSection(const GlobalVariable &GV,
214 const TargetMachine &TM) {
215
216 SectionKind Kind = TargetLoweringObjectFile::getKindForGlobal(&GV, TM);
217 return Kind.isData() || Kind.isReadOnly() || Kind.isReadOnlyWithRel() ||
218 Kind.isBSS();
219}
220
221void StaticDataSplitter::updateStatsWithProfiles(const MachineFunction &MF) {
222 if (!AreStatisticsEnabled())
223 return;
224
225 if (const MachineJumpTableInfo *MJTI = MF.getJumpTableInfo()) {
226 for (const auto &JumpTable : MJTI->getJumpTables()) {
227 if (JumpTable.Hotness == MachineFunctionDataHotness::Hot) {
228 ++NumHotJumpTables;
229 } else {
230 assert(JumpTable.Hotness == MachineFunctionDataHotness::Cold &&
231 "A jump table is either hot or cold when profile information is "
232 "available.");
233 ++NumColdJumpTables;
234 }
235 }
236 }
237}
238
239void StaticDataSplitter::annotateStaticDataWithoutProfiles(
240 const MachineFunction &MF) {
241 for (const auto &MBB : MF)
242 for (const MachineInstr &I : MBB)
243 for (const MachineOperand &Op : I.operands())
244 if (const Constant *C =
245 getConstant(Op, MF.getTarget(), MF.getConstantPool()))
246 SDPI->addConstantProfileCount(C, std::nullopt);
247}
248
249void StaticDataSplitter::updateStatsWithoutProfiles(const MachineFunction &MF) {
250 if (!AreStatisticsEnabled())
251 return;
252
253 if (const MachineJumpTableInfo *MJTI = MF.getJumpTableInfo()) {
254 NumUnknownJumpTables += MJTI->getJumpTables().size();
255 }
256}
257
258char StaticDataSplitter::ID = 0;
259
260 INITIALIZE_PASS_BEGIN(StaticDataSplitter, DEBUG_TYPE, "Split static data",
261 false, false)
262INITIALIZE_PASS_DEPENDENCY(MachineBranchProbabilityInfoWrapperPass)
263INITIALIZE_PASS_DEPENDENCY(MachineBlockFrequencyInfoWrapperPass)
264INITIALIZE_PASS_DEPENDENCY(ProfileSummaryInfoWrapperPass)
265INITIALIZE_PASS_DEPENDENCY(StaticDataProfileInfoWrapperPass)
266 INITIALIZE_PASS_END(StaticDataSplitter, DEBUG_TYPE, "Split static data", false,
267 false)
268
assert(UImm &&(UImm !=~static_cast< T >(0)) &&"Invalid immediate!")
MachineBasicBlock & MBB
#define DEBUG_TYPE
I
#define I(x, y, z)
Definition MD5.cpp:58
This file declares the MachineConstantPool class which is an abstract constant pool to keep track of ...
#define INITIALIZE_PASS_DEPENDENCY(depName)
Definition PassSupport.h:42
#define INITIALIZE_PASS_END(passName, arg, name, cfg, analysis)
Definition PassSupport.h:44
#define INITIALIZE_PASS_BEGIN(passName, arg, name, cfg, analysis)
Definition PassSupport.h:39
This file defines the 'Statistic' class, which is designed to be an easy way to expose various metric...
#define STATISTIC(VARNAME, DESC)
Definition Statistic.h:171
StringRef getPassName() const override
getPassName - Return a nice clean name for a pass.
void getAnalysisUsage(AnalysisUsage &AU) const override
getAnalysisUsage - Subclasses that override getAnalysisUsage must call this.
bool runOnMachineFunction(MachineFunction &MF) override
runOnMachineFunction - This method must be overloaded to perform the desired machine code transformat...
Represent the analysis usage information of a pass.
AnalysisUsage & addRequired()
void setPreservesAll()
Set by analyses that do not transform their input at all.
This is an important base class in LLVM.
Definition Constant.h:43
bool hasProfileData(bool IncludeSynthetic=false) const
Return true if the function is annotated with profile data.
Definition Function.h:334
bool hasLocalLinkage() const
Definition GlobalValue.h:530
MachineBlockFrequencyInfo pass uses BlockFrequencyInfoImpl implementation to estimate machine basic b...
union llvm::MachineConstantPoolEntry::@004270020304201266316354007027341142157160323045 Val
The constant itself.
bool isMachineConstantPoolEntry() const
isMachineConstantPoolEntry - Return true if the MachineConstantPoolEntry is indeed a target specific ...
The MachineConstantPool class keeps track of constants referenced by a function which must be spilled...
const std::vector< MachineConstantPoolEntry > & getConstants() const
MachineFunctionPass - This class adapts the FunctionPass interface to allow convenient creation of pa...
void getAnalysisUsage(AnalysisUsage &AU) const override
getAnalysisUsage - Subclasses that override getAnalysisUsage must call this.
Function & getFunction()
Return the LLVM function that this machine code represents.
const MachineJumpTableInfo * getJumpTableInfo() const
getJumpTableInfo - Return the jump table info object for the current function.
LLVM_ABI bool updateJumpTableEntryHotness(size_t JTI, MachineFunctionDataHotness Hotness)
const std::vector< MachineJumpTableEntry > & getJumpTables() const
MachineOperand class - Representation of each machine instruction operand.
static LLVM_ABI PassRegistry * getPassRegistry()
getPassRegistry - Access the global registry object, which is automatically initialized at applicatio...
AnalysisType & getAnalysis() const
getAnalysis<AnalysisType>() - This function is used by subclasses to get to the analysis information ...
An analysis pass based on legacy pass manager to deliver ProfileSummaryInfo.
Analysis providing profile information.
This wraps the StaticDataProfileInfo object as an immutable pass, for a backend pass to operate on.
A class that holds the constants that represent static data and their profile information and provide...
StringRef - Represent a constant reference to a string, i.e.
Definition StringRef.h:55
static SectionKind getKindForGlobal(const GlobalObject *GO, const TargetMachine &TM)
Classify the specified global variable into a set of target independent categories embodied in Sectio...
Primary interface to the complete machine description for the target machine.
Definition TargetMachine.h:83
Changed
@ C
The default llvm calling convention, compatible with C.
Definition CallingConv.h:34
@ JumpTable
Definition ISDOpcodes.h:91
bool IsAnnotationOK(const GlobalVariable &GV)
Returns true if the annotation kind of the global variable GV is AnnotationOK.
This is an optimization pass for GlobalISel generic memory operations.
Definition AddressRanges.h:18
decltype(auto) dyn_cast(const From &Val)
dyn_cast<X> - Return the argument parameter cast to the specified type.
Definition Casting.h:643
LLVM_ABI void initializeStaticDataSplitterPass(PassRegistry &)
LLVM_ABI MachineFunctionPass * createStaticDataSplitterPass()
createStaticDataSplitterPass - This is a machine-function pass that categorizes static data hotness u...
FunctionAddr VTableAddr Count
Definition InstrProf.h:139
LLVM_ABI bool AreStatisticsEnabled()
Check if statistics are enabled.
Definition Statistic.cpp:139
DWARFExpression::Operation Op
Generated on for LLVM by doxygen 1.14.0

AltStyle によって変換されたページ (->オリジナル) /