PostgreSQL Source Code git master
Data Structures | Macros | Typedefs | Functions | Variables
llvmjit_inline.cpp File Reference
#include "postgres.h"
#include "jit/llvmjit.h"
#include <fcntl.h>
#include <sys/mman.h>
#include <sys/stat.h>
#include <sys/types.h>
#include <unistd.h>
#include "common/string.h"
#include "miscadmin.h"
#include "storage/fd.h"
#include <llvm-c/Core.h>
#include <llvm-c/BitReader.h>
#include <llvm/ADT/SetVector.h>
#include <llvm/ADT/StringSet.h>
#include <llvm/ADT/StringMap.h>
#include <llvm/Analysis/ModuleSummaryAnalysis.h>
#include <llvm/Bitcode/BitcodeReader.h>
#include <llvm/IR/Attributes.h>
#include <llvm/IR/DebugInfo.h>
#include <llvm/IR/IntrinsicInst.h>
#include <llvm/IR/IRBuilder.h>
#include <llvm/IR/ModuleSummaryIndex.h>
#include <llvm/Linker/IRMover.h>
#include <llvm/Support/ManagedStatic.h>
#include <llvm/Support/MemoryBuffer.h>
Include dependency graph for llvmjit_inline.cpp:

Go to the source code of this file.

Data Structures

struct   InlineWorkListItem
 
struct   FunctionInlineState
 

Macros

#define  ilog(...)   (void) 0
 
#define  starts_with   startswith
 

Typedefs

typedef llvm::SmallVector< llvm::ModuleSummaryIndex *, 2 >  InlineSearchPath
 
typedef struct InlineWorkListItem  InlineWorkListItem
 
typedef llvm::SmallVector< InlineWorkListItem, 128 >  InlineWorkList
 
typedef struct FunctionInlineState  FunctionInlineState
 
typedef llvm::StringMap< FunctionInlineState >  FunctionInlineStates
 
typedef llvm::StringMap< llvm::StringSet<> >  ImportMapTy
 
typedef llvm::StringMap< std::unique_ptr< llvm::Module > >  ModuleCache
 
typedef llvm::StringMap< std::unique_ptr< llvm::ModuleSummaryIndex > >  SummaryCache
 

Functions

static std::unique_ptr< ImportMapTy >  llvm_build_inline_plan (LLVMContextRef lc, llvm::Module *mod)
 
static void  llvm_execute_inline_plan (llvm::Module *mod, ImportMapTy *globalsToInline)
 
static llvm::Module *  load_module_cached (LLVMContextRef c, llvm::StringRef modPath)
 
static std::unique_ptr< llvm::Module >  load_module (LLVMContextRef c, llvm::StringRef Identifier)
 
static std::unique_ptr< llvm::ModuleSummaryIndex >  llvm_load_summary (llvm::StringRef path)
 
static llvm::Function *  create_redirection_function (std::unique_ptr< llvm::Module > &importMod, llvm::Function *F, llvm::StringRef Name)
 
static bool  function_inlinable (llvm::Function &F, int threshold, FunctionInlineStates &functionState, InlineWorkList &worklist, InlineSearchPath &searchpath, llvm::SmallPtrSet< const llvm::Function *, 8 > &visitedFunctions, int &running_instcount, llvm::StringSet<> &importVars)
 
static void  function_references (llvm::Function &F, int &running_instcount, llvm::SmallPtrSet< llvm::GlobalVariable *, 8 > &referencedVars, llvm::SmallPtrSet< llvm::Function *, 8 > &referencedFunctions)
 
static void  add_module_to_inline_search_path (InlineSearchPath &path, llvm::StringRef modpath)
 
static llvm::SmallVector< llvm::GlobalValueSummary *, 1 >  summaries_for_guid (const InlineSearchPath &path, llvm::GlobalValue::GUID guid)
 
void  llvm_inline_reset_caches (void)
 
void  llvm_inline (LLVMModuleRef M)
 

Variables

const float  inline_cost_decay_factor = 0.5
 
const int  inline_initial_cost = 150
 
llvm::ManagedStatic< ModuleCache >  module_cache
 
llvm::ManagedStatic< SummaryCache >  summary_cache
 

Macro Definition Documentation

◆ ilog

#define ilog (   ... )    (void) 0

Definition at line 147 of file llvmjit_inline.cpp.

◆ starts_with

#define starts_with   startswith

Typedef Documentation

◆ FunctionInlineState

typedef struct FunctionInlineState FunctionInlineState

◆ FunctionInlineStates

typedef llvm::StringMap<FunctionInlineState> FunctionInlineStates

Definition at line 90 of file llvmjit_inline.cpp.

◆ ImportMapTy

typedef llvm::StringMap<llvm::StringSet<> > ImportMapTy

Definition at line 96 of file llvmjit_inline.cpp.

◆ InlineSearchPath

typedef llvm::SmallVector<llvm::ModuleSummaryIndex *, 2> InlineSearchPath

Definition at line 67 of file llvmjit_inline.cpp.

◆ InlineWorkList

typedef llvm::SmallVector<InlineWorkListItem, 128> InlineWorkList

Definition at line 77 of file llvmjit_inline.cpp.

◆ InlineWorkListItem

typedef struct InlineWorkListItem InlineWorkListItem

◆ ModuleCache

typedef llvm::StringMap<std::unique_ptr<llvm::Module> > ModuleCache

Definition at line 106 of file llvmjit_inline.cpp.

◆ SummaryCache

typedef llvm::StringMap<std::unique_ptr<llvm::ModuleSummaryIndex> > SummaryCache

Definition at line 108 of file llvmjit_inline.cpp.

Function Documentation

◆ add_module_to_inline_search_path()

static void add_module_to_inline_search_path ( InlineSearchPath &  path,
llvm::StringRef  modpath 
)
static

Definition at line 797 of file llvmjit_inline.cpp.

798{
799 /* only extension in libdir are candidates for inlining for now */
800#if LLVM_VERSION_MAJOR < 16
801#define starts_with startswith
802#endif
803 if (!modpath.starts_with("$libdir/"))
804 return;
805
806 /* if there's no match, attempt to load */
807 auto it = summary_cache->find(modpath);
808 if (it == summary_cache->end())
809 {
810 std::string path(modpath);
811 path = path.replace(0, strlen("$libdir"), std::string(pkglib_path) + "/bitcode");
812 path += ".index.bc";
813 (*summary_cache)[modpath] = llvm_load_summary(path);
814 it = summary_cache->find(modpath);
815 }
816
817 Assert(it != summary_cache->end());
818
819 /* if the entry isn't NULL, it's validly loaded */
820 if (it->second)
821 searchpath.push_back(it->second.get());
822}
char pkglib_path[MAXPGPATH]
Definition: globals.c:82
Assert(PointerIsAligned(start, uint64))
static std::unique_ptr< llvm::ModuleSummaryIndex > llvm_load_summary(llvm::StringRef path)
llvm::ManagedStatic< SummaryCache > summary_cache
char string[11]
Definition: preproc-type.c:52

References Assert(), llvm_load_summary(), pkglib_path, and summary_cache.

Referenced by llvm_build_inline_plan().

◆ create_redirection_function()

static llvm::Function * create_redirection_function ( std::unique_ptr< llvm::Module > &  importMod,
llvm::Function *  F,
llvm::StringRef  Name 
)
static

Definition at line 850 of file llvmjit_inline.cpp.

853{
854 typedef llvm::GlobalValue::LinkageTypes LinkageTypes;
855
856 llvm::LLVMContext &Context = F->getContext();
857 llvm::IRBuilder<> Builder(Context);
858 llvm::Function *AF;
859 llvm::BasicBlock *BB;
860 llvm::CallInst *fwdcall;
861
862 AF = llvm::Function::Create(F->getFunctionType(),
863 LinkageTypes::AvailableExternallyLinkage,
864 Name, importMod.get());
865 BB = llvm::BasicBlock::Create(Context, "entry", AF);
866
867 Builder.SetInsertPoint(BB);
868 fwdcall = Builder.CreateCall(F, &*AF->arg_begin());
869 fwdcall->addFnAttr(llvm::Attribute::AlwaysInline);
870 Builder.CreateRet(fwdcall);
871
872 return AF;
873}
F
#define F(X, Y, Z)
Definition: md5.c:60
Definition: c.h:746

References F.

Referenced by llvm_execute_inline_plan().

◆ function_inlinable()

static bool function_inlinable ( llvm::Function &  F,
int  threshold,
FunctionInlineStates &  functionState,
InlineWorkList &  worklist,
InlineSearchPath &  searchpath,
llvm::SmallPtrSet< const llvm::Function *, 8 > &  visitedFunctions,
int &  running_instcount,
llvm::StringSet<> &  importVars 
)
static

Definition at line 572 of file llvmjit_inline.cpp.

580{
581 int subThreshold = threshold * inline_cost_decay_factor;
582 llvm::SmallPtrSet<llvm::GlobalVariable *, 8> referencedVars;
583 llvm::SmallPtrSet<llvm::Function *, 8> referencedFunctions;
584
585 /* can't rely on what may be inlined */
586 if (F.isInterposable())
587 return false;
588
589 /*
590 * Can't rely on function being present. Alternatively we could create a
591 * static version of these functions?
592 */
593 if (F.hasAvailableExternallyLinkage())
594 return false;
595
596 ilog(DEBUG1, "checking inlinability of %s", F.getName().data());
597
598 if (F.materialize())
599 elog(FATAL, "failed to materialize metadata");
600
601 if (F.getAttributes().hasFnAttr(llvm::Attribute::NoInline))
602 {
603 ilog(DEBUG1, "ineligibile to import %s due to noinline",
604 F.getName().data());
605 return false;
606 }
607
608 function_references(F, running_instcount, referencedVars, referencedFunctions);
609
610 for (llvm::GlobalVariable* rv: referencedVars)
611 {
612 if (rv->materialize())
613 elog(FATAL, "failed to materialize metadata");
614
615 /*
616 * Don't inline functions that access thread local variables. That
617 * doesn't work on current LLVM releases (but might in future).
618 */
619 if (rv->isThreadLocal())
620 {
621 ilog(DEBUG1, "cannot inline %s due to thread-local variable %s",
622 F.getName().data(), rv->getName().data());
623 return false;
624 }
625
626 /*
627 * Never want to inline externally visible vars, cheap enough to
628 * reference.
629 */
630 if (rv->hasExternalLinkage() || rv->hasAvailableExternallyLinkage())
631 continue;
632
633 /*
634 * If variable is file-local, we need to inline it, to be able to
635 * inline the function itself. Can't do that if the variable can be
636 * modified, because they'd obviously get out of sync.
637 *
638 * XXX: Currently not a problem, but there'd be problems with
639 * nontrivial initializers if they were allowed for postgres.
640 */
641 if (!rv->isConstant())
642 {
643 ilog(DEBUG1, "cannot inline %s due to uncloneable variable %s",
644 F.getName().data(), rv->getName().data());
645 return false;
646 }
647
648 ilog(DEBUG1, "memorizing global var %s linkage %d for inlining",
649 rv->getName().data(), (int)rv->getLinkage());
650
651 importVars.insert(rv->getName());
652 /* small cost attributed to each cloned global */
653 running_instcount += 5;
654 }
655
656 visitedFunctions.insert(&F);
657
658 /*
659 * Check referenced functions. Check whether used static ones are
660 * inlinable, and remember external ones for inlining.
661 */
662 for (llvm::Function* referencedFunction: referencedFunctions)
663 {
664 llvm::StringSet<> recImportVars;
665
666 if (referencedFunction->materialize())
667 elog(FATAL, "failed to materialize metadata");
668
669 if (referencedFunction->isIntrinsic())
670 continue;
671
672 /* if already visited skip, otherwise remember */
673 if (!visitedFunctions.insert(referencedFunction).second)
674 continue;
675
676 /*
677 * We don't inline external functions directly here, instead we put
678 * them on the worklist if appropriate and check them from
679 * llvm_build_inline_plan().
680 */
681 if (referencedFunction->hasExternalLinkage())
682 {
683 llvm::StringRef funcName = referencedFunction->getName();
684
685 /*
686 * Don't bother checking for inlining if remaining cost budget is
687 * very small.
688 */
689 if (subThreshold < 5)
690 continue;
691
692 auto it = functionStates.find(funcName);
693 if (it == functionStates.end())
694 {
695 FunctionInlineState inlineState;
696
697 inlineState.costLimit = subThreshold;
698 inlineState.processed = false;
699 inlineState.inlined = false;
700 inlineState.allowReconsidering = false;
701
702 functionStates[funcName] = inlineState;
703 worklist.push_back({funcName, searchpath});
704
705 ilog(DEBUG1,
706 "considering extern function %s at %d for inlining",
707 funcName.data(), subThreshold);
708 }
709 else if (!it->second.inlined &&
710 (!it->second.processed || it->second.allowReconsidering) &&
711 it->second.costLimit < subThreshold)
712 {
713 /*
714 * Update inlining threshold if higher. Need to re-queue
715 * to be processed if already processed with lower
716 * threshold.
717 */
718 if (it->second.processed)
719 {
720 ilog(DEBUG1,
721 "reconsidering extern function %s at %d for inlining, increasing from %d",
722 funcName.data(), subThreshold, it->second.costLimit);
723
724 it->second.processed = false;
725 it->second.allowReconsidering = false;
726 worklist.push_back({funcName, searchpath});
727 }
728 it->second.costLimit = subThreshold;
729 }
730 continue;
731 }
732
733 /* can't rely on what may be inlined */
734 if (referencedFunction->isInterposable())
735 return false;
736
737 if (!function_inlinable(*referencedFunction,
738 subThreshold,
739 functionStates,
740 worklist,
741 searchpath,
742 visitedFunctions,
743 running_instcount,
744 recImportVars))
745 {
746 ilog(DEBUG1,
747 "cannot inline %s due to required function %s not being inlinable",
748 F.getName().data(), referencedFunction->getName().data());
749 return false;
750 }
751
752 /* import referenced function itself */
753 importVars.insert(referencedFunction->getName());
754
755 /* import referenced function and its dependents */
756 for (auto& recImportVar : recImportVars)
757 importVars.insert(recImportVar.first());
758 }
759
760 return true;
761}
#define FATAL
Definition: elog.h:41
#define DEBUG1
Definition: elog.h:30
#define elog(elevel,...)
Definition: elog.h:226
const float inline_cost_decay_factor
static void function_references(llvm::Function &F, int &running_instcount, llvm::SmallPtrSet< llvm::GlobalVariable *, 8 > &referencedVars, llvm::SmallPtrSet< llvm::Function *, 8 > &referencedFunctions)
#define ilog(...)
static bool function_inlinable(llvm::Function &F, int threshold, FunctionInlineStates &functionState, InlineWorkList &worklist, InlineSearchPath &searchpath, llvm::SmallPtrSet< const llvm::Function *, 8 > &visitedFunctions, int &running_instcount, llvm::StringSet<> &importVars)

References FunctionInlineState::allowReconsidering, FunctionInlineState::costLimit, DEBUG1, elog, F, FATAL, function_inlinable(), function_references(), ilog, inline_cost_decay_factor, FunctionInlineState::inlined, and FunctionInlineState::processed.

Referenced by function_inlinable(), and llvm_build_inline_plan().

◆ function_references()

static void function_references ( llvm::Function &  F,
int &  running_instcount,
llvm::SmallPtrSet< llvm::GlobalVariable *, 8 > &  referencedVars,
llvm::SmallPtrSet< llvm::Function *, 8 > &  referencedFunctions 
)
static

Definition at line 515 of file llvmjit_inline.cpp.

519{
520 llvm::SmallPtrSet<const llvm::User *, 32> Visited;
521
522 for (llvm::BasicBlock &BB : F)
523 {
524 for (llvm::Instruction &I : BB)
525 {
526 if (llvm::isa<llvm::DbgInfoIntrinsic>(I))
527 continue;
528
529 llvm::SmallVector<llvm::User *, 8> Worklist;
530 Worklist.push_back(&I);
531
532 running_instcount++;
533
534 while (!Worklist.empty()) {
535 llvm::User *U = Worklist.pop_back_val();
536
537 /* visited before */
538 if (!Visited.insert(U).second)
539 continue;
540
541 for (auto &OI : U->operands()) {
542 llvm::User *Operand = llvm::dyn_cast<llvm::User>(OI);
543 if (!Operand)
544 continue;
545 if (llvm::isa<llvm::BlockAddress>(Operand))
546 continue;
547 if (auto *GV = llvm::dyn_cast<llvm::GlobalVariable>(Operand)) {
548 referencedVars.insert(GV);
549 if (GV->hasInitializer())
550 Worklist.push_back(GV->getInitializer());
551 continue;
552 }
553 if (auto *CF = llvm::dyn_cast<llvm::Function>(Operand)) {
554 referencedFunctions.insert(CF);
555 continue;
556 }
557 Worklist.push_back(Operand);
558 }
559 }
560 }
561 }
562}
I
#define I(X, Y, Z)
Definition: md5.c:63

References F, and I.

Referenced by function_inlinable().

◆ llvm_build_inline_plan()

static std::unique_ptr< ImportMapTy > llvm_build_inline_plan ( LLVMContextRef  lc,
llvm::Module *  mod 
)
static

Definition at line 183 of file llvmjit_inline.cpp.

184{
185 std::unique_ptr<ImportMapTy> globalsToInline(new ImportMapTy());
186 FunctionInlineStates functionStates;
187 InlineWorkList worklist;
188
189 InlineSearchPath defaultSearchPath;
190
191 /* attempt to add module to search path */
192 add_module_to_inline_search_path(defaultSearchPath, "$libdir/postgres");
193 /* if postgres isn't available, no point continuing */
194 if (defaultSearchPath.empty())
195 return nullptr;
196
197 /*
198 * Start inlining with current references to external functions by putting
199 * them on the inlining worklist. If, during inlining of those, new extern
200 * functions need to be inlined, they'll also be put there, with a lower
201 * priority.
202 */
203 for (const llvm::Function &funcDecl : mod->functions())
204 {
205 InlineWorkListItem item = {};
206 FunctionInlineState inlineState = {};
207
208 /* already has a definition */
209 if (!funcDecl.isDeclaration())
210 continue;
211
212 /* llvm provides implementation */
213 if (funcDecl.isIntrinsic())
214 continue;
215
216 item.symbolName = funcDecl.getName();
217 item.searchpath = defaultSearchPath;
218 worklist.push_back(item);
219 inlineState.costLimit = inline_initial_cost;
220 inlineState.processed = false;
221 inlineState.inlined = false;
222 inlineState.allowReconsidering = false;
223 functionStates[funcDecl.getName()] = inlineState;
224 }
225
226 /*
227 * Iterate over pending worklist items, look them up in index, check
228 * whether they should be inlined.
229 */
230 while (!worklist.empty())
231 {
232 InlineWorkListItem item = worklist.pop_back_val();
233 llvm::StringRef symbolName = item.symbolName;
234 char *cmodname;
235 char *cfuncname;
236 FunctionInlineState &inlineState = functionStates[symbolName];
237 llvm::GlobalValue::GUID funcGUID;
238
239 llvm_split_symbol_name(symbolName.data(), &cmodname, &cfuncname);
240
241#if LLVM_VERSION_MAJOR >= 21
242 funcGUID = llvm::GlobalValue::getGUIDAssumingExternalLinkage(cfuncname);
243#else
244 funcGUID = llvm::GlobalValue::getGUID(cfuncname);
245#endif
246
247 /* already processed */
248 if (inlineState.processed)
249 continue;
250
251
252 if (cmodname)
254
255 /*
256 * Iterate over all known definitions of function, via the index. Then
257 * look up module(s), check if function actually is defined (there
258 * could be hash conflicts).
259 */
260 for (const auto &gvs : summaries_for_guid(item.searchpath, funcGUID))
261 {
262 const llvm::FunctionSummary *fs;
263 llvm::StringRef modPath = gvs->modulePath();
264 llvm::Module *defMod;
265 llvm::Function *funcDef;
266
267 fs = llvm::cast<llvm::FunctionSummary>(gvs);
268
269 if (gvs->notEligibleToImport())
270 {
271 ilog(DEBUG1, "ineligibile to import %s due to summary",
272 symbolName.data());
273 continue;
274 }
275
276 if ((int) fs->instCount() > inlineState.costLimit)
277 {
278 ilog(DEBUG1, "ineligibile to import %s due to early threshold: %u vs %u",
279 symbolName.data(), fs->instCount(), inlineState.costLimit);
280 inlineState.allowReconsidering = true;
281 continue;
282 }
283
284 defMod = load_module_cached(lc, modPath);
285 if (defMod->materializeMetadata())
286 elog(FATAL, "failed to materialize metadata");
287
288 funcDef = defMod->getFunction(cfuncname);
289
290 /*
291 * This can happen e.g. in case of a hash collision of the
292 * function's name.
293 */
294 if (!funcDef)
295 continue;
296
297 if (funcDef->materialize())
298 elog(FATAL, "failed to materialize metadata");
299
300 Assert(!funcDef->isDeclaration());
301 Assert(funcDef->hasExternalLinkage());
302
303 llvm::StringSet<> importVars;
304 llvm::SmallPtrSet<const llvm::Function *, 8> visitedFunctions;
305 int running_instcount = 0;
306
307 /*
308 * Check whether function, and objects it depends on, are
309 * inlinable.
310 */
311 if (function_inlinable(*funcDef,
312 inlineState.costLimit,
313 functionStates,
314 worklist,
315 item.searchpath,
316 visitedFunctions,
317 running_instcount,
318 importVars))
319 {
320 /*
321 * Check whether function and all its dependencies are too
322 * big. Dependencies already counted for other functions that
323 * will get inlined are not counted again. While this make
324 * things somewhat order dependent, I can't quite see a point
325 * in a different behaviour.
326 */
327 if (running_instcount > inlineState.costLimit)
328 {
329 ilog(DEBUG1, "skipping inlining of %s due to late threshold %d vs %d",
330 symbolName.data(), running_instcount, inlineState.costLimit);
331 inlineState.allowReconsidering = true;
332 continue;
333 }
334
335 ilog(DEBUG1, "inline top function %s total_instcount: %d, partial: %d",
336 symbolName.data(), running_instcount, fs->instCount());
337
338 /* import referenced function itself */
339 importVars.insert(symbolName);
340
341 {
342 llvm::StringSet<> &modGlobalsToInline = (*globalsToInline)[modPath];
343 for (auto& importVar : importVars)
344 modGlobalsToInline.insert(importVar.first());
345 Assert(modGlobalsToInline.size() > 0);
346 }
347
348 /* mark function as inlined */
349 inlineState.inlined = true;
350
351 /*
352 * Found definition to inline, don't look for further
353 * potential definitions.
354 */
355 break;
356 }
357 else
358 {
359 ilog(DEBUG1, "had to skip inlining %s",
360 symbolName.data());
361
362 /* It's possible there's another definition that's inlinable. */
363 }
364 }
365
366 /*
367 * Signal that we're done with symbol, whether successful (inlined =
368 * true above) or not.
369 */
370 inlineState.processed = true;
371 }
372
373 return globalsToInline;
374}
void llvm_split_symbol_name(const char *name, char **modname, char **funcname)
Definition: llvmjit.c:1047
llvm::SmallVector< InlineWorkListItem, 128 > InlineWorkList
const int inline_initial_cost
llvm::SmallVector< llvm::ModuleSummaryIndex *, 2 > InlineSearchPath
static llvm::SmallVector< llvm::GlobalValueSummary *, 1 > summaries_for_guid(const InlineSearchPath &path, llvm::GlobalValue::GUID guid)
llvm::StringMap< llvm::StringSet<> > ImportMapTy
static void add_module_to_inline_search_path(InlineSearchPath &path, llvm::StringRef modpath)
static llvm::Module * load_module_cached(LLVMContextRef c, llvm::StringRef modPath)
llvm::StringMap< FunctionInlineState > FunctionInlineStates
llvm::StringRef symbolName
llvm::SmallVector< llvm::ModuleSummaryIndex *, 2 > searchpath

References add_module_to_inline_search_path(), FunctionInlineState::allowReconsidering, Assert(), FunctionInlineState::costLimit, DEBUG1, elog, FATAL, function_inlinable(), ilog, inline_initial_cost, FunctionInlineState::inlined, llvm_split_symbol_name(), load_module_cached(), FunctionInlineState::processed, InlineWorkListItem::searchpath, summaries_for_guid(), and InlineWorkListItem::symbolName.

Referenced by llvm_inline().

◆ llvm_execute_inline_plan()

static void llvm_execute_inline_plan ( llvm::Module *  mod,
ImportMapTy *  globalsToInline 
)
static

Definition at line 381 of file llvmjit_inline.cpp.

382{
383 llvm::IRMover Mover(*mod);
384
385 for (const auto& toInline : *globalsToInline)
386 {
387 const llvm::StringRef& modPath = toInline.first();
388 const llvm::StringSet<>& modGlobalsToInline = toInline.second;
389 llvm::SetVector<llvm::GlobalValue *> GlobalsToImport;
390
391 Assert(module_cache->count(modPath));
392 std::unique_ptr<llvm::Module> importMod(std::move((*module_cache)[modPath]));
393 module_cache->erase(modPath);
394
395 if (modGlobalsToInline.empty())
396 continue;
397
398 for (auto &glob: modGlobalsToInline)
399 {
400 llvm::StringRef SymbolName = glob.first();
401 char *modname;
402 char *funcname;
403
404 llvm_split_symbol_name(SymbolName.data(), &modname, &funcname);
405
406 llvm::GlobalValue *valueToImport = importMod->getNamedValue(funcname);
407
408 if (!valueToImport)
409 elog(FATAL, "didn't refind value %s to import", SymbolName.data());
410
411 /*
412 * For functions (global vars are only inlined if already static),
413 * mark imported variables as being clones from other
414 * functions. That a) avoids symbol conflicts b) allows the
415 * optimizer to perform inlining.
416 */
417 if (llvm::isa<llvm::Function>(valueToImport))
418 {
419 llvm::Function *F = llvm::dyn_cast<llvm::Function>(valueToImport);
420 typedef llvm::GlobalValue::LinkageTypes LinkageTypes;
421
422 /*
423 * Per-function info isn't necessarily stripped yet, as the
424 * module is lazy-loaded when stripped above.
425 */
426 llvm::stripDebugInfo(*F);
427
428 /*
429 * If the to-be-imported function is one referenced including
430 * its module name, create a tiny inline function that just
431 * forwards the call. One might think a GlobalAlias would do
432 * the trick, but a) IRMover doesn't override a declaration
433 * with an alias pointing to a definition (instead renaming
434 * it), b) Aliases can't be AvailableExternally.
435 */
436 if (modname)
437 {
438 llvm::Function *AF;
439
440 AF = create_redirection_function(importMod, F, SymbolName);
441
442 GlobalsToImport.insert(AF);
443 llvm::stripDebugInfo(*AF);
444 }
445
446 if (valueToImport->hasExternalLinkage())
447 {
448 valueToImport->setLinkage(LinkageTypes::AvailableExternallyLinkage);
449 }
450 }
451
452 GlobalsToImport.insert(valueToImport);
453 ilog(DEBUG1, "performing import of %s %s",
454 modPath.data(), SymbolName.data());
455
456 }
457
458 if (Mover.move(std::move(importMod), GlobalsToImport.getArrayRef(),
459 [](llvm::GlobalValue &, llvm::IRMover::ValueAdder) {},
460 /*IsPerformingImport=*/false))
461 elog(FATAL, "function import failed with linker error");
462 }
463}
#define funcname
Definition: indent_codes.h:69
static llvm::Function * create_redirection_function(std::unique_ptr< llvm::Module > &importMod, llvm::Function *F, llvm::StringRef Name)
llvm::ManagedStatic< ModuleCache > module_cache

References Assert(), create_redirection_function(), DEBUG1, elog, F, FATAL, funcname, ilog, llvm_split_symbol_name(), and module_cache.

Referenced by llvm_inline().

◆ llvm_inline()

void llvm_inline ( LLVMModuleRef  M )

Definition at line 167 of file llvmjit_inline.cpp.

168{
169 LLVMContextRef lc = LLVMGetModuleContext(M);
170 llvm::Module *mod = llvm::unwrap(M);
171
172 std::unique_ptr<ImportMapTy> globalsToInline = llvm_build_inline_plan(lc, mod);
173 if (!globalsToInline)
174 return;
175 llvm_execute_inline_plan(mod, globalsToInline.get());
176}
static std::unique_ptr< ImportMapTy > llvm_build_inline_plan(LLVMContextRef lc, llvm::Module *mod)
static void llvm_execute_inline_plan(llvm::Module *mod, ImportMapTy *globalsToInline)

References llvm_build_inline_plan(), and llvm_execute_inline_plan().

Referenced by llvm_compile_module().

◆ llvm_inline_reset_caches()

void llvm_inline_reset_caches ( void  )

Definition at line 156 of file llvmjit_inline.cpp.

157{
158 module_cache->clear();
159 summary_cache->clear();
160}

References module_cache, and summary_cache.

Referenced by llvm_recreate_llvm_context().

◆ llvm_load_summary()

static std::unique_ptr< llvm::ModuleSummaryIndex > llvm_load_summary ( llvm::StringRef  path )
static

Definition at line 768 of file llvmjit_inline.cpp.

769{
770 llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer> > MBOrErr =
771 llvm::MemoryBuffer::getFile(path);
772
773 if (std::error_code EC = MBOrErr.getError())
774 {
775 ilog(DEBUG1, "failed to open %s: %s", path.data(),
776 EC.message().c_str());
777 }
778 else
779 {
780 llvm::MemoryBufferRef ref(*MBOrErr.get().get());
781
782 llvm::Expected<std::unique_ptr<llvm::ModuleSummaryIndex> > IndexOrErr =
783 llvm::getModuleSummaryIndex(ref);
784 if (IndexOrErr)
785 return std::move(IndexOrErr.get());
786 elog(FATAL, "failed to load summary \"%s\": %s",
787 path.data(),
788 toString(IndexOrErr.takeError()).c_str());
789 }
790 return nullptr;
791}

References DEBUG1, elog, FATAL, and ilog.

Referenced by add_module_to_inline_search_path().

◆ load_module()

static std::unique_ptr< llvm::Module > load_module ( LLVMContextRef  c,
llvm::StringRef  Identifier 
)
static

Definition at line 485 of file llvmjit_inline.cpp.

486{
487 LLVMMemoryBufferRef buf;
488 LLVMModuleRef mod;
489 char path[MAXPGPATH];
490 char *msg;
491
492 snprintf(path, MAXPGPATH,"%s/bitcode/%s", pkglib_path, Identifier.data());
493
494 if (LLVMCreateMemoryBufferWithContentsOfFile(path, &buf, &msg))
495 elog(FATAL, "failed to open bitcode file \"%s\": %s",
496 path, msg);
497 if (LLVMGetBitcodeModuleInContext2(lc, buf, &mod))
498 elog(FATAL, "failed to parse bitcode in file \"%s\"", path);
499
500 /*
501 * Currently there's no use in more detailed debug info for JITed
502 * code. Until that changes, not much point in wasting memory and cycles
503 * on processing debuginfo.
504 */
505 llvm::StripDebugInfo(*llvm::unwrap(mod));
506
507 return std::unique_ptr<llvm::Module>(llvm::unwrap(mod));
508}
#define MAXPGPATH
static char * buf
Definition: pg_test_fsync.c:72
#define snprintf
Definition: port.h:239

References buf, elog, FATAL, MAXPGPATH, pkglib_path, and snprintf.

Referenced by load_module_cached().

◆ load_module_cached()

static llvm::Module * load_module_cached ( LLVMContextRef  c,
llvm::StringRef  modPath 
)
static

Definition at line 472 of file llvmjit_inline.cpp.

473{
474 auto it = module_cache->find(modPath);
475 if (it == module_cache->end())
476 {
477 it = module_cache->insert(
478 std::make_pair(modPath, load_module(lc, modPath))).first;
479 }
480
481 return it->second.get();
482}
static std::unique_ptr< llvm::Module > load_module(LLVMContextRef c, llvm::StringRef Identifier)

References load_module(), and module_cache.

Referenced by llvm_build_inline_plan().

◆ summaries_for_guid()

static llvm::SmallVector< llvm::GlobalValueSummary *, 1 > summaries_for_guid ( const InlineSearchPath &  path,
llvm::GlobalValue::GUID  guid 
)
static

Definition at line 829 of file llvmjit_inline.cpp.

830{
831 llvm::SmallVector<llvm::GlobalValueSummary *, 1> matches;
832
833 for (auto index : path)
834 {
835 llvm::ValueInfo funcVI = index->getValueInfo(guid);
836
837 /* if index doesn't know function, we don't have a body, continue */
838 if (funcVI)
839 for (auto &gv : funcVI.getSummaryList())
840 matches.push_back(gv.get());
841 }
842
843 return matches;
844}
Definition: type.h:96

Referenced by llvm_build_inline_plan().

Variable Documentation

◆ inline_cost_decay_factor

const float inline_cost_decay_factor = 0.5

Definition at line 99 of file llvmjit_inline.cpp.

Referenced by function_inlinable().

◆ inline_initial_cost

const int inline_initial_cost = 150

Definition at line 100 of file llvmjit_inline.cpp.

Referenced by llvm_build_inline_plan().

◆ module_cache

llvm::ManagedStatic<ModuleCache> module_cache

Definition at line 107 of file llvmjit_inline.cpp.

Referenced by llvm_execute_inline_plan(), llvm_inline_reset_caches(), and load_module_cached().

◆ summary_cache

llvm::ManagedStatic<SummaryCache> summary_cache

Definition at line 109 of file llvmjit_inline.cpp.

Referenced by add_module_to_inline_search_path(), and llvm_inline_reset_caches().

AltStyle によって変換されたページ (->オリジナル) /