Index: mesa-24.0.7/meson.build =================================================================== --- mesa-24.0.7.orig/meson.build +++ mesa-24.0.7/meson.build @@ -1747,7 +1747,9 @@ llvm_optional_modules += ['lto'] endif -if with_amd_vk or with_gallium_radeonsi or with_clc +llvm_with_orcjit = get_option('llvm-orcjit') + +if with_amd_vk or with_gallium_radeonsi or with_clc or llvm_with_orcjit _llvm_version = '>= 15.0.0' elif with_gallium_opencl _llvm_version = '>= 11.0.0' @@ -1827,6 +1829,7 @@ endif pre_args += '-DLLVM_AVAILABLE=' + (with_llvm ? '1' : '0') pre_args += '-DDRAW_LLVM_AVAILABLE=' + (with_llvm and draw_with_llvm ? '1' : '0') +pre_args += '-DGALLIVM_USE_ORCJIT=' + (with_llvm and llvm_with_orcjit ? '1' : '0') if with_clover_spirv or with_clc chosen_llvm_version_array = dep_llvm.version().split('.') Index: mesa-24.0.7/meson_options.txt =================================================================== --- mesa-24.0.7.orig/meson_options.txt +++ mesa-24.0.7/meson_options.txt @@ -416,6 +416,13 @@ option( 'is included.' ) +option ( + 'llvm-orcjit', + type : 'boolean', + value : false, + description: 'Build llvmpipe with LLVM ORCJIT support.' +) + option( 'valgrind', type : 'feature', Index: mesa-24.0.7/src/gallium/auxiliary/draw/draw_context.c =================================================================== --- mesa-24.0.7.orig/src/gallium/auxiliary/draw/draw_context.c +++ mesa-24.0.7/src/gallium/auxiliary/draw/draw_context.c @@ -77,7 +77,11 @@ draw_create_context(struct pipe_context #if DRAW_LLVM_AVAILABLE if (try_llvm && draw_get_option_use_llvm()) { +#if GALLIVM_USE_ORCJIT == 1 + draw->llvm = draw_llvm_create(draw, (LLVMOrcThreadSafeContextRef)context); +#else draw->llvm = draw_llvm_create(draw, (LLVMContextRef)context); +#endif } #endif Index: mesa-24.0.7/src/gallium/auxiliary/draw/draw_llvm.c =================================================================== --- mesa-24.0.7.orig/src/gallium/auxiliary/draw/draw_llvm.c +++ mesa-24.0.7/src/gallium/auxiliary/draw/draw_llvm.c @@ -381,8 +381,13 @@ get_vertex_header_ptr_type(struct draw_l /** * Create per-context LLVM info. */ +#if GALLIVM_USE_ORCJIT == 1 +struct draw_llvm * +draw_llvm_create(struct draw_context *draw, LLVMOrcThreadSafeContextRef context) +#else struct draw_llvm * draw_llvm_create(struct draw_context *draw, LLVMContextRef context) +#endif { struct draw_llvm *llvm; @@ -395,6 +400,16 @@ draw_llvm_create(struct draw_context *dr llvm->draw = draw; +#if GALLIVM_USE_ORCJIT == 1 + llvm->_ts_context = context; + if (!llvm->_ts_context) { + llvm->_ts_context = LLVMOrcCreateNewThreadSafeContext(); + llvm->context_owned = true; + } + if (!llvm->_ts_context) + goto fail; + llvm->context = LLVMOrcThreadSafeContextGetContext(llvm->_ts_context); +#else llvm->context = context; if (!llvm->context) { llvm->context = LLVMContextCreate(); @@ -407,6 +422,7 @@ draw_llvm_create(struct draw_context *dr } if (!llvm->context) goto fail; +#endif llvm->nr_variants = 0; list_inithead(&llvm->vs_variants_list.list); @@ -434,9 +450,16 @@ fail: void draw_llvm_destroy(struct draw_llvm *llvm) { +#if GALLIVM_USE_ORCJIT == 1 + if (llvm->context_owned) + LLVMOrcDisposeThreadSafeContext(llvm->_ts_context); + llvm->_ts_context = NULL; + llvm->context = NULL; +#else if (llvm->context_owned) LLVMContextDispose(llvm->context); llvm->context = NULL; +#endif /* XXX free other draw_llvm data? */ FREE(llvm); @@ -510,7 +533,11 @@ draw_llvm_create_variant(struct draw_llv if (!cached.data_size) needs_caching = true; } +#if GALLIVM_USE_ORCJIT == 1 + variant->gallivm = gallivm_create(module_name, llvm->_ts_context, &cached); +#else variant->gallivm = gallivm_create(module_name, llvm->context, &cached); +#endif create_vs_jit_types(variant); @@ -530,7 +557,8 @@ draw_llvm_create_variant(struct draw_llv gallivm_compile_module(variant->gallivm); variant->jit_func = (draw_jit_vert_func) - gallivm_jit_function(variant->gallivm, variant->function); + gallivm_jit_function(variant->gallivm, variant->function, + variant->function_name); if (needs_caching) llvm->draw->disk_cache_insert_shader(llvm->draw->disk_cache_cookie, @@ -1633,6 +1661,8 @@ draw_llvm_generate(struct draw_llvm *llv variant_func = LLVMAddFunction(gallivm->module, func_name, func_type); variant->function = variant_func; + variant->function_name = MALLOC(strlen(func_name)+1); + strcpy(variant->function_name, func_name); LLVMSetFunctionCallConv(variant_func, LLVMCCallConv); for (i = 0; i < num_arg_types; ++i) @@ -2249,6 +2279,8 @@ draw_llvm_destroy_variant(struct draw_ll variant->shader->variants_cached--; list_del(&variant->list_item_global.list); llvm->nr_variants--; + if(variant->function_name) + FREE(variant->function_name); FREE(variant); } @@ -2358,6 +2390,8 @@ draw_gs_llvm_generate(struct draw_llvm * variant_func = LLVMAddFunction(gallivm->module, func_name, func_type); variant->function = variant_func; + variant->function_name = MALLOC(strlen(func_name)+1); + strcpy(variant->function_name, func_name); LLVMSetFunctionCallConv(variant_func, LLVMCCallConv); @@ -2524,7 +2558,11 @@ draw_gs_llvm_create_variant(struct draw_ if (!cached.data_size) needs_caching = true; } +#if GALLIVM_USE_ORCJIT == 1 + variant->gallivm = gallivm_create(module_name, llvm->_ts_context, &cached); +#else variant->gallivm = gallivm_create(module_name, llvm->context, &cached); +#endif create_gs_jit_types(variant); @@ -2536,7 +2574,8 @@ draw_gs_llvm_create_variant(struct draw_ gallivm_compile_module(variant->gallivm); variant->jit_func = (draw_gs_jit_func) - gallivm_jit_function(variant->gallivm, variant->function); + gallivm_jit_function(variant->gallivm, variant->function, + variant->function_name); if (needs_caching) llvm->draw->disk_cache_insert_shader(llvm->draw->disk_cache_cookie, @@ -2569,6 +2608,8 @@ draw_gs_llvm_destroy_variant(struct draw variant->shader->variants_cached--; list_del(&variant->list_item_global.list); llvm->nr_gs_variants--; + if(variant->function_name) + FREE(variant->function_name); FREE(variant); } @@ -2942,6 +2983,8 @@ draw_tcs_llvm_generate(struct draw_llvm variant_coro = LLVMAddFunction(gallivm->module, func_name_coro, coro_func_type); variant->function = variant_func; + variant->function_name = MALLOC(strlen(func_name)+1); + strcpy(variant->function_name, func_name); LLVMSetFunctionCallConv(variant_func, LLVMCCallConv); LLVMSetFunctionCallConv(variant_coro, LLVMCCallConv); @@ -3177,8 +3220,11 @@ draw_tcs_llvm_create_variant(struct draw if (!cached.data_size) needs_caching = true; } - +#if GALLIVM_USE_ORCJIT == 1 + variant->gallivm = gallivm_create(module_name, llvm->_ts_context, &cached); +#else variant->gallivm = gallivm_create(module_name, llvm->context, &cached); +#endif create_tcs_jit_types(variant); @@ -3192,7 +3238,8 @@ draw_tcs_llvm_create_variant(struct draw gallivm_compile_module(variant->gallivm); variant->jit_func = (draw_tcs_jit_func) - gallivm_jit_function(variant->gallivm, variant->function); + gallivm_jit_function(variant->gallivm, variant->function, + variant->function_name); if (needs_caching) llvm->draw->disk_cache_insert_shader(llvm->draw->disk_cache_cookie, @@ -3225,6 +3272,8 @@ draw_tcs_llvm_destroy_variant(struct dra variant->shader->variants_cached--; list_del(&variant->list_item_global.list); llvm->nr_tcs_variants--; + if(variant->function_name) + FREE(variant->function_name); FREE(variant); } @@ -3507,6 +3556,8 @@ draw_tes_llvm_generate(struct draw_llvm variant_func = LLVMAddFunction(gallivm->module, func_name, func_type); variant->function = variant_func; + variant->function_name = MALLOC(strlen(func_name)+1); + strcpy(variant->function_name, func_name); LLVMSetFunctionCallConv(variant_func, LLVMCCallConv); for (i = 0; i < ARRAY_SIZE(arg_types); ++i) @@ -3697,7 +3748,11 @@ draw_tes_llvm_create_variant(struct draw if (!cached.data_size) needs_caching = true; } +#if GALLIVM_USE_ORCJIT == 1 + variant->gallivm = gallivm_create(module_name, llvm->_ts_context, &cached); +#else variant->gallivm = gallivm_create(module_name, llvm->context, &cached); +#endif create_tes_jit_types(variant); @@ -3714,7 +3769,8 @@ draw_tes_llvm_create_variant(struct draw gallivm_compile_module(variant->gallivm); variant->jit_func = (draw_tes_jit_func) - gallivm_jit_function(variant->gallivm, variant->function); + gallivm_jit_function(variant->gallivm, variant->function, + variant->function_name); if (needs_caching) llvm->draw->disk_cache_insert_shader(llvm->draw->disk_cache_cookie, @@ -3747,6 +3803,8 @@ draw_tes_llvm_destroy_variant(struct dra variant->shader->variants_cached--; list_del(&variant->list_item_global.list); llvm->nr_tes_variants--; + if(variant->function_name) + FREE(variant->function_name); FREE(variant); } Index: mesa-24.0.7/src/gallium/auxiliary/draw/draw_llvm.h =================================================================== --- mesa-24.0.7.orig/src/gallium/auxiliary/draw/draw_llvm.h +++ mesa-24.0.7/src/gallium/auxiliary/draw/draw_llvm.h @@ -42,6 +42,9 @@ #include "pipe/p_context.h" #include "util/list.h" +#if GALLIVM_USE_ORCJIT == 1 +#include +#endif struct draw_llvm; struct llvm_vertex_shader; @@ -398,6 +401,7 @@ struct draw_llvm_variant LLVMTypeRef vertex_header_ptr_type; LLVMValueRef function; + char *function_name; draw_jit_vert_func jit_func; struct llvm_vertex_shader *shader; @@ -431,6 +435,7 @@ struct draw_gs_llvm_variant LLVMValueRef io_ptr; LLVMValueRef num_prims; LLVMValueRef function; + char *function_name; draw_gs_jit_func jit_func; struct llvm_geometry_shader *shader; @@ -457,6 +462,7 @@ struct draw_tcs_llvm_variant /* LLVMValueRef io_ptr; */ LLVMValueRef num_prims; LLVMValueRef function; + char *function_name; draw_tcs_jit_func jit_func; struct llvm_tess_ctrl_shader *shader; @@ -487,6 +493,7 @@ struct draw_tes_llvm_variant LLVMValueRef io_ptr; LLVMValueRef num_prims; LLVMValueRef function; + char *function_name; draw_tes_jit_func jit_func; struct llvm_tess_eval_shader *shader; @@ -539,6 +546,9 @@ struct draw_llvm { struct draw_context *draw; LLVMContextRef context; +#if GALLIVM_USE_ORCJIT == 1 + LLVMOrcThreadSafeContextRef _ts_context; +#endif bool context_owned; struct draw_vs_jit_context vs_jit_context; @@ -584,8 +594,13 @@ llvm_tess_eval_shader(struct draw_tess_e return (struct llvm_tess_eval_shader *)tes; } +#if GALLIVM_USE_ORCJIT == 1 struct draw_llvm * -draw_llvm_create(struct draw_context *draw, LLVMContextRef llvm_context); +draw_llvm_create(struct draw_context *draw, LLVMOrcThreadSafeContextRef context); +#else +struct draw_llvm * +draw_llvm_create(struct draw_context *draw, LLVMContextRef context); +#endif void draw_llvm_destroy(struct draw_llvm *llvm); Index: mesa-24.0.7/src/gallium/auxiliary/gallivm/lp_bld_coro.c =================================================================== --- mesa-24.0.7.orig/src/gallium/auxiliary/gallivm/lp_bld_coro.c +++ mesa-24.0.7/src/gallium/auxiliary/gallivm/lp_bld_coro.c @@ -156,12 +156,14 @@ coro_free(char *ptr) void lp_build_coro_add_malloc_hooks(struct gallivm_state *gallivm) { +#if GALLIVM_USE_ORCJIT == 0 assert(gallivm->engine); +#endif assert(gallivm->coro_malloc_hook); assert(gallivm->coro_free_hook); - LLVMAddGlobalMapping(gallivm->engine, gallivm->coro_malloc_hook, coro_malloc); - LLVMAddGlobalMapping(gallivm->engine, gallivm->coro_free_hook, coro_free); + gallivm_add_global_mapping(gallivm, gallivm->coro_malloc_hook, coro_malloc); + gallivm_add_global_mapping(gallivm, gallivm->coro_free_hook, coro_free); } void lp_build_coro_declare_malloc_hooks(struct gallivm_state *gallivm) Index: mesa-24.0.7/src/gallium/auxiliary/gallivm/lp_bld_coro.h =================================================================== --- mesa-24.0.7.orig/src/gallium/auxiliary/gallivm/lp_bld_coro.h +++ mesa-24.0.7/src/gallium/auxiliary/gallivm/lp_bld_coro.h @@ -31,6 +31,10 @@ #include "gallivm/lp_bld.h" #include "gallivm/lp_bld_intr.h" +#ifdef __cplusplus +extern "C" { +#endif + struct gallivm_state; LLVMValueRef lp_build_coro_id(struct gallivm_state *gallivm); @@ -84,4 +88,8 @@ static inline void lp_build_coro_add_pre #endif } +#ifdef __cplusplus +} +#endif + #endif Index: mesa-24.0.7/src/gallium/auxiliary/gallivm/lp_bld_init.c =================================================================== --- mesa-24.0.7.orig/src/gallium/auxiliary/gallivm/lp_bld_init.c +++ mesa-24.0.7/src/gallium/auxiliary/gallivm/lp_bld_init.c @@ -515,6 +515,11 @@ gallivm_destroy(struct gallivm_state *ga FREE(gallivm); } +void +gallivm_add_global_mapping(struct gallivm_state *gallivm, LLVMValueRef sym, void* addr) +{ + LLVMAddGlobalMapping(gallivm->engine, sym, addr); +} /** * Validate a function. @@ -674,7 +679,7 @@ gallivm_compile_module(struct gallivm_st ++gallivm->compiled; lp_init_printf_hook(gallivm); - LLVMAddGlobalMapping(gallivm->engine, gallivm->debug_printf_hook, debug_printf); + gallivm_add_global_mapping(gallivm, gallivm->debug_printf_hook, debug_printf); lp_init_clock_hook(gallivm); LLVMAddGlobalMapping(gallivm->engine, gallivm->get_time_hook, os_time_get_nano); @@ -717,7 +722,7 @@ gallivm_compile_module(struct gallivm_st func_pointer gallivm_jit_function(struct gallivm_state *gallivm, - LLVMValueRef func) + LLVMValueRef func, const char *func_name) { void *code; func_pointer jit_func; Index: mesa-24.0.7/src/gallium/auxiliary/gallivm/lp_bld_init.h =================================================================== --- mesa-24.0.7.orig/src/gallium/auxiliary/gallivm/lp_bld_init.h +++ mesa-24.0.7/src/gallium/auxiliary/gallivm/lp_bld_init.h @@ -29,11 +29,14 @@ #ifndef LP_BLD_INIT_H #define LP_BLD_INIT_H - #include "util/compiler.h" #include "util/u_pointer.h" // for func_pointer #include "lp_bld.h" +#if GALLIVM_USE_ORCJIT == 1 +#include +#else #include +#endif #ifdef __cplusplus extern "C" { @@ -44,18 +47,25 @@ struct gallivm_state { char *module_name; LLVMModuleRef module; - LLVMExecutionEngineRef engine; LLVMTargetDataRef target; +#if GALLIVM_USE_ORCJIT == 1 + /* own this->module */ + LLVMOrcThreadSafeContextRef _ts_context; + /* each module is in its own jitdylib */ + LLVMOrcJITDylibRef _per_module_jd; +#else + LLVMExecutionEngineRef engine; #if GALLIVM_USE_NEW_PASS == 0 LLVMPassManagerRef passmgr; #if GALLIVM_HAVE_CORO == 1 LLVMPassManagerRef cgpassmgr; #endif #endif - LLVMContextRef context; - LLVMBuilderRef builder; LLVMMCJITMemoryManagerRef memorymgr; struct lp_generated_code *code; +#endif + LLVMContextRef context; + LLVMBuilderRef builder; struct lp_cached_code *cache; unsigned compiled; LLVMValueRef coro_malloc_hook; @@ -77,10 +87,15 @@ lp_build_init_native_width(void); bool lp_build_init(void); - +#if GALLIVM_USE_ORCJIT == 1 +struct gallivm_state * +gallivm_create(const char *name, LLVMOrcThreadSafeContextRef context, + struct lp_cached_code *cache); +#else struct gallivm_state * gallivm_create(const char *name, LLVMContextRef context, struct lp_cached_code *cache); +#endif void gallivm_destroy(struct gallivm_state *gallivm); @@ -93,11 +108,19 @@ gallivm_verify_function(struct gallivm_s LLVMValueRef func); void +gallivm_add_global_mapping(struct gallivm_state *gallivm, LLVMValueRef sym, void* addr); + +/** + * for ORCJIT, after this function gets called, all access and modification to + * module and any structure associated to it should be avoided, + * as module has been moved into ORCJIT and may be recycled + */ +void gallivm_compile_module(struct gallivm_state *gallivm); func_pointer gallivm_jit_function(struct gallivm_state *gallivm, - LLVMValueRef func); + LLVMValueRef func, const char *func_name); unsigned gallivm_get_perf_flags(void); Index: mesa-24.0.7/src/gallium/auxiliary/gallivm/lp_bld_init_orc.cpp =================================================================== --- /dev/null +++ mesa-24.0.7/src/gallium/auxiliary/gallivm/lp_bld_init_orc.cpp @@ -0,0 +1,959 @@ +#include "util/detect.h" +#include "util/u_cpu_detect.h" +#include "util/u_debug.h" +#include "util/os_time.h" +#include "lp_bld.h" +#include "lp_bld_debug.h" +#include "lp_bld_init.h" +#include "lp_bld_coro.h" +#include "lp_bld_printf.h" + +#include +#include +#include +#include +#include +#include + +#include +#if LLVM_VERSION_MAJOR < 17 +#include +#if LLVM_VERSION_MAJOR >= 7 +#include +#endif +#endif +#include +#if GALLIVM_USE_NEW_PASS == 1 +#include +#elif GALLIVM_HAVE_CORO == 1 +#if LLVM_VERSION_MAJOR <= 8 && (DETECT_ARCH_AARCH64 == 1 || DETECT_ARCH_ARM == 1 || DETECT_ARCH_S390 == 1 || DETECT_ARCH_MIPS64 == 1) +#include +#endif +#include +#endif + +#include +#include +#include +#include "llvm/ExecutionEngine/JITLink/JITLink.h" +#include +#include +#include +#if LLVM_VERSION_MAJOR >= 18 +#include +#else +#include +#endif +#include +#if LLVM_USE_INTEL_JITEVENTS +#include +#endif + +/* conflict with ObjectLinkingLayer.h */ +#include "util/u_memory.h" + +#if DETECT_ARCH_RISCV64 == 1 || DETECT_ARCH_RISCV32 == 1 || (defined(_WIN32) && LLVM_VERSION_MAJOR >= 15) +/* use ObjectLinkingLayer (JITLINK backend) */ +#define USE_JITLINK +#endif +/* else use old RTDyldObjectLinkingLayer (RuntimeDyld backend) */ + +unsigned lp_native_vector_width; + +unsigned gallivm_perf = 0; + +static const struct debug_named_value lp_bld_perf_flags[] = { + { "brilinear", GALLIVM_PERF_BRILINEAR, "enable brilinear optimization" }, + { "rho_approx", GALLIVM_PERF_RHO_APPROX, "enable rho_approx optimization" }, + { "no_quad_lod", GALLIVM_PERF_NO_QUAD_LOD, "disable quad_lod optimization" }, + { "no_aos_sampling", GALLIVM_PERF_NO_AOS_SAMPLING, "disable aos sampling optimization" }, + { "nopt", GALLIVM_PERF_NO_OPT, "disable optimization passes to speed up shader compilation" }, + DEBUG_NAMED_VALUE_END +}; + +unsigned gallivm_debug = 0; + +static const struct debug_named_value lp_bld_debug_flags[] = { + { "tgsi", GALLIVM_DEBUG_TGSI, NULL }, + { "ir", GALLIVM_DEBUG_IR, NULL }, + { "asm", GALLIVM_DEBUG_ASM, NULL }, + { "perf", GALLIVM_DEBUG_PERF, NULL }, + { "gc", GALLIVM_DEBUG_GC, NULL }, +/* Don't allow setting DUMP_BC for release builds, since writing the files may be an issue with setuid. */ +#ifdef DEBUG + { "dumpbc", GALLIVM_DEBUG_DUMP_BC, NULL }, +#endif + DEBUG_NAMED_VALUE_END +}; + +DEBUG_GET_ONCE_FLAGS_OPTION(gallivm_debug, "GALLIVM_DEBUG", lp_bld_debug_flags, 0) + +struct lp_cached_code { + void *data; + size_t data_size; + bool dont_cache; + void *jit_obj_cache; +}; + +namespace { + +class LPJit; + +class LLVMEnsureMultithreaded { +public: + LLVMEnsureMultithreaded() + { + if (!LLVMIsMultithreaded()) { + LLVMStartMultithreaded(); + } + } +}; + +LLVMEnsureMultithreaded lLVMEnsureMultithreaded; + +DEFINE_SIMPLE_CONVERSION_FUNCTIONS(llvm::orc::ThreadSafeContext, + LLVMOrcThreadSafeContextRef) +DEFINE_SIMPLE_CONVERSION_FUNCTIONS(llvm::orc::IRTransformLayer, + LLVMOrcIRTransformLayerRef) +DEFINE_SIMPLE_CONVERSION_FUNCTIONS(llvm::orc::JITDylib, LLVMOrcJITDylibRef) +DEFINE_SIMPLE_CONVERSION_FUNCTIONS(llvm::orc::JITTargetMachineBuilder, + LLVMOrcJITTargetMachineBuilderRef) +LLVMTargetMachineRef wrap(const llvm::TargetMachine *P) { + return reinterpret_cast(const_cast(P)); +} + +llvm::ExitOnError ExitOnErr; + +inline const char* get_module_name(LLVMModuleRef mod) { + using llvm::Module; + return llvm::unwrap(mod)->getModuleIdentifier().c_str(); +} + +once_flag init_lpjit_once_flag = ONCE_FLAG_INIT; + +/* A JIT singleton built upon LLJIT */ +class LPJit +{ +public: + static LPJit* get_instance() { + call_once(&init_lpjit_once_flag, init_lpjit); + return jit; + } + + gallivm_state *find_gallivm_state(LLVMModuleRef mod) { +#if DEBUG + using llvm::Module; + auto I = gallivm_modules.find(llvm::unwrap(mod)->getModuleIdentifier()); + if (I == gallivm_modules.end()) { + debug_printf("No gallivm state found for module: %s", get_module_name(mod)); + return NULL; + } + return I->second; +#endif + return NULL; + } + + static char *get_unique_name(const char* name) { + LPJit* jit = get_instance(); + size_t size = name == NULL? 16: strlen(name) + 16; + char *name_uniq = (char *)MALLOC(size); + if (!name_uniq) { + return NULL; + } + do { + snprintf(name_uniq, size, "%s_%u", name, jit->jit_dylib_count++); + } while(jit->lljit->getExecutionSession().getJITDylibByName(name_uniq)); + return name_uniq; + } + + static LLVMOrcJITDylibRef create_jit_dylib(const char * name) { + using llvm::orc::JITDylib; + LPJit* jit = get_instance(); + JITDylib& tmp = ExitOnErr(jit->lljit->createJITDylib(name)); + return wrap(&tmp); + } + + static void register_gallivm_state(gallivm_state *gallivm) { +#if DEBUG + LPJit* jit = get_instance(); + jit->gallivm_modules[gallivm->module_name] = gallivm; +#endif + } + + static void deregister_gallivm_state(gallivm_state *gallivm) { +#if DEBUG + LPJit* jit = get_instance(); + (void)jit->gallivm_modules.erase(gallivm->module_name); +#endif + } + + static void add_ir_module_to_jd( + LLVMOrcThreadSafeContextRef ts_context, + LLVMModuleRef mod, + LLVMOrcJITDylibRef jd) { + using llvm::Module; + using llvm::orc::ThreadSafeModule; + using llvm::orc::JITDylib; + ThreadSafeModule tsm( + std::unique_ptr(llvm::unwrap(mod)), *::unwrap(ts_context)); + ExitOnErr(get_instance()->lljit->addIRModule( + *::unwrap(jd), std::move(tsm) + )); + } + + static void add_mapping_to_jd( + LLVMValueRef sym, + void *addr, + LLVMOrcJITDylibRef jd) { +#if LLVM_VERSION_MAJOR >= 17 + using llvm::orc::ExecutorAddr; + using llvm::orc::ExecutorSymbolDef; + using llvm::JITSymbolFlags; +#else + using llvm::JITEvaluatedSymbol; +#endif + using llvm::orc::ExecutionSession; + using llvm::orc::JITDylib; + using llvm::orc::SymbolMap; + JITDylib* JD = ::unwrap(jd); + auto& es = LPJit::get_instance()->lljit->getExecutionSession(); + auto name = es.intern(llvm::unwrap(sym)->getName()); + SymbolMap map(1); +#if LLVM_VERSION_MAJOR >= 17 + map[name] = ExecutorSymbolDef(ExecutorAddr::fromPtr(addr), JITSymbolFlags::Exported); +#else + map[name] = JITEvaluatedSymbol::fromPointer(addr); +#endif + auto munit = llvm::orc::absoluteSymbols(map); + llvm::cantFail(JD->define(std::move(munit))); + } + + static void *lookup_in_jd( + const char *func_name, + LLVMOrcJITDylibRef jd) { + using llvm::orc::JITDylib; + using llvm::JITEvaluatedSymbol; + using llvm::orc::ExecutorAddr; + JITDylib* JD = ::unwrap(jd); + auto func = ExitOnErr(LPJit::get_instance()->lljit->lookup(*JD, func_name)); +#if LLVM_VERSION_MAJOR >= 16 + return func.toPtr(); +#else + return (void *)(func.getAddress()); +#endif + } + + static void remove_jd(LLVMOrcJITDylibRef jd) { + using llvm::orc::ExecutionSession; + using llvm::orc::JITDylib; + auto& es = LPJit::get_instance()->lljit->getExecutionSession(); + ExitOnErr(es.removeJITDylib(* ::unwrap(jd))); + } + + LLVMTargetMachineRef tm; + +private: + LPJit(); + ~LPJit() = default; + LPJit(const LPJit&) = delete; + LPJit& operator=(const LPJit&) = delete; + + static void init_native_targets(); + llvm::orc::JITTargetMachineBuilder create_jtdb(); + + static void init_lpjit() { + jit = new LPJit; + } + static LPJit* jit; + + std::unique_ptr lljit; + /* avoid name conflict */ + unsigned jit_dylib_count; + +#if DEBUG + /* map from module name to gallivm_state */ + llvm::StringMap gallivm_modules; +#endif +}; + +LPJit* LPJit::jit = NULL; + +LLVMErrorRef module_transform(void *Ctx, LLVMModuleRef mod) { + int64_t time_begin = 0; + if (::gallivm_debug & GALLIVM_DEBUG_PERF) + time_begin = os_time_get(); +#if GALLIVM_USE_NEW_PASS == 1 + char passes[1024]; + passes[0] = 0; + + /* + * there should be some way to combine these two pass runs but I'm not seeing it, + * at the time of writing. + */ + strcpy(passes, "default"); + + LLVMPassBuilderOptionsRef opts = LLVMCreatePassBuilderOptions(); + LLVMRunPasses(mod, passes, LPJit::get_instance()->tm, opts); + + if (!(gallivm_perf & GALLIVM_PERF_NO_OPT)) + strcpy(passes, "sroa,early-cse,simplifycfg,reassociate,mem2reg,instsimplify,instcombine,"); + else + strcpy(passes, "mem2reg"); + + LLVMRunPasses(mod, passes, LPJit::get_instance()->tm, opts); + LLVMDisposePassBuilderOptions(opts); + + return LLVMErrorSuccess; + +#else /* GALLIVM_USE_NEW_PASS */ + LLVMPassManagerRef passmgr = LLVMCreateFunctionPassManagerForModule(mod); + +#if GALLIVM_HAVE_CORO == 1 + LLVMPassManagerRef cgpassmgr = LLVMCreatePassManager(); +#endif + +#if GALLIVM_HAVE_CORO == 1 +#if LLVM_VERSION_MAJOR <= 8 && (DETECT_ARCH_AARCH64 == 1 || DETECT_ARCH_ARM == 1 || DETECT_ARCH_S390 == 1 || DETECT_ARCH_MIPS64 == 1) + LLVMAddArgumentPromotionPass(cgpassmgr); + LLVMAddFunctionAttrsPass(cgpassmgr); +#endif + LLVMAddCoroEarlyPass(cgpassmgr); + LLVMAddCoroSplitPass(cgpassmgr); + LLVMAddCoroElidePass(cgpassmgr); +#endif + + if ((gallivm_perf & GALLIVM_PERF_NO_OPT) == 0) { + /* + * TODO: Evaluate passes some more - keeping in mind + * both quality of generated code and compile times. + */ + /* + * NOTE: if you change this, don't forget to change the output + * with GALLIVM_DEBUG_DUMP_BC in gallivm_compile_module. + */ + LLVMAddScalarReplAggregatesPass(passmgr); + LLVMAddEarlyCSEPass(passmgr); + LLVMAddCFGSimplificationPass(passmgr); + /* + * FIXME: LICM is potentially quite useful. However, for some + * rather crazy shaders the compile time can reach _hours_ per shader, + * due to licm implying lcssa (since llvm 3.5), which can take forever. + * Even for sane shaders, the cost of licm is rather high (and not just + * due to lcssa, licm itself too), though mostly only in cases when it + * can actually move things, so having to disable it is a pity. + * LLVMAddLICMPass(passmgr); + */ + LLVMAddReassociatePass(passmgr); + LLVMAddPromoteMemoryToRegisterPass(passmgr); +#if LLVM_VERSION_MAJOR <= 11 + LLVMAddConstantPropagationPass(passmgr); +#else + LLVMAddInstructionSimplifyPass(passmgr); +#endif + LLVMAddInstructionCombiningPass(passmgr); + LLVMAddGVNPass(passmgr); + } + else { + /* We need at least this pass to prevent the backends to fail in + * unexpected ways. + */ + LLVMAddPromoteMemoryToRegisterPass(passmgr); + } +#if GALLIVM_HAVE_CORO == 1 + LLVMAddCoroCleanupPass(passmgr); + + LLVMRunPassManager(cgpassmgr, mod); +#endif + /* Run optimization passes */ + LLVMInitializeFunctionPassManager(passmgr); + LLVMValueRef func; + func = LLVMGetFirstFunction(mod); + while (func) { + if (0) { + debug_printf("optimizing func %s...\n", LLVMGetValueName(func)); + } + + /* Disable frame pointer omission on debug/profile builds */ + /* XXX: And workaround http://llvm.org/PR21435 */ +#if defined(DEBUG) || defined(PROFILE) || (DETECT_ARCH_X86 == 1 || DETECT_ARCH_X86_64 == 1) + LLVMAddTargetDependentFunctionAttr(func, "no-frame-pointer-elim", "true"); + LLVMAddTargetDependentFunctionAttr(func, "no-frame-pointer-elim-non-leaf", "true"); +#endif + + LLVMRunFunctionPassManager(passmgr, func); + func = LLVMGetNextFunction(func); + } + LLVMFinalizeFunctionPassManager(passmgr); + if (gallivm_debug & GALLIVM_DEBUG_PERF) { + int64_t time_end = os_time_get(); + int time_msec = (int)((time_end - time_begin) / 1000); + + const char *module_name = get_module_name(mod); + debug_printf("optimizing module %s took %d msec\n", + module_name, time_msec); + } + +#if GALLIVM_HAVE_CORO == 1 + LLVMDisposePassManager(cgpassmgr); +#endif + LLVMDisposePassManager(passmgr); + return LLVMErrorSuccess; +#endif /* GALLIVM_USE_NEW_PASS */ +} + +LLVMErrorRef module_transform_wrapper( + void *Ctx, LLVMOrcThreadSafeModuleRef *ModInOut, + LLVMOrcMaterializationResponsibilityRef MR) { + return LLVMOrcThreadSafeModuleWithModuleDo(*ModInOut, *module_transform, Ctx); +} + +LPJit::LPJit() :jit_dylib_count(0) { + using namespace llvm::orc; + + ::gallivm_debug = debug_get_option_gallivm_debug(); + + + gallivm_perf = debug_get_flags_option("GALLIVM_PERF", lp_bld_perf_flags, 0 ); + + init_native_targets(); + JITTargetMachineBuilder JTMB = create_jtdb(); + tm = wrap(ExitOnErr(JTMB.createTargetMachine()).release()); + + /* Create an LLJIT instance with an ObjectLinkingLayer (JITLINK) + * or RuntimeDyld as the base layer. + * intel & perf listeners are not supported by ObjectLinkingLayer yet + */ + lljit = ExitOnErr( + LLJITBuilder() + .setJITTargetMachineBuilder(std::move(JTMB)) +#ifdef USE_JITLINK + .setObjectLinkingLayerCreator( + [&](ExecutionSession &ES, const llvm::Triple &TT) { + return std::make_unique( + ES, ExitOnErr(llvm::jitlink::InProcessMemoryManager::Create())); + }) +#else +#if LLVM_USE_INTEL_JITEVENTS + .RegisterJITEventListener( + llvm::JITEventListener::createIntelJITEventListener()) +#endif +#endif + .create()); + + LLVMOrcIRTransformLayerRef TL = wrap(&lljit->getIRTransformLayer()); + LLVMOrcIRTransformLayerSetTransform(TL, *module_transform_wrapper, NULL); +} + +void LPJit::init_native_targets() { + // If we have a native target, initialize it to ensure it is linked in and + // usable by the JIT. + llvm::InitializeNativeTarget(); + + llvm::InitializeNativeTargetAsmPrinter(); + + llvm::InitializeNativeTargetDisassembler(); +#if DEBUG + { + char *env_llc_options = getenv("GALLIVM_LLC_OPTIONS"); + if (env_llc_options) { + char *option; + char *options[64] = {(char *) "llc"}; // Warning without cast + int n; + for (n = 0, option = strtok(env_llc_options, " "); option; n++, option = strtok(NULL, " ")) { + options[n + 1] = option; + } + if (::gallivm_debug & (GALLIVM_DEBUG_IR | GALLIVM_DEBUG_ASM | GALLIVM_DEBUG_DUMP_BC)) { + debug_printf("llc additional options (%d):\n", n); + for (int i = 1; i <= n; i++) + debug_printf("\t%s\n", options[i]); + debug_printf("\n"); + } + LLVMParseCommandLineOptions(n + 1, options, NULL); + } + } +#endif + + if (util_get_cpu_caps()->has_avx2 || util_get_cpu_caps()->has_avx) { + ::lp_native_vector_width = 256; + } else { + /* Leave it at 128, even when no SIMD extensions are available. + * Really needs to be a multiple of 128 so can fit 4 floats. + */ + ::lp_native_vector_width = 128; + } + + ::lp_native_vector_width = debug_get_num_option("LP_NATIVE_VECTOR_WIDTH", + lp_native_vector_width); + +#if DETECT_ARCH_PPC_64 == 1 + /* Set the NJ bit in VSCR to 0 so denormalized values are handled as + * specified by IEEE standard (PowerISA 2.06 - Section 6.3). This guarantees + * that some rounding and half-float to float handling does not round + * incorrectly to 0. + * XXX: should eventually follow same logic on all platforms. + * Right now denorms get explicitly disabled (but elsewhere) for x86, + * whereas ppc64 explicitly enables them... + */ + if (util_get_cpu_caps()->has_altivec) { + unsigned short mask[] = { 0xFFFF, 0xFFFF, 0xFFFF, 0xFFFF, + 0xFFFF, 0xFFFF, 0xFFFE, 0xFFFF }; + __asm ( + "mfvscr %%v1\n" + "vand %0,%%v1,%0\n" + "mtvscr %0" + : + : "r" (*mask) + ); + } +#endif +} + +llvm::orc::JITTargetMachineBuilder LPJit::create_jtdb() { + using namespace llvm; + using orc::JITTargetMachineBuilder; + +#if defined(_WIN32) && LLVM_VERSION_MAJOR < 15 + /* + * JITLink works on Windows, but only through ELF object format. + * + * XXX: We could use `LLVM_HOST_TRIPLE "-elf"` but LLVM_HOST_TRIPLE has + * different strings for MinGW/MSVC, so better play it safe and be + * explicit. + */ +# ifdef _WIN64 + JITTargetMachineBuilder JTMB((Triple("x86_64-pc-win32-elf"))); +# else + JITTargetMachineBuilder JTMB((Triple("i686-pc-win32-elf"))); +# endif +#else + /* + * llvm::sys::getProcessTriple() is bogus. It returns the host LLVM was + * compiled on. Be careful when doing cross compilation + */ + JITTargetMachineBuilder JTMB((Triple(sys::getProcessTriple()))); +#endif + + TargetOptions options; + /** + * LLVM 3.1+ haven't more "extern unsigned llvm::StackAlignmentOverride" and + * friends for configuring code generation options, like stack alignment. + */ +#if DETECT_ARCH_X86 == 1 && LLVM_VERSION_MAJOR < 13 + options.StackAlignmentOverride = 4; +#endif + +#if DETECT_ARCH_RISCV64 == 1 +#if defined(__riscv_float_abi_soft) + options.MCOptions.ABIName = "lp64"; +#elif defined(__riscv_float_abi_single) + options.MCOptions.ABIName = "lp64f"; +#elif defined(__riscv_float_abi_double) + options.MCOptions.ABIName = "lp64d"; +#else +#error "GALLIVM: unknown target riscv float abi" +#endif +#endif + +#if DETECT_ARCH_RISCV32 == 1 +#if defined(__riscv_float_abi_soft) + options.MCOptions.ABIName = "ilp32"; +#elif defined(__riscv_float_abi_single) + options.MCOptions.ABIName = "ilp32f"; +#elif defined(__riscv_float_abi_double) + options.MCOptions.ABIName = "ilp32d"; +#else +#error "GALLIVM: unknown target riscv float abi" +#endif +#endif + + JTMB.setOptions(options); + + std::vector MAttrs; + +#if LLVM_VERSION_MAJOR >= 4 && (DETECT_ARCH_X86 == 1 || DETECT_ARCH_X86_64 == 1 || DETECT_ARCH_ARM == 1) + /* llvm-3.3+ implements sys::getHostCPUFeatures for Arm + * and llvm-3.7+ for x86, which allows us to enable/disable + * code generation based on the results of cpuid on these + * architectures. + */ + StringMap features; + sys::getHostCPUFeatures(features); + + for (StringMapIterator f = features.begin(); + f != features.end(); + ++f) { + MAttrs.push_back(((*f).second ? "+" : "-") + (*f).first().str()); + } +#elif DETECT_ARCH_X86 == 1 || DETECT_ARCH_X86_64 == 1 + /* + * We need to unset attributes because sometimes LLVM mistakenly assumes + * certain features are present given the processor name. + * + * https://bugs.freedesktop.org/show_bug.cgi?id=92214 + * http://llvm.org/PR25021 + * http://llvm.org/PR19429 + * http://llvm.org/PR16721 + */ + MAttrs.push_back(util_get_cpu_caps()->has_sse ? "+sse" : "-sse" ); + MAttrs.push_back(util_get_cpu_caps()->has_sse2 ? "+sse2" : "-sse2" ); + MAttrs.push_back(util_get_cpu_caps()->has_sse3 ? "+sse3" : "-sse3" ); + MAttrs.push_back(util_get_cpu_caps()->has_ssse3 ? "+ssse3" : "-ssse3" ); + MAttrs.push_back(util_get_cpu_caps()->has_sse4_1 ? "+sse4.1" : "-sse4.1"); + MAttrs.push_back(util_get_cpu_caps()->has_sse4_2 ? "+sse4.2" : "-sse4.2"); + /* + * AVX feature is not automatically detected from CPUID by the X86 target + * yet, because the old (yet default) JIT engine is not capable of + * emitting the opcodes. On newer llvm versions it is and at least some + * versions (tested with 3.3) will emit avx opcodes without this anyway. + */ + MAttrs.push_back(util_get_cpu_caps()->has_avx ? "+avx" : "-avx"); + MAttrs.push_back(util_get_cpu_caps()->has_f16c ? "+f16c" : "-f16c"); + MAttrs.push_back(util_get_cpu_caps()->has_fma ? "+fma" : "-fma"); + MAttrs.push_back(util_get_cpu_caps()->has_avx2 ? "+avx2" : "-avx2"); + /* disable avx512 and all subvariants */ + MAttrs.push_back("-avx512cd"); + MAttrs.push_back("-avx512er"); + MAttrs.push_back("-avx512f"); + MAttrs.push_back("-avx512pf"); + MAttrs.push_back("-avx512bw"); + MAttrs.push_back("-avx512dq"); + MAttrs.push_back("-avx512vl"); +#endif +#if DETECT_ARCH_ARM == 1 + if (!util_get_cpu_caps()->has_neon) { + MAttrs.push_back("-neon"); + MAttrs.push_back("-crypto"); + MAttrs.push_back("-vfp2"); + } +#endif + +#if DETECT_ARCH_PPC == 1 + MAttrs.push_back(util_get_cpu_caps()->has_altivec ? "+altivec" : "-altivec"); +#if (LLVM_VERSION_MAJOR < 4) + /* + * Make sure VSX instructions are disabled + * See LLVM bugs: + * https://llvm.org/bugs/show_bug.cgi?id=25503#c7 (fixed in 3.8.1) + * https://llvm.org/bugs/show_bug.cgi?id=26775 (fixed in 3.8.1) + * https://llvm.org/bugs/show_bug.cgi?id=33531 (fixed in 4.0) + * https://llvm.org/bugs/show_bug.cgi?id=34647 (llc performance on certain unusual shader IR; intro'd in 4.0, pending as of 5.0) + */ + if (util_get_cpu_caps()->has_altivec) { + MAttrs.push_back("-vsx"); + } +#else + /* + * Bug 25503 is fixed, by the same fix that fixed + * bug 26775, in versions of LLVM later than 3.8 (starting with 3.8.1). + * BZ 33531 actually comprises more than one bug, all of + * which are fixed in LLVM 4.0. + * + * With LLVM 4.0 or higher: + * Make sure VSX instructions are ENABLED (if supported), unless + * VSX instructions are explicitly enabled/disabled via GALLIVM_VSX=1 or 0. + */ + if (util_get_cpu_caps()->has_altivec) { + MAttrs.push_back(util_get_cpu_caps()->has_vsx ? "+vsx" : "-vsx"); + } +#endif +#endif + +#if DETECT_ARCH_MIPS64 == 1 + MAttrs.push_back(util_get_cpu_caps()->has_msa ? "+msa" : "-msa"); + /* MSA requires a 64-bit FPU register file */ + MAttrs.push_back("+fp64"); +#endif + +#if DETECT_ARCH_RISCV64 == 1 + /* Before riscv is more matured and util_get_cpu_caps() is implemented, + * assume this for now since most of linux capable riscv machine are + * riscv64gc + */ + MAttrs = {"+m","+c","+a","+d","+f"}; +#endif + + JTMB.addFeatures(MAttrs); + + if (::gallivm_debug & (GALLIVM_DEBUG_IR | GALLIVM_DEBUG_ASM | GALLIVM_DEBUG_DUMP_BC)) { + int n = MAttrs.size(); + if (n > 0) { + debug_printf("llc -mattr option(s): "); + for (int i = 0; i < n; i++) + debug_printf("%s%s", MAttrs[i].c_str(), (i < n - 1) ? "," : ""); + debug_printf("\n"); + } + } + + std::string MCPU = llvm::sys::getHostCPUName().str(); + /* + * Note that the MAttrs set above will be sort of ignored (since we should + * not set any which would not be set by specifying the cpu anyway). + * It ought to be safe though since getHostCPUName() should include bits + * not only from the cpu but environment as well (for instance if it's safe + * to use avx instructions which need OS support). According to + * http://llvm.org/bugs/show_bug.cgi?id=19429 however if I understand this + * right it may be necessary to specify older cpu (or disable mattrs) though + * when not using MCJIT so no instructions are generated which the old JIT + * can't handle. Not entirely sure if we really need to do anything yet. + * + * Not sure if the above is also the case for ORCJIT, but we need set CPU + * manually since we don't use JITTargetMachineBuilder::detectHost() + */ + +#if DETECT_ARCH_PPC_64 == 1 + /* + * Large programs, e.g. gnome-shell and firefox, may tax the addressability + * of the Medium code model once dynamically generated JIT-compiled shader + * programs are linked in and relocated. Yet the default code model as of + * LLVM 8 is Medium or even Small. + * The cost of changing from Medium to Large is negligible: + * - an additional 8-byte pointer stored immediately before the shader entrypoint; + * - change an add-immediate (addis) instruction to a load (ld). + */ + JTMB.setCodeModel(CodeModel::Large); + +#if UTIL_ARCH_LITTLE_ENDIAN + /* + * Versions of LLVM prior to 4.0 lacked a table entry for "POWER8NVL", + * resulting in (big-endian) "generic" being returned on + * little-endian Power8NVL systems. The result was that code that + * attempted to load the least significant 32 bits of a 64-bit quantity + * from memory loaded the wrong half. This resulted in failures in some + * Piglit tests, e.g. + * .../arb_gpu_shader_fp64/execution/conversion/frag-conversion-explicit-double-uint + */ + if (MCPU == "generic") + MCPU = "pwr8"; +#endif +#endif + +#if DETECT_ARCH_MIPS64 == 1 + /* + * ls3a4000 CPU and ls2k1000 SoC is a mips64r5 compatible with MSA SIMD + * instruction set implemented, while ls3a3000 is mips64r2 compatible + * only. getHostCPUName() return "generic" on all loongson + * mips CPU currently. So we override the MCPU to mips64r5 if MSA is + * implemented, feedback to mips64r2 for all other ordinary mips64 cpu. + */ + if (MCPU == "generic") + MCPU = util_get_cpu_caps()->has_msa ? "mips64r5" : "mips64r2"; +#endif + +#if DETECT_ARCH_RISCV64 == 1 + /** + * should be fixed with https://reviews.llvm.org/D121149 in llvm 15, + * set it anyway for llvm 14 + */ + if (MCPU == "generic") + MCPU = "generic-rv64"; + + JTMB.setCodeModel(CodeModel::Medium); + JTMB.setRelocationModel(Reloc::PIC_); +#endif + +#if DETECT_ARCH_RISCV32 == 1 + /** + * should be fixed with https://reviews.llvm.org/D121149 in llvm 15, + * set it anyway for llvm 14 + */ + if (MCPU == "generic") + MCPU = "generic-rv32"; + + JTMB.setCodeModel(CodeModel::Medium); + JTMB.setRelocationModel(Reloc::PIC_); +#endif + + JTMB.setCPU(MCPU); + if (gallivm_debug & (GALLIVM_DEBUG_IR | GALLIVM_DEBUG_ASM | GALLIVM_DEBUG_DUMP_BC)) { + debug_printf("llc -mcpu option: %s\n", MCPU.c_str()); + } + + return JTMB; +} + + +} /* Anonymous namespace */ + +unsigned +lp_build_init_native_width(void) +{ + // Default to 256 until we're confident llvmpipe with 512 is as correct and not slower than 256 + lp_native_vector_width = MIN2(util_get_cpu_caps()->max_vector_bits, 256); + assert(lp_native_vector_width); + + lp_native_vector_width = debug_get_num_option("LP_NATIVE_VECTOR_WIDTH", lp_native_vector_width); + assert(lp_native_vector_width); + + return lp_native_vector_width; +} + +bool +lp_build_init(void) +{ + (void)LPJit::get_instance(); + return true; +} + +bool +init_gallivm_state(struct gallivm_state *gallivm, const char *name, + LLVMOrcThreadSafeContextRef context, struct lp_cached_code *cache) +{ + assert(!gallivm->context); + assert(!gallivm->_ts_context); + assert(!gallivm->module); + + if (!lp_build_init()) + return false; + + // cache is not implemented + gallivm->cache = cache; + if (gallivm->cache) + gallivm->cache->data_size = 0; + + gallivm->_ts_context = context; + gallivm->context = LLVMOrcThreadSafeContextGetContext(context); + + gallivm->module_name = LPJit::get_unique_name(name); + gallivm->module = LLVMModuleCreateWithNameInContext(gallivm->module_name, + gallivm->context); +#if DETECT_ARCH_X86 == 1 + lp_set_module_stack_alignment_override(gallivm->module, 4); +#endif + gallivm->builder = LLVMCreateBuilderInContext(gallivm->context); + gallivm->_per_module_jd = LPJit::create_jit_dylib(gallivm->module_name); + + gallivm->target = LLVMCreateTargetDataLayout(LPJit::get_instance()->tm); + + lp_build_coro_declare_malloc_hooks(gallivm); + return true; +} + +struct gallivm_state * +gallivm_create(const char *name, LLVMOrcThreadSafeContextRef context, + struct lp_cached_code *cache){ + struct gallivm_state *gallivm; + + gallivm = CALLOC_STRUCT(gallivm_state); + if (gallivm) { + if (!init_gallivm_state(gallivm, name, context, cache)) { + FREE(gallivm); + gallivm = NULL; + } + } + + assert(gallivm != NULL); + return gallivm; +} + +void +gallivm_destroy(struct gallivm_state *gallivm) +{ + LPJit::remove_jd(gallivm->_per_module_jd); + gallivm->_per_module_jd = nullptr; +} + +void +gallivm_free_ir(struct gallivm_state *gallivm) +{ + if (gallivm->module) + LLVMDisposeModule(gallivm->module); + FREE(gallivm->module_name); + + if (gallivm->target) { + LLVMDisposeTargetData(gallivm->target); + } + + if (gallivm->builder) + LLVMDisposeBuilder(gallivm->builder); + + gallivm->target = NULL; + gallivm->module=NULL; + gallivm->module_name=NULL; + gallivm->builder=NULL; + gallivm->context=NULL; + gallivm->_ts_context=NULL; + gallivm->cache=NULL; + LPJit::deregister_gallivm_state(gallivm); +} + +void +gallivm_verify_function(struct gallivm_state *gallivm, + LLVMValueRef func) +{ + /* Verify the LLVM IR. If invalid, dump and abort */ +#ifdef DEBUG + if (LLVMVerifyFunction(func, LLVMPrintMessageAction)) { + lp_debug_dump_value(func); + assert(0); + return; + } +#endif + + if (gallivm_debug & GALLIVM_DEBUG_IR) { + /* Print the LLVM IR to stderr */ + lp_debug_dump_value(func); + debug_printf("\n"); + } +return; +} + +void +gallivm_add_global_mapping(struct gallivm_state *gallivm, LLVMValueRef sym, void* addr) +{ + LPJit::add_mapping_to_jd(sym, addr, gallivm->_per_module_jd); +} + +void lp_init_clock_hook(struct gallivm_state *gallivm) +{ + if (gallivm->get_time_hook) + return; + + LLVMTypeRef get_time_type = LLVMFunctionType(LLVMInt64TypeInContext(gallivm->context), NULL, 0, 1); + gallivm->get_time_hook = LLVMAddFunction(gallivm->module, "get_time_hook", get_time_type); +} + +void +gallivm_compile_module(struct gallivm_state *gallivm) +{ + lp_init_printf_hook(gallivm); + gallivm_add_global_mapping(gallivm, gallivm->debug_printf_hook, + (void *)debug_printf); + + lp_init_clock_hook(gallivm); + gallivm_add_global_mapping(gallivm, gallivm->get_time_hook, + (void *)os_time_get_nano); + + lp_build_coro_add_malloc_hooks(gallivm); + + LPJit::add_ir_module_to_jd(gallivm->_ts_context, gallivm->module, + gallivm->_per_module_jd); + /* ownership of module is now transferred into orc jit, + * disallow modifying it + */ + LPJit::register_gallivm_state(gallivm); + gallivm->module=nullptr; + + /* defer compilation till first lookup by gallivm_jit_function */ +} + +func_pointer +gallivm_jit_function(struct gallivm_state *gallivm, + LLVMValueRef func, const char *func_name) +{ + return pointer_to_func( + LPJit::lookup_in_jd(func_name, gallivm->_per_module_jd)); +} + +unsigned +gallivm_get_perf_flags(void){ + return gallivm_perf; +} + +void +lp_set_module_stack_alignment_override(LLVMModuleRef MRef, unsigned align) +{ +#if LLVM_VERSION_MAJOR >= 13 + llvm::Module *M = llvm::unwrap(MRef); + M->setOverrideStackAlignment(align); +#endif +} Index: mesa-24.0.7/src/gallium/auxiliary/meson.build =================================================================== --- mesa-24.0.7.orig/src/gallium/auxiliary/meson.build +++ mesa-24.0.7/src/gallium/auxiliary/meson.build @@ -347,7 +347,6 @@ if draw_with_llvm 'gallivm/lp_bld_gather.c', 'gallivm/lp_bld_gather.h', 'gallivm/lp_bld.h', - 'gallivm/lp_bld_init.c', 'gallivm/lp_bld_init.h', 'gallivm/lp_bld_intr.c', 'gallivm/lp_bld_intr.h', @@ -399,6 +398,11 @@ if draw_with_llvm 'nir/nir_to_tgsi_info.c', 'nir/nir_to_tgsi_info.h', ) + if llvm_with_orcjit + files_libgallium += files('gallivm/lp_bld_init_orc.cpp',) + else + files_libgallium += files('gallivm/lp_bld_init.c',) + endif endif files_libgalliumvl = files( Index: mesa-24.0.7/src/gallium/drivers/llvmpipe/lp_context.c =================================================================== --- mesa-24.0.7.orig/src/gallium/drivers/llvmpipe/lp_context.c +++ mesa-24.0.7/src/gallium/drivers/llvmpipe/lp_context.c @@ -49,6 +49,10 @@ #include "lp_screen.h" #include "lp_fence.h" +#if ! (GALLIVM_USE_ORCJIT == 1) +#define USE_GLOBAL_LLVM_CONTEXT +#endif + static void llvmpipe_destroy(struct pipe_context *pipe) { @@ -108,8 +112,12 @@ llvmpipe_destroy(struct pipe_context *pi llvmpipe_sampler_matrix_destroy(llvmpipe); #ifndef USE_GLOBAL_LLVM_CONTEXT +#if GALLIVM_USE_ORCJIT == 1 + LLVMOrcDisposeThreadSafeContext(llvmpipe->context); +#else LLVMContextDispose(llvmpipe->context); #endif +#endif llvmpipe->context = NULL; align_free(llvmpipe); @@ -259,15 +267,25 @@ llvmpipe_create_context(struct pipe_scre #ifdef USE_GLOBAL_LLVM_CONTEXT llvmpipe->context = LLVMGetGlobalContext(); #else +#if GALLIVM_USE_ORCJIT == 1 + llvmpipe->context = LLVMOrcCreateNewThreadSafeContext(); +#else llvmpipe->context = LLVMContextCreate(); #endif +#endif if (!llvmpipe->context) goto fail; +#if GALLIVM_USE_ORCJIT == 1 +#if LLVM_VERSION_MAJOR == 15 + LLVMContextSetOpaquePointers(LLVMOrcThreadSafeContextGetContext(llvmpipe->context), false); +#endif +#else #if LLVM_VERSION_MAJOR == 15 LLVMContextSetOpaquePointers(llvmpipe->context, false); #endif +#endif /* * Create drawing context and plug our rendering stage into it. Index: mesa-24.0.7/src/gallium/drivers/llvmpipe/lp_context.h =================================================================== --- mesa-24.0.7.orig/src/gallium/drivers/llvmpipe/lp_context.h +++ mesa-24.0.7/src/gallium/drivers/llvmpipe/lp_context.h @@ -190,7 +190,11 @@ struct llvmpipe_context { unsigned render_cond_offset; /** The LLVMContext to use for LLVM related work */ +#if GALLIVM_USE_ORCJIT == 1 + LLVMOrcThreadSafeContextRef context; +#else LLVMContextRef context; +#endif int max_global_buffers; struct pipe_resource **global_buffers; Index: mesa-24.0.7/src/gallium/drivers/llvmpipe/lp_screen.c =================================================================== --- mesa-24.0.7.orig/src/gallium/drivers/llvmpipe/lp_screen.c +++ mesa-24.0.7/src/gallium/drivers/llvmpipe/lp_screen.c @@ -955,7 +955,11 @@ static void lp_disk_cache_create(struct llvmpipe_screen *screen) { struct mesa_sha1 ctx; +#if GALLIVM_USE_ORCJIT == 1 + unsigned gallivm_perf = 0; +#else unsigned gallivm_perf = gallivm_get_perf_flags(); +#endif unsigned char sha1[20]; char cache_id[20 * 2 + 1]; _mesa_sha1_init(&ctx); Index: mesa-24.0.7/src/gallium/drivers/llvmpipe/lp_state_cs.c =================================================================== --- mesa-24.0.7.orig/src/gallium/drivers/llvmpipe/lp_state_cs.c +++ mesa-24.0.7/src/gallium/drivers/llvmpipe/lp_state_cs.c @@ -397,6 +397,8 @@ generate_compute(struct llvmpipe_context lp_build_coro_add_presplit(coro); variant->function = function; + variant->function_name = MALLOC(strlen(func_name)+1); + strcpy(variant->function_name, func_name); for (i = 0; i < CS_ARG_MAX - !is_mesh; ++i) { if (LLVMGetTypeKind(arg_types[i]) == LLVMPointerTypeKind) { @@ -1006,6 +1008,8 @@ llvmpipe_remove_cs_shader_variant(struct lp->nr_cs_variants--; lp->nr_cs_instrs -= variant->nr_instrs; + if(variant->function_name) + FREE(variant->function_name); FREE(variant); } @@ -1264,12 +1268,20 @@ generate_variant(struct llvmpipe_context generate_compute(lp, shader, variant); +#if GALLIVM_USE_ORCJIT == 1 +/* module has been moved into ORCJIT after gallivm_compile_module */ + variant->nr_instrs += lp_build_count_ir_module(variant->gallivm->module); + + gallivm_compile_module(variant->gallivm); +#else gallivm_compile_module(variant->gallivm); variant->nr_instrs += lp_build_count_ir_module(variant->gallivm->module); +#endif variant->jit_function = (lp_jit_cs_func) - gallivm_jit_function(variant->gallivm, variant->function); + gallivm_jit_function(variant->gallivm, variant->function, + variant->function_name); if (needs_caching) { lp_disk_cache_insert_shader(screen, &cached, ir_sha1_cache_key); Index: mesa-24.0.7/src/gallium/drivers/llvmpipe/lp_state_cs.h =================================================================== --- mesa-24.0.7.orig/src/gallium/drivers/llvmpipe/lp_state_cs.h +++ mesa-24.0.7/src/gallium/drivers/llvmpipe/lp_state_cs.h @@ -91,6 +91,7 @@ struct lp_compute_shader_variant LLVMTypeRef jit_vertex_header_ptr_type; LLVMTypeRef jit_prim_type; LLVMValueRef function; + char* function_name; lp_jit_cs_func jit_function; /* Total number of LLVM instructions generated */ Index: mesa-24.0.7/src/gallium/drivers/llvmpipe/lp_state_fs.c =================================================================== --- mesa-24.0.7.orig/src/gallium/drivers/llvmpipe/lp_state_fs.c +++ mesa-24.0.7/src/gallium/drivers/llvmpipe/lp_state_fs.c @@ -3207,6 +3207,8 @@ generate_fragment(struct llvmpipe_contex LLVMSetFunctionCallConv(function, LLVMCCallConv); variant->function[partial_mask] = function; + variant->function_name[partial_mask] = MALLOC(strlen(func_name)+1); + strcpy(variant->function_name[partial_mask], func_name); /* XXX: need to propagate noalias down into color param now we are * passing a pointer-to-pointer? @@ -3306,6 +3308,7 @@ generate_fragment(struct llvmpipe_contex LLVMValueRef glob_sample_pos = LLVMAddGlobal(gallivm->module, LLVMArrayType(flt_type, key->coverage_samples * 2), ""); + LLVMSetLinkage(glob_sample_pos, LLVMInternalLinkage); LLVMValueRef sample_pos_array; if (key->multisample && key->coverage_samples == 4) { @@ -3913,20 +3916,29 @@ generate_variant(struct llvmpipe_context * Compile everything */ +#if GALLIVM_USE_ORCJIT == 1 +/* module has been moved into ORCJIT after gallivm_compile_module */ + variant->nr_instrs += lp_build_count_ir_module(variant->gallivm->module); + + gallivm_compile_module(variant->gallivm); +#else gallivm_compile_module(variant->gallivm); variant->nr_instrs += lp_build_count_ir_module(variant->gallivm->module); +#endif if (variant->function[RAST_EDGE_TEST]) { variant->jit_function[RAST_EDGE_TEST] = (lp_jit_frag_func) gallivm_jit_function(variant->gallivm, - variant->function[RAST_EDGE_TEST]); + variant->function[RAST_EDGE_TEST], + variant->function_name[RAST_EDGE_TEST]); } if (variant->function[RAST_WHOLE]) { variant->jit_function[RAST_WHOLE] = (lp_jit_frag_func) gallivm_jit_function(variant->gallivm, - variant->function[RAST_WHOLE]); + variant->function[RAST_WHOLE], + variant->function_name[RAST_WHOLE]); } else if (!variant->jit_function[RAST_WHOLE]) { variant->jit_function[RAST_WHOLE] = (lp_jit_frag_func) variant->jit_function[RAST_EDGE_TEST]; @@ -3935,7 +3947,8 @@ generate_variant(struct llvmpipe_context if (linear_pipeline) { if (variant->linear_function) { variant->jit_linear_llvm = (lp_jit_linear_llvm_func) - gallivm_jit_function(variant->gallivm, variant->linear_function); + gallivm_jit_function(variant->gallivm, variant->linear_function, + variant->linear_function_name); } /* @@ -4111,6 +4124,12 @@ llvmpipe_destroy_shader_variant(struct l { gallivm_destroy(variant->gallivm); lp_fs_reference(lp, &variant->shader, NULL); + if (variant->function_name[RAST_EDGE_TEST]) + FREE(variant->function_name[RAST_EDGE_TEST]); + if (variant->function_name[RAST_WHOLE]) + FREE(variant->function_name[RAST_WHOLE]); + if (variant->linear_function_name) + FREE(variant->linear_function_name); FREE(variant); } Index: mesa-24.0.7/src/gallium/drivers/llvmpipe/lp_state_fs.h =================================================================== --- mesa-24.0.7.orig/src/gallium/drivers/llvmpipe/lp_state_fs.h +++ mesa-24.0.7/src/gallium/drivers/llvmpipe/lp_state_fs.h @@ -168,6 +168,7 @@ struct lp_fragment_shader_variant LLVMTypeRef jit_linear_textures_type; LLVMValueRef function[2]; // [RAST_WHOLE], [RAST_EDGE_TEST] + char* function_name[2]; lp_jit_frag_func jit_function[2]; // [RAST_WHOLE], [RAST_EDGE_TEST] @@ -177,6 +178,7 @@ struct lp_fragment_shader_variant /* Functions within the linear path: */ LLVMValueRef linear_function; + char* linear_function_name; lp_jit_linear_llvm_func jit_linear_llvm; /* Bitmask to say what cbufs are unswizzled */ Index: mesa-24.0.7/src/gallium/drivers/llvmpipe/lp_state_fs_linear_llvm.c =================================================================== --- mesa-24.0.7.orig/src/gallium/drivers/llvmpipe/lp_state_fs_linear_llvm.c +++ mesa-24.0.7/src/gallium/drivers/llvmpipe/lp_state_fs_linear_llvm.c @@ -289,6 +289,8 @@ llvmpipe_fs_variant_linear_llvm(struct l LLVMSetFunctionCallConv(function, LLVMCCallConv); variant->linear_function = function; + variant->linear_function_name = MALLOC(strlen(func_name)+1); + strcpy(variant->linear_function_name, func_name); /* XXX: need to propagate noalias down into color param now we are * passing a pointer-to-pointer? Index: mesa-24.0.7/src/gallium/drivers/llvmpipe/lp_state_setup.c =================================================================== --- mesa-24.0.7.orig/src/gallium/drivers/llvmpipe/lp_state_setup.c +++ mesa-24.0.7/src/gallium/drivers/llvmpipe/lp_state_setup.c @@ -688,6 +688,8 @@ generate_setup_variant(struct lp_setup_v arg_types, ARRAY_SIZE(arg_types), 0); variant->function = LLVMAddFunction(gallivm->module, func_name, func_type); + variant->function_name = MALLOC(strlen(func_name)+1); + strcpy(variant->function_name, func_name); if (!variant->function) goto fail; @@ -732,7 +734,7 @@ generate_setup_variant(struct lp_setup_v gallivm_compile_module(gallivm); variant->jit_function = (lp_jit_setup_triangle) - gallivm_jit_function(gallivm, variant->function); + gallivm_jit_function(gallivm, variant->function, variant->function_name); if (!variant->jit_function) goto fail; Index: mesa-24.0.7/src/gallium/drivers/llvmpipe/lp_state_setup.h =================================================================== --- mesa-24.0.7.orig/src/gallium/drivers/llvmpipe/lp_state_setup.h +++ mesa-24.0.7/src/gallium/drivers/llvmpipe/lp_state_setup.h @@ -66,6 +66,7 @@ struct lp_setup_variant { * assembly. */ LLVMValueRef function; + char *function_name; /* The actual generated setup function: */ Index: mesa-24.0.7/src/gallium/drivers/llvmpipe/lp_test_arit.c =================================================================== --- mesa-24.0.7.orig/src/gallium/drivers/llvmpipe/lp_test_arit.c +++ mesa-24.0.7/src/gallium/drivers/llvmpipe/lp_test_arit.c @@ -417,7 +417,11 @@ test_unary(unsigned verbose, FILE *fp, c { char test_name[128]; snprintf(test_name, sizeof test_name, "%s.v%u", test->name, length); +#if GALLIVM_USE_ORCJIT == 1 + LLVMOrcThreadSafeContextRef context; +#else LLVMContextRef context; +#endif struct gallivm_state *gallivm; LLVMValueRef test_func; unary_func_t test_func_jit; @@ -433,17 +437,25 @@ test_unary(unsigned verbose, FILE *fp, c in[i] = 1.0; } +#if GALLIVM_USE_ORCJIT == 1 + context = LLVMOrcCreateNewThreadSafeContext(); +#if LLVM_VERSION_MAJOR == 15 + LLVMContextSetOpaquePointers(LLVMOrcThreadSafeContextGetContext(context), false); +#endif +#else context = LLVMContextCreate(); #if LLVM_VERSION_MAJOR == 15 LLVMContextSetOpaquePointers(context, false); #endif +#endif gallivm = gallivm_create("test_module", context, NULL); test_func = build_unary_test_func(gallivm, test, length, test_name); gallivm_compile_module(gallivm); - test_func_jit = (unary_func_t) gallivm_jit_function(gallivm, test_func); + test_func_jit = (unary_func_t) gallivm_jit_function(gallivm, test_func, + test_name); gallivm_free_ir(gallivm); @@ -512,7 +524,11 @@ test_unary(unsigned verbose, FILE *fp, c } gallivm_destroy(gallivm); +#if GALLIVM_USE_ORCJIT == 1 + LLVMOrcDisposeThreadSafeContext(context); +#else LLVMContextDispose(context); +#endif align_free(in); align_free(out); Index: mesa-24.0.7/src/gallium/drivers/llvmpipe/lp_test_blend.c =================================================================== --- mesa-24.0.7.orig/src/gallium/drivers/llvmpipe/lp_test_blend.c +++ mesa-24.0.7/src/gallium/drivers/llvmpipe/lp_test_blend.c @@ -437,7 +437,11 @@ test_one(unsigned verbose, const struct pipe_blend_state *blend, struct lp_type type) { +#if GALLIVM_USE_ORCJIT == 1 + LLVMOrcThreadSafeContextRef context; +#else LLVMContextRef context; +#endif struct gallivm_state *gallivm; LLVMValueRef func = NULL; blend_test_ptr_t blend_test_ptr; @@ -451,17 +455,25 @@ test_one(unsigned verbose, if (verbose >= 1) dump_blend_type(stdout, blend, type); +#if GALLIVM_USE_ORCJIT == 1 + context = LLVMOrcCreateNewThreadSafeContext(); +#if LLVM_VERSION_MAJOR == 15 + LLVMContextSetOpaquePointers(LLVMOrcThreadSafeContextGetContext(context), false); +#endif +#else context = LLVMContextCreate(); #if LLVM_VERSION_MAJOR == 15 LLVMContextSetOpaquePointers(context, false); #endif +#endif gallivm = gallivm_create("test_module", context, NULL); func = add_blend_test(gallivm, blend, type); gallivm_compile_module(gallivm); - blend_test_ptr = (blend_test_ptr_t)gallivm_jit_function(gallivm, func); + blend_test_ptr = (blend_test_ptr_t)gallivm_jit_function(gallivm, func, + "test"); gallivm_free_ir(gallivm); @@ -584,7 +596,11 @@ test_one(unsigned verbose, write_tsv_row(fp, blend, type, cycles_avg, success); gallivm_destroy(gallivm); +#if GALLIVM_USE_ORCJIT == 1 + LLVMOrcDisposeThreadSafeContext(context); +#else LLVMContextDispose(context); +#endif return success; } Index: mesa-24.0.7/src/gallium/drivers/llvmpipe/lp_test_conv.c =================================================================== --- mesa-24.0.7.orig/src/gallium/drivers/llvmpipe/lp_test_conv.c +++ mesa-24.0.7/src/gallium/drivers/llvmpipe/lp_test_conv.c @@ -157,7 +157,11 @@ test_one(unsigned verbose, struct lp_type src_type, struct lp_type dst_type) { +#if GALLIVM_USE_ORCJIT == 1 + LLVMOrcThreadSafeContextRef context; +#else LLVMContextRef context; +#endif struct gallivm_state *gallivm; LLVMValueRef func = NULL; conv_test_ptr_t conv_test_ptr; @@ -222,17 +226,25 @@ test_one(unsigned verbose, eps *= 2; } +#if GALLIVM_USE_ORCJIT == 1 + context = LLVMOrcCreateNewThreadSafeContext(); +#if LLVM_VERSION_MAJOR == 15 + LLVMContextSetOpaquePointers(LLVMOrcThreadSafeContextGetContext(context), false); +#endif +#else context = LLVMContextCreate(); #if LLVM_VERSION_MAJOR == 15 LLVMContextSetOpaquePointers(context, false); #endif +#endif gallivm = gallivm_create("test_module", context, NULL); func = add_conv_test(gallivm, src_type, num_srcs, dst_type, num_dsts); gallivm_compile_module(gallivm); - conv_test_ptr = (conv_test_ptr_t)gallivm_jit_function(gallivm, func); + conv_test_ptr = (conv_test_ptr_t)gallivm_jit_function(gallivm, func, + "test"); gallivm_free_ir(gallivm); @@ -337,7 +349,11 @@ test_one(unsigned verbose, write_tsv_row(fp, src_type, dst_type, cycles_avg, success); gallivm_destroy(gallivm); +#if GALLIVM_USE_ORCJIT == 1 + LLVMOrcDisposeThreadSafeContext(context); +#else LLVMContextDispose(context); +#endif return success; } Index: mesa-24.0.7/src/gallium/drivers/llvmpipe/lp_test_format.c =================================================================== --- mesa-24.0.7.orig/src/gallium/drivers/llvmpipe/lp_test_format.c +++ mesa-24.0.7/src/gallium/drivers/llvmpipe/lp_test_format.c @@ -79,9 +79,9 @@ static LLVMValueRef add_fetch_rgba_test(struct gallivm_state *gallivm, unsigned verbose, const struct util_format_description *desc, struct lp_type type, - unsigned use_cache) + unsigned use_cache, + char *name) { - char name[256]; LLVMContextRef context = gallivm->context; LLVMModuleRef module = gallivm->module; LLVMBuilderRef builder = gallivm->builder; @@ -96,7 +96,7 @@ add_fetch_rgba_test(struct gallivm_state LLVMValueRef rgba; LLVMValueRef cache = NULL; - snprintf(name, sizeof name, "fetch_%s_%s", desc->short_name, + snprintf(name, 64 * sizeof(char), "fetch_%s_%s", desc->short_name, type.floating ? "float" : "unorm8"); args[0] = LLVMPointerType(lp_build_vec_type(gallivm, type), 0); @@ -139,9 +139,14 @@ test_format_float(unsigned verbose, FILE const struct util_format_description *desc, unsigned use_cache) { +#if GALLIVM_USE_ORCJIT == 1 + LLVMOrcThreadSafeContextRef context; +#else LLVMContextRef context; +#endif struct gallivm_state *gallivm; LLVMValueRef fetch = NULL; + char fetch_name[64]; fetch_ptr_t fetch_ptr; alignas(16) uint8_t packed[UTIL_FORMAT_MAX_PACKED_BYTES]; alignas(16) float unpacked[4]; @@ -149,18 +154,25 @@ test_format_float(unsigned verbose, FILE bool success = true; unsigned i, j, k, l; +#if GALLIVM_USE_ORCJIT == 1 + context = LLVMOrcCreateNewThreadSafeContext(); +#if LLVM_VERSION_MAJOR == 15 + LLVMContextSetOpaquePointers(LLVMOrcThreadSafeContextGetContext(context), false); +#endif +#else context = LLVMContextCreate(); #if LLVM_VERSION_MAJOR == 15 LLVMContextSetOpaquePointers(context, false); #endif +#endif gallivm = gallivm_create("test_module_float", context, NULL); fetch = add_fetch_rgba_test(gallivm, verbose, desc, - lp_float32_vec4_type(), use_cache); + lp_float32_vec4_type(), use_cache, fetch_name); gallivm_compile_module(gallivm); - fetch_ptr = (fetch_ptr_t) gallivm_jit_function(gallivm, fetch); + fetch_ptr = (fetch_ptr_t) gallivm_jit_function(gallivm, fetch, fetch_name); gallivm_free_ir(gallivm); @@ -228,7 +240,11 @@ test_format_float(unsigned verbose, FILE } gallivm_destroy(gallivm); +#if GALLIVM_USE_ORCJIT == 1 + LLVMOrcDisposeThreadSafeContext(context); +#else LLVMContextDispose(context); +#endif if (fp) write_tsv_row(fp, desc, success); @@ -243,9 +259,14 @@ test_format_unorm8(unsigned verbose, FIL const struct util_format_description *desc, unsigned use_cache) { +#if GALLIVM_USE_ORCJIT == 1 + LLVMOrcThreadSafeContextRef context; +#else LLVMContextRef context; +#endif struct gallivm_state *gallivm; LLVMValueRef fetch = NULL; + char fetch_name[64]; fetch_ptr_t fetch_ptr; alignas(16) uint8_t packed[UTIL_FORMAT_MAX_PACKED_BYTES]; uint8_t unpacked[4]; @@ -253,18 +274,25 @@ test_format_unorm8(unsigned verbose, FIL bool success = true; unsigned i, j, k, l; +#if GALLIVM_USE_ORCJIT == 1 + context = LLVMOrcCreateNewThreadSafeContext(); +#if LLVM_VERSION_MAJOR == 15 + LLVMContextSetOpaquePointers(LLVMOrcThreadSafeContextGetContext(context), false); +#endif +#else context = LLVMContextCreate(); #if LLVM_VERSION_MAJOR == 15 LLVMContextSetOpaquePointers(context, false); #endif +#endif gallivm = gallivm_create("test_module_unorm8", context, NULL); fetch = add_fetch_rgba_test(gallivm, verbose, desc, - lp_unorm8_vec4_type(), use_cache); + lp_unorm8_vec4_type(), use_cache, fetch_name); gallivm_compile_module(gallivm); - fetch_ptr = (fetch_ptr_t) gallivm_jit_function(gallivm, fetch); + fetch_ptr = (fetch_ptr_t) gallivm_jit_function(gallivm, fetch, fetch_name); gallivm_free_ir(gallivm); @@ -331,7 +359,11 @@ test_format_unorm8(unsigned verbose, FIL } gallivm_destroy(gallivm); +#if GALLIVM_USE_ORCJIT == 1 + LLVMOrcDisposeThreadSafeContext(context); +#else LLVMContextDispose(context); +#endif if (fp) write_tsv_row(fp, desc, success); Index: mesa-24.0.7/src/gallium/drivers/llvmpipe/lp_test_lookup_multiple.c =================================================================== --- /dev/null +++ mesa-24.0.7/src/gallium/drivers/llvmpipe/lp_test_lookup_multiple.c @@ -0,0 +1,167 @@ +/************************************************************************** + * + * Copyright 2010 VMware, Inc. + * All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the + * "Software"), to deal in the Software without restriction, including + * without limitation the rights to use, copy, modify, merge, publish, + * distribute, sub license, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to + * the following conditions: + * + * The above copyright notice and this permission notice (including the + * next paragraph) shall be included in all copies or substantial portions + * of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS + * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. + * IN NO EVENT SHALL VMWARE AND/OR ITS SUPPLIERS BE LIABLE FOR + * ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, + * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + * + **************************************************************************/ + + +#include +#include + +#include "util/u_pointer.h" +#include "gallivm/lp_bld.h" +#include "gallivm/lp_bld_init.h" +#include "gallivm/lp_bld_assert.h" +#include "gallivm/lp_bld_printf.h" + +#include "lp_test.h" + + +struct printf_test_case { + int foo; +}; + +void +write_tsv_header(FILE *fp) +{ + fprintf(fp, + "result\t" + "format\n"); + + fflush(fp); +} + + + +typedef void (*test_printf_t)(int i); + + +static LLVMValueRef +add_printf_test(struct gallivm_state *gallivm, int n, char *func_name) +{ + LLVMModuleRef module = gallivm->module; + LLVMTypeRef args[1] = { LLVMIntTypeInContext(gallivm->context, 32) }; + snprintf(func_name, 64 * sizeof(char), "test_lookup_multiple_%d", n); + LLVMValueRef func = LLVMAddFunction(module, func_name, LLVMFunctionType(LLVMVoidTypeInContext(gallivm->context), args, 1, 0)); + LLVMBuilderRef builder = gallivm->builder; + LLVMBasicBlockRef block = LLVMAppendBasicBlockInContext(gallivm->context, func, "entry"); + + LLVMSetFunctionCallConv(func, LLVMCCallConv); + + LLVMPositionBuilderAtEnd(builder, block); + lp_build_printf(gallivm, "hello, world from "); + lp_build_printf(gallivm, func_name); + lp_build_printf(gallivm, "print 5 6: %d %d\n", LLVMConstInt(LLVMInt32TypeInContext(gallivm->context), 5, 0), + LLVMConstInt(LLVMInt32TypeInContext(gallivm->context), 6, 0)); + + /* Also test lp_build_assert(). This should not fail. */ + lp_build_assert(gallivm, LLVMConstInt(LLVMInt32TypeInContext(gallivm->context), 1, 0), "assert(1)"); + + LLVMBuildRetVoid(builder); + + gallivm_verify_function(gallivm, func); + + return func; +} + + +static bool +test_lookup_multiple(unsigned verbose, FILE *fp, + const struct printf_test_case *testcase) +{ + struct gallivm_state *gallivm; + const int N = 10; + LLVMValueRef *func = + (LLVMValueRef *) malloc(N * sizeof(LLVMValueRef)); + char func_name[N][64]; + test_printf_t *test_lookup_multiple_func = + (test_printf_t *)malloc(N * sizeof(test_printf_t)); + bool success = true; + int i; + +#if GALLIVM_USE_ORCJIT == 1 + LLVMOrcThreadSafeContextRef context = LLVMOrcCreateNewThreadSafeContext(); +#if LLVM_VERSION_MAJOR == 15 + LLVMContextSetOpaquePointers(LLVMOrcThreadSafeContextGetContext(context), false); +#endif +#else + LLVMContextRef context = LLVMContextCreate(); +#if LLVM_VERSION_MAJOR == 15 + LLVMContextSetOpaquePointers(context, false); +#endif +#endif + gallivm = gallivm_create("test_module", context, NULL); + + for(i = 0; i < N; i++){ + func[i] = add_printf_test(gallivm, i, func_name[i]); + } + + gallivm_compile_module(gallivm); + + for(i = 0; i < N; i++){ + test_lookup_multiple_func[i] = (test_printf_t) gallivm_jit_function(gallivm, func[i], func_name[i]); + } + + gallivm_free_ir(gallivm); + + for(i = 0; i < N; i++){ + test_lookup_multiple_func[i](0); + } + FREE(func); + FREE(test_lookup_multiple_func); + gallivm_destroy(gallivm); +#if GALLIVM_USE_ORCJIT == 1 + LLVMOrcDisposeThreadSafeContext(context); +#else + LLVMContextDispose(context); +#endif + return success; +} + + +bool +test_all(unsigned verbose, FILE *fp) +{ + bool success = true; + + test_lookup_multiple(verbose, fp, NULL); + + return success; +} + + +bool +test_some(unsigned verbose, FILE *fp, + unsigned long n) +{ + return test_all(verbose, fp); +} + + +bool +test_single(unsigned verbose, FILE *fp) +{ + printf("no test_single()"); + return true; +} Index: mesa-24.0.7/src/gallium/drivers/llvmpipe/lp_test_printf.c =================================================================== --- mesa-24.0.7.orig/src/gallium/drivers/llvmpipe/lp_test_printf.c +++ mesa-24.0.7/src/gallium/drivers/llvmpipe/lp_test_printf.c @@ -89,30 +89,46 @@ static bool test_printf(unsigned verbose, FILE *fp, const struct printf_test_case *testcase) { +#if GALLIVM_USE_ORCJIT == 1 + LLVMOrcThreadSafeContextRef context; +#else LLVMContextRef context; +#endif struct gallivm_state *gallivm; LLVMValueRef test; test_printf_t test_printf_func; bool success = true; +#if GALLIVM_USE_ORCJIT == 1 + context = LLVMOrcCreateNewThreadSafeContext(); +#if LLVM_VERSION_MAJOR == 15 + LLVMContextSetOpaquePointers(LLVMOrcThreadSafeContextGetContext(context), false); +#endif +#else context = LLVMContextCreate(); #if LLVM_VERSION_MAJOR == 15 LLVMContextSetOpaquePointers(context, false); #endif +#endif gallivm = gallivm_create("test_module", context, NULL); test = add_printf_test(gallivm); gallivm_compile_module(gallivm); - test_printf_func = (test_printf_t) gallivm_jit_function(gallivm, test); + test_printf_func = (test_printf_t) gallivm_jit_function(gallivm, test, + "test_printf"); gallivm_free_ir(gallivm); test_printf_func(0); gallivm_destroy(gallivm); +#if GALLIVM_USE_ORCJIT == 1 + LLVMOrcDisposeThreadSafeContext(context); +#else LLVMContextDispose(context); +#endif return success; } Index: mesa-24.0.7/src/gallium/drivers/llvmpipe/lp_texture_handle.c =================================================================== --- mesa-24.0.7.orig/src/gallium/drivers/llvmpipe/lp_texture_handle.c +++ mesa-24.0.7/src/gallium/drivers/llvmpipe/lp_texture_handle.c @@ -201,14 +201,15 @@ llvmpipe_sampler_matrix_destroy(struct l } static void * -compile_function(struct llvmpipe_context *ctx, struct gallivm_state *gallivm, LLVMValueRef function, +compile_function(struct llvmpipe_context *ctx, struct gallivm_state *gallivm, LLVMValueRef function, const char *func_name, bool needs_caching, uint8_t cache_key[SHA1_DIGEST_LENGTH]) { gallivm_verify_function(gallivm, function); gallivm_compile_module(gallivm); - void *function_ptr = func_to_pointer(gallivm_jit_function(gallivm, function)); + void *function_ptr = func_to_pointer(gallivm_jit_function(gallivm, function, + func_name)); if (needs_caching) lp_disk_cache_insert_shader(llvmpipe_screen(ctx->pipe.screen), gallivm->cache, cache_key); @@ -336,7 +337,7 @@ compile_image_function(struct llvmpipe_c free(image_soa); - return compile_function(ctx, gallivm, function, needs_caching, cache_key); + return compile_function(ctx, gallivm, function, "image", needs_caching, cache_key); } static void * @@ -484,7 +485,7 @@ compile_sample_function(struct llvmpipe_ free(sampler_soa); - return compile_function(ctx, gallivm, function, needs_caching, cache_key); + return compile_function(ctx, gallivm, function, "sample", needs_caching, cache_key); } static void * @@ -566,7 +567,8 @@ compile_size_function(struct llvmpipe_co free(sampler_soa); - return compile_function(ctx, gallivm, function, needs_caching, cache_key); + return compile_function(ctx, gallivm, function, "sample", needs_caching, + cache_key); } static void Index: mesa-24.0.7/src/gallium/drivers/llvmpipe/meson.build =================================================================== --- mesa-24.0.7.orig/src/gallium/drivers/llvmpipe/meson.build +++ mesa-24.0.7/src/gallium/drivers/llvmpipe/meson.build @@ -132,7 +132,7 @@ driver_swrast = declare_dependency( if with_tests and with_gallium_softpipe and draw_with_llvm foreach t : ['lp_test_format', 'lp_test_arit', 'lp_test_blend', - 'lp_test_conv', 'lp_test_printf'] + 'lp_test_conv', 'lp_test_printf', 'lp_test_lookup_multiple'] test( t, executable( Index: mesa-24.0.7/src/util/detect_arch.h =================================================================== --- mesa-24.0.7.orig/src/util/detect_arch.h +++ mesa-24.0.7/src/util/detect_arch.h @@ -101,6 +101,17 @@ #define DETECT_ARCH_HPPA 1 #endif +#if defined(__riscv) +#define DETECT_ARCH_RISCV 1 +#if __riscv_xlen == 64 +#define DETECT_ARCH_RISCV64 1 +#elif __riscv_xlen == 32 +#define DETECT_ARCH_RISCV32 1 +#else +#error "detect_arch: unknown target riscv xlen" +#endif +#endif + #ifndef DETECT_ARCH_X86 #define DETECT_ARCH_X86 0 #endif @@ -145,4 +156,16 @@ #define DETECT_ARCH_HPPA 0 #endif +#ifndef DETECT_ARCH_RISCV +#define DETECT_ARCH_RISCV 0 +#endif + +#ifndef DETECT_ARCH_RISCV32 +#define DETECT_ARCH_RISCV32 0 +#endif + +#ifndef DETECT_ARCH_RISCV64 +#define DETECT_ARCH_RISCV64 0 +#endif + #endif /* UTIL_DETECT_ARCH_H_ */