From 885a4df473e7b237aef23d38a50ee6535de372c1 Mon Sep 17 00:00:00 2001 From: Shaw Summa Date: Fri, 8 Mar 2024 09:19:41 -0500 Subject: [PATCH] tmp Signed-off-by: Shaw Summa --- .gitignore | 3 +- .gitmodules | 16 + base.mak | 4 + makefile | 55 +- pre.sh | 19 - tcc.mak | 7 + vendor/bdwgc/allchblk.c | 990 - vendor/bdwgc/alloc.c | 1697 - vendor/bdwgc/blacklst.c | 296 - vendor/bdwgc/dbg_mlc.c | 1168 - vendor/bdwgc/dyn_load.c | 1516 - vendor/bdwgc/finalize.c | 1376 - vendor/bdwgc/headers.c | 414 - vendor/bdwgc/mach_dep.c | 448 - vendor/bdwgc/malloc.c | 634 - vendor/bdwgc/mallocx.c | 597 - vendor/bdwgc/mark.c | 1912 - vendor/bdwgc/mark_rts.c | 904 - vendor/bdwgc/misc.c | 2533 -- vendor/bdwgc/new_hblk.c | 190 - vendor/bdwgc/obj_map.c | 89 - vendor/bdwgc/os_dep.c | 5091 --- vendor/bdwgc/private/darwin_semaphore.h | 86 - vendor/bdwgc/private/dbg_mlc.h | 180 - vendor/bdwgc/private/gc/gc.h | 2248 -- vendor/bdwgc/private/gc/gc_config_macros.h | 413 - vendor/bdwgc/private/gc/gc_inline.h | 208 - vendor/bdwgc/private/gc/gc_mark.h | 388 - vendor/bdwgc/private/gc/gc_tiny_fl.h | 86 - vendor/bdwgc/private/gc/gc_version.h | 45 - vendor/bdwgc/private/gc_alloc_ptrs.h | 60 - vendor/bdwgc/private/gc_atomic_ops.h | 122 - vendor/bdwgc/private/gc_hdrs.h | 215 - vendor/bdwgc/private/gc_locks.h | 308 - vendor/bdwgc/private/gc_pmark.h | 456 - vendor/bdwgc/private/gc_priv.h | 3130 -- vendor/bdwgc/private/gcconfig.h | 3224 -- vendor/bdwgc/private/specific.h | 132 - vendor/bdwgc/ptr_chck.c | 293 - vendor/bdwgc/reclaim.c | 820 - vendor/c11threads/threads.h | 151 - vendor/c11threads/threads_msvc.c | 460 - vendor/c11threads/threads_posix.c | 271 - vendor/common/README.txt | 10 - vendor/common/arena.h | 77 - vendor/common/buffer.h | 55 - vendor/common/chunked_array.h | 87 - vendor/common/common.c | 379 - vendor/common/common.h | 89 - vendor/common/dyn_array.h | 119 - vendor/common/file_map.h | 83 - vendor/common/futex.h | 16 - vendor/common/hash_map.h | 363 - vendor/common/hashes.h | 84 - vendor/common/log.c | 167 - vendor/common/log.h | 58 - vendor/common/new_hash_map.h | 357 - vendor/common/perf.c | 226 - vendor/common/perf.h | 36 - vendor/common/spall.h | 454 - vendor/common/spall_native_auto.h | 803 - vendor/cuik | 1 + vendor/isocline | 1 + vendor/isocline/attr.c | 294 - vendor/isocline/attr.h | 70 - vendor/isocline/bbcode.c | 842 - vendor/isocline/bbcode.h | 37 - vendor/isocline/bbcode_colors.c | 194 - vendor/isocline/common.c | 347 - vendor/isocline/common.h | 187 - vendor/isocline/completers.c | 675 - vendor/isocline/completions.c | 326 - vendor/isocline/completions.h | 52 - vendor/isocline/editline.c | 1142 - vendor/isocline/editline_completion.c | 277 - vendor/isocline/editline_help.c | 140 - vendor/isocline/editline_history.c | 260 - vendor/isocline/env.h | 60 - vendor/isocline/highlight.c | 259 - vendor/isocline/highlight.h | 24 - vendor/isocline/history.c | 269 - vendor/isocline/history.h | 38 - vendor/isocline/isocline.c | 774 - vendor/isocline/isocline.h | 605 - vendor/isocline/stringbuf.c | 1038 - vendor/isocline/stringbuf.h | 121 - vendor/isocline/term.c | 1124 - vendor/isocline/term.h | 85 - vendor/isocline/term_color.c | 371 - vendor/isocline/tty.c | 889 - vendor/isocline/tty.h | 160 - vendor/isocline/tty_esc.c | 401 - vendor/isocline/undo.c | 67 - vendor/isocline/undo.h | 24 - vendor/isocline/wcwidth.c | 292 - vendor/tb/.editorconfig | 11 - vendor/tb/.gitignore | 26 - vendor/tb/IR.txt | 58 - vendor/tb/LICENSE.txt | 21 - vendor/tb/NOTES.txt | 17 - vendor/tb/README.txt | 22 - vendor/tb/include/sdg.h | 111 - vendor/tb/include/tb.h | 1468 - vendor/tb/include/tb_coff.h | 334 - vendor/tb/include/tb_elf.h | 171 - vendor/tb/include/tb_formats.h | 132 - vendor/tb/include/tb_x64.h | 89 - vendor/tb/src/aarch64/aarch64.c | 218 - vendor/tb/src/aarch64/aarch64_emitter.h | 186 - vendor/tb/src/aarch64/aarch64_target.c | 481 - vendor/tb/src/abi.c | 247 - vendor/tb/src/builtins.h | 108 - vendor/tb/src/chaitin.c | 268 - vendor/tb/src/codegen.h | 309 - vendor/tb/src/codegen_impl.h | 726 - vendor/tb/src/debug/cv.c | 553 - vendor/tb/src/debug/cv.h | 53 - vendor/tb/src/debug/cv_type_builder.c | 330 - vendor/tb/src/debug/sdg.c | 221 - vendor/tb/src/debug_builder.c | 127 - vendor/tb/src/disasm.c | 21 - vendor/tb/src/emitter.h | 168 - vendor/tb/src/exporter.c | 246 - vendor/tb/src/generic_cg.h | 1109 - vendor/tb/src/hash.c | 112 - vendor/tb/src/host.h | 19 - vendor/tb/src/ir_printer.c | 373 - vendor/tb/src/jit.c | 584 - vendor/tb/src/libtb.c | 109 - vendor/tb/src/linker/README.txt | 40 - vendor/tb/src/linker/elf.c | 215 - vendor/tb/src/linker/linker.c | 797 - vendor/tb/src/linker/linker.h | 289 - vendor/tb/src/linker/pe.c | 1150 - vendor/tb/src/lsra.c | 942 - vendor/tb/src/mips/mips_insts.inc | 38 - vendor/tb/src/mips/mips_target.c | 910 - vendor/tb/src/mips/targets.txt | 23 - vendor/tb/src/new_builder.c | 639 - vendor/tb/src/objects/coff.c | 669 - vendor/tb/src/objects/coff.h | 380 - vendor/tb/src/objects/coff_parse.c | 53 - vendor/tb/src/objects/elf64.c | 289 - vendor/tb/src/objects/lib_parse.h | 143 - vendor/tb/src/objects/macho.c | 117 - vendor/tb/src/objects/macho.h | 97 - vendor/tb/src/objects/wasm_obj.c | 157 - vendor/tb/src/opt/branches.h | 550 - vendor/tb/src/opt/cfg.h | 288 - vendor/tb/src/opt/fold.h | 1026 - vendor/tb/src/opt/gcm.h | 290 - vendor/tb/src/opt/gvn.h | 190 - vendor/tb/src/opt/lattice.h | 362 - vendor/tb/src/opt/legalizer.h | 171 - vendor/tb/src/opt/libcalls.h | 47 - vendor/tb/src/opt/loop.h | 670 - vendor/tb/src/opt/mem2reg.h | 426 - vendor/tb/src/opt/mem_opt.h | 295 - vendor/tb/src/opt/optimizer.c | 1757 - vendor/tb/src/opt/passes.h | 551 - vendor/tb/src/opt/peeps.h | 96 - vendor/tb/src/opt/print.h | 521 - vendor/tb/src/opt/print_c.h | 1672 - vendor/tb/src/opt/print_dumb.h | 105 - vendor/tb/src/opt/properties.h | 89 - vendor/tb/src/opt/scheduler.h | 193 - vendor/tb/src/opt/sroa.h | 123 - vendor/tb/src/reg_alloc.h | 970 - vendor/tb/src/set.h | 161 - vendor/tb/src/symbols.c | 75 - vendor/tb/src/tb.c | 690 - vendor/tb/src/tb_builder.c | 1057 - vendor/tb/src/tb_internal.h | 589 - vendor/tb/src/tb_platform.h | 30 - vendor/tb/src/wasm/wasm_target.c | 864 - vendor/tb/src/x64/x64.c | 2391 -- vendor/tb/src/x64/x64.h | 264 - vendor/tb/src/x64/x64_disasm.c | 987 - vendor/tb/src/x64/x64_emitter.h | 328 - vendor/tb/src/x64/x64_insts.inc | 138 - vendor/tb/src/x64/x64_target.c | 2177 -- vendor/tb/tests/cg_test.c | 28 - vendor/tb/unittests/tb_test_exit_status.inc | 11 - vendor/tb/unittests/tb_test_int_arith.inc | 67 - vendor/tb/unittests/tb_test_regressions.inc | 26 - vendor/tb/unittests/tb_unittests.c | 110 - vendor/tb/unittests/util.inc | 226 - vendor/tcc | 1 + vendor/tcc/coff.h | 446 - vendor/tcc/config.h | 11 - vendor/tcc/dwarf.h | 1046 - vendor/tcc/i386-asm.c | 1744 - vendor/tcc/i386-asm.h | 487 - vendor/tcc/i386-tok.h | 332 - vendor/tcc/lib/libtcc1.c | 641 - vendor/tcc/libtcc.c | 2253 -- vendor/tcc/libtcc.h | 111 - vendor/tcc/stab.def | 234 - vendor/tcc/stab.h | 17 - vendor/tcc/tcc.h | 1935 - vendor/tcc/tccasm.c | 1363 - vendor/tcc/tccdbg.c | 2181 -- vendor/tcc/tccdefs_.h | 318 - vendor/tcc/tccelf.c | 3956 -- vendor/tcc/tccgen.c | 8698 ----- vendor/tcc/tcclib.h | 80 - vendor/tcc/tccpp.c | 3961 -- vendor/tcc/tccrun.c | 1423 - vendor/tcc/tcctok.h | 420 - vendor/tcc/tcctools.c | 649 - vendor/tcc/x86_64-asm.h | 544 - vendor/tcc/x86_64-gen.c | 2331 -- vendor/tcc/x86_64-link.c | 403 - vendor/tree-sitter | 1 + vendor/trees/alloc.c | 47 - vendor/trees/api.h | 1180 - vendor/trees/array.h | 249 - vendor/trees/atomic.h | 67 - vendor/trees/clock.h | 146 - vendor/trees/error_costs.h | 11 - vendor/trees/get_changed_ranges.c | 537 - vendor/trees/get_changed_ranges.h | 36 - vendor/trees/host.h | 21 - vendor/trees/language.c | 222 - vendor/trees/language.h | 296 - vendor/trees/length.h | 52 - vendor/trees/lexer.c | 425 - vendor/trees/lexer.h | 49 - vendor/trees/node.c | 815 - vendor/trees/parser.c | 2106 - vendor/trees/parser.h | 236 - vendor/trees/point.h | 62 - vendor/trees/query.c | 4265 -- vendor/trees/reduce_action.h | 34 - vendor/trees/reusable_node.h | 95 - vendor/trees/stack.c | 962 - vendor/trees/stack.h | 133 - vendor/trees/subtree.c | 1077 - vendor/trees/subtree.h | 382 - vendor/trees/tree.c | 145 - vendor/trees/tree.h | 31 - vendor/trees/tree_cursor.c | 734 - vendor/trees/tree_cursor.h | 47 - vendor/trees/unicode.h | 50 - vendor/trees/unicode/ICU_SHA | 1 - vendor/trees/unicode/LICENSE | 414 - vendor/trees/unicode/README.md | 29 - vendor/trees/unicode/ptypes.h | 1 - vendor/trees/unicode/umachine.h | 448 - vendor/trees/unicode/urename.h | 1 - vendor/trees/unicode/utf.h | 1 - vendor/trees/unicode/utf16.h | 733 - vendor/trees/unicode/utf8.h | 881 - vendor/xxhash | 1 + vendor/xxhash/xxhash.c | 42 - vendor/xxhash/xxhash.h | 7048 ---- vm/backend/tb.c | 8 +- vm/lua/.gitignore | 1 + vm/lua/ast.c | 2 +- vm/lua/parser.c | 32440 ---------------- vm/lua/{ => parser}/grammar.js | 0 vm/lua/parser/parser.c | 25778 ++++++++++++ vm/lua/{ => parser}/scan.c | 2 +- .../lua/parser/tree_sitter}/alloc.h | 27 +- vm/lua/parser/tree_sitter/array.h | 287 + vm/lua/{ => parser/tree_sitter}/parser.h | 16 +- vm/lua/repl.c | 14 +- vm/lua/repl.h | 4 +- 268 files changed, 26192 insertions(+), 176836 deletions(-) create mode 100644 .gitmodules create mode 100644 base.mak delete mode 100755 pre.sh create mode 100644 tcc.mak delete mode 100644 vendor/bdwgc/allchblk.c delete mode 100644 vendor/bdwgc/alloc.c delete mode 100644 vendor/bdwgc/blacklst.c delete mode 100644 vendor/bdwgc/dbg_mlc.c delete mode 100644 vendor/bdwgc/dyn_load.c delete mode 100644 vendor/bdwgc/finalize.c delete mode 100644 vendor/bdwgc/headers.c delete mode 100644 vendor/bdwgc/mach_dep.c delete mode 100644 vendor/bdwgc/malloc.c delete mode 100644 vendor/bdwgc/mallocx.c delete mode 100644 vendor/bdwgc/mark.c delete mode 100644 vendor/bdwgc/mark_rts.c delete mode 100644 vendor/bdwgc/misc.c delete mode 100644 vendor/bdwgc/new_hblk.c delete mode 100644 vendor/bdwgc/obj_map.c delete mode 100644 vendor/bdwgc/os_dep.c delete mode 100644 vendor/bdwgc/private/darwin_semaphore.h delete mode 100644 vendor/bdwgc/private/dbg_mlc.h delete mode 100644 vendor/bdwgc/private/gc/gc.h delete mode 100644 vendor/bdwgc/private/gc/gc_config_macros.h delete mode 100644 vendor/bdwgc/private/gc/gc_inline.h delete mode 100644 vendor/bdwgc/private/gc/gc_mark.h delete mode 100644 vendor/bdwgc/private/gc/gc_tiny_fl.h delete mode 100644 vendor/bdwgc/private/gc/gc_version.h delete mode 100644 vendor/bdwgc/private/gc_alloc_ptrs.h delete mode 100644 vendor/bdwgc/private/gc_atomic_ops.h delete mode 100644 vendor/bdwgc/private/gc_hdrs.h delete mode 100644 vendor/bdwgc/private/gc_locks.h delete mode 100644 vendor/bdwgc/private/gc_pmark.h delete mode 100644 vendor/bdwgc/private/gc_priv.h delete mode 100644 vendor/bdwgc/private/gcconfig.h delete mode 100644 vendor/bdwgc/private/specific.h delete mode 100644 vendor/bdwgc/ptr_chck.c delete mode 100644 vendor/bdwgc/reclaim.c delete mode 100644 vendor/c11threads/threads.h delete mode 100644 vendor/c11threads/threads_msvc.c delete mode 100644 vendor/c11threads/threads_posix.c delete mode 100644 vendor/common/README.txt delete mode 100644 vendor/common/arena.h delete mode 100644 vendor/common/buffer.h delete mode 100644 vendor/common/chunked_array.h delete mode 100644 vendor/common/common.c delete mode 100644 vendor/common/common.h delete mode 100644 vendor/common/dyn_array.h delete mode 100644 vendor/common/file_map.h delete mode 100644 vendor/common/futex.h delete mode 100644 vendor/common/hash_map.h delete mode 100644 vendor/common/hashes.h delete mode 100644 vendor/common/log.c delete mode 100644 vendor/common/log.h delete mode 100644 vendor/common/new_hash_map.h delete mode 100644 vendor/common/perf.c delete mode 100644 vendor/common/perf.h delete mode 100644 vendor/common/spall.h delete mode 100644 vendor/common/spall_native_auto.h create mode 160000 vendor/cuik create mode 160000 vendor/isocline delete mode 100644 vendor/isocline/attr.c delete mode 100644 vendor/isocline/attr.h delete mode 100644 vendor/isocline/bbcode.c delete mode 100644 vendor/isocline/bbcode.h delete mode 100644 vendor/isocline/bbcode_colors.c delete mode 100644 vendor/isocline/common.c delete mode 100644 vendor/isocline/common.h delete mode 100644 vendor/isocline/completers.c delete mode 100644 vendor/isocline/completions.c delete mode 100644 vendor/isocline/completions.h delete mode 100644 vendor/isocline/editline.c delete mode 100644 vendor/isocline/editline_completion.c delete mode 100644 vendor/isocline/editline_help.c delete mode 100644 vendor/isocline/editline_history.c delete mode 100644 vendor/isocline/env.h delete mode 100644 vendor/isocline/highlight.c delete mode 100644 vendor/isocline/highlight.h delete mode 100644 vendor/isocline/history.c delete mode 100644 vendor/isocline/history.h delete mode 100644 vendor/isocline/isocline.c delete mode 100644 vendor/isocline/isocline.h delete mode 100644 vendor/isocline/stringbuf.c delete mode 100644 vendor/isocline/stringbuf.h delete mode 100644 vendor/isocline/term.c delete mode 100644 vendor/isocline/term.h delete mode 100644 vendor/isocline/term_color.c delete mode 100644 vendor/isocline/tty.c delete mode 100644 vendor/isocline/tty.h delete mode 100644 vendor/isocline/tty_esc.c delete mode 100644 vendor/isocline/undo.c delete mode 100644 vendor/isocline/undo.h delete mode 100644 vendor/isocline/wcwidth.c delete mode 100644 vendor/tb/.editorconfig delete mode 100644 vendor/tb/.gitignore delete mode 100644 vendor/tb/IR.txt delete mode 100644 vendor/tb/LICENSE.txt delete mode 100644 vendor/tb/NOTES.txt delete mode 100644 vendor/tb/README.txt delete mode 100644 vendor/tb/include/sdg.h delete mode 100644 vendor/tb/include/tb.h delete mode 100644 vendor/tb/include/tb_coff.h delete mode 100644 vendor/tb/include/tb_elf.h delete mode 100644 vendor/tb/include/tb_formats.h delete mode 100644 vendor/tb/include/tb_x64.h delete mode 100644 vendor/tb/src/aarch64/aarch64.c delete mode 100644 vendor/tb/src/aarch64/aarch64_emitter.h delete mode 100644 vendor/tb/src/aarch64/aarch64_target.c delete mode 100644 vendor/tb/src/abi.c delete mode 100644 vendor/tb/src/builtins.h delete mode 100644 vendor/tb/src/chaitin.c delete mode 100644 vendor/tb/src/codegen.h delete mode 100644 vendor/tb/src/codegen_impl.h delete mode 100644 vendor/tb/src/debug/cv.c delete mode 100644 vendor/tb/src/debug/cv.h delete mode 100644 vendor/tb/src/debug/cv_type_builder.c delete mode 100644 vendor/tb/src/debug/sdg.c delete mode 100644 vendor/tb/src/debug_builder.c delete mode 100644 vendor/tb/src/disasm.c delete mode 100644 vendor/tb/src/emitter.h delete mode 100644 vendor/tb/src/exporter.c delete mode 100644 vendor/tb/src/generic_cg.h delete mode 100644 vendor/tb/src/hash.c delete mode 100644 vendor/tb/src/host.h delete mode 100644 vendor/tb/src/ir_printer.c delete mode 100644 vendor/tb/src/jit.c delete mode 100644 vendor/tb/src/libtb.c delete mode 100644 vendor/tb/src/linker/README.txt delete mode 100644 vendor/tb/src/linker/elf.c delete mode 100644 vendor/tb/src/linker/linker.c delete mode 100644 vendor/tb/src/linker/linker.h delete mode 100644 vendor/tb/src/linker/pe.c delete mode 100644 vendor/tb/src/lsra.c delete mode 100644 vendor/tb/src/mips/mips_insts.inc delete mode 100644 vendor/tb/src/mips/mips_target.c delete mode 100644 vendor/tb/src/mips/targets.txt delete mode 100644 vendor/tb/src/new_builder.c delete mode 100644 vendor/tb/src/objects/coff.c delete mode 100644 vendor/tb/src/objects/coff.h delete mode 100644 vendor/tb/src/objects/coff_parse.c delete mode 100644 vendor/tb/src/objects/elf64.c delete mode 100644 vendor/tb/src/objects/lib_parse.h delete mode 100644 vendor/tb/src/objects/macho.c delete mode 100644 vendor/tb/src/objects/macho.h delete mode 100644 vendor/tb/src/objects/wasm_obj.c delete mode 100644 vendor/tb/src/opt/branches.h delete mode 100644 vendor/tb/src/opt/cfg.h delete mode 100644 vendor/tb/src/opt/fold.h delete mode 100644 vendor/tb/src/opt/gcm.h delete mode 100644 vendor/tb/src/opt/gvn.h delete mode 100644 vendor/tb/src/opt/lattice.h delete mode 100644 vendor/tb/src/opt/legalizer.h delete mode 100644 vendor/tb/src/opt/libcalls.h delete mode 100644 vendor/tb/src/opt/loop.h delete mode 100644 vendor/tb/src/opt/mem2reg.h delete mode 100644 vendor/tb/src/opt/mem_opt.h delete mode 100644 vendor/tb/src/opt/optimizer.c delete mode 100644 vendor/tb/src/opt/passes.h delete mode 100644 vendor/tb/src/opt/peeps.h delete mode 100644 vendor/tb/src/opt/print.h delete mode 100644 vendor/tb/src/opt/print_c.h delete mode 100644 vendor/tb/src/opt/print_dumb.h delete mode 100644 vendor/tb/src/opt/properties.h delete mode 100644 vendor/tb/src/opt/scheduler.h delete mode 100644 vendor/tb/src/opt/sroa.h delete mode 100644 vendor/tb/src/reg_alloc.h delete mode 100644 vendor/tb/src/set.h delete mode 100644 vendor/tb/src/symbols.c delete mode 100644 vendor/tb/src/tb.c delete mode 100644 vendor/tb/src/tb_builder.c delete mode 100644 vendor/tb/src/tb_internal.h delete mode 100644 vendor/tb/src/tb_platform.h delete mode 100644 vendor/tb/src/wasm/wasm_target.c delete mode 100644 vendor/tb/src/x64/x64.c delete mode 100644 vendor/tb/src/x64/x64.h delete mode 100644 vendor/tb/src/x64/x64_disasm.c delete mode 100644 vendor/tb/src/x64/x64_emitter.h delete mode 100644 vendor/tb/src/x64/x64_insts.inc delete mode 100644 vendor/tb/src/x64/x64_target.c delete mode 100644 vendor/tb/tests/cg_test.c delete mode 100644 vendor/tb/unittests/tb_test_exit_status.inc delete mode 100644 vendor/tb/unittests/tb_test_int_arith.inc delete mode 100644 vendor/tb/unittests/tb_test_regressions.inc delete mode 100644 vendor/tb/unittests/tb_unittests.c delete mode 100644 vendor/tb/unittests/util.inc create mode 160000 vendor/tcc delete mode 100644 vendor/tcc/coff.h delete mode 100644 vendor/tcc/config.h delete mode 100644 vendor/tcc/dwarf.h delete mode 100644 vendor/tcc/i386-asm.c delete mode 100644 vendor/tcc/i386-asm.h delete mode 100644 vendor/tcc/i386-tok.h delete mode 100644 vendor/tcc/lib/libtcc1.c delete mode 100644 vendor/tcc/libtcc.c delete mode 100644 vendor/tcc/libtcc.h delete mode 100644 vendor/tcc/stab.def delete mode 100644 vendor/tcc/stab.h delete mode 100644 vendor/tcc/tcc.h delete mode 100644 vendor/tcc/tccasm.c delete mode 100644 vendor/tcc/tccdbg.c delete mode 100644 vendor/tcc/tccdefs_.h delete mode 100644 vendor/tcc/tccelf.c delete mode 100644 vendor/tcc/tccgen.c delete mode 100644 vendor/tcc/tcclib.h delete mode 100644 vendor/tcc/tccpp.c delete mode 100644 vendor/tcc/tccrun.c delete mode 100644 vendor/tcc/tcctok.h delete mode 100644 vendor/tcc/tcctools.c delete mode 100644 vendor/tcc/x86_64-asm.h delete mode 100644 vendor/tcc/x86_64-gen.c delete mode 100644 vendor/tcc/x86_64-link.c create mode 160000 vendor/tree-sitter delete mode 100644 vendor/trees/alloc.c delete mode 100644 vendor/trees/api.h delete mode 100644 vendor/trees/array.h delete mode 100644 vendor/trees/atomic.h delete mode 100644 vendor/trees/clock.h delete mode 100644 vendor/trees/error_costs.h delete mode 100644 vendor/trees/get_changed_ranges.c delete mode 100644 vendor/trees/get_changed_ranges.h delete mode 100644 vendor/trees/host.h delete mode 100644 vendor/trees/language.c delete mode 100644 vendor/trees/language.h delete mode 100644 vendor/trees/length.h delete mode 100644 vendor/trees/lexer.c delete mode 100644 vendor/trees/lexer.h delete mode 100644 vendor/trees/node.c delete mode 100644 vendor/trees/parser.c delete mode 100644 vendor/trees/parser.h delete mode 100644 vendor/trees/point.h delete mode 100644 vendor/trees/query.c delete mode 100644 vendor/trees/reduce_action.h delete mode 100644 vendor/trees/reusable_node.h delete mode 100644 vendor/trees/stack.c delete mode 100644 vendor/trees/stack.h delete mode 100644 vendor/trees/subtree.c delete mode 100644 vendor/trees/subtree.h delete mode 100644 vendor/trees/tree.c delete mode 100644 vendor/trees/tree.h delete mode 100644 vendor/trees/tree_cursor.c delete mode 100644 vendor/trees/tree_cursor.h delete mode 100644 vendor/trees/unicode.h delete mode 100644 vendor/trees/unicode/ICU_SHA delete mode 100644 vendor/trees/unicode/LICENSE delete mode 100644 vendor/trees/unicode/README.md delete mode 100644 vendor/trees/unicode/ptypes.h delete mode 100644 vendor/trees/unicode/umachine.h delete mode 100644 vendor/trees/unicode/urename.h delete mode 100644 vendor/trees/unicode/utf.h delete mode 100644 vendor/trees/unicode/utf16.h delete mode 100644 vendor/trees/unicode/utf8.h create mode 160000 vendor/xxhash delete mode 100644 vendor/xxhash/xxhash.c delete mode 100644 vendor/xxhash/xxhash.h create mode 100644 vm/lua/.gitignore delete mode 100644 vm/lua/parser.c rename vm/lua/{ => parser}/grammar.js (100%) create mode 100644 vm/lua/parser/parser.c rename vm/lua/{ => parser}/scan.c (99%) rename {vendor/trees => vm/lua/parser/tree_sitter}/alloc.h (70%) create mode 100644 vm/lua/parser/tree_sitter/array.h rename vm/lua/{ => parser/tree_sitter}/parser.h (94%) diff --git a/.gitignore b/.gitignore index c9267b34..4af41ebd 100644 --- a/.gitignore +++ b/.gitignore @@ -25,4 +25,5 @@ out.* *.tmp.c .vscode .vscode -.mypy_cache \ No newline at end of file +.mypy_cache +*.core diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 00000000..7abd21ae --- /dev/null +++ b/.gitmodules @@ -0,0 +1,16 @@ +[submodule "vendor/tcc"] + path = vendor/tcc + url = https://github.com/fastvm/minivm-tinycc +[submodule "vendor/cuik"] + path = vendor/cuik + url = https://github.com/fastvm/minivm-cuik + branch = tb2c +[submodule "vendor/xxhash"] + path = vendor/xxhash + url = https://github.com/fastvm/minivm-xxhash +[submodule "vendor/tree-sitter"] + path = vendor/tree-sitter + url = https://github.com/fastvm/minivm-tree-sitter +[submodule "vendor/isocline"] + path = vendor/isocline + url = https://github.com/fastvm/minivm-isocline diff --git a/base.mak b/base.mak new file mode 100644 index 00000000..b7d43222 --- /dev/null +++ b/base.mak @@ -0,0 +1,4 @@ + +CFLAGS := -DVM_USE_TCC $(CFLAGS) + +include makefile diff --git a/makefile b/makefile index e1a81317..80a56975 100644 --- a/makefile +++ b/makefile @@ -3,12 +3,21 @@ OPT ?= -Os -flto EXE ?= -BUILD_DIR ?= build +CURRENT_DIR != pwd + +BUILD_DIR ?= $(CURRENT_DIR)/build OBJ_DIR ?= $(BUILD_DIR)/obj TMP_DIR ?= $(BUILD_DIR)/tmp BIN_DIR ?= $(BUILD_DIR)/bin RES_DIR ?= $(BUILD_DIR)/res +VENDOR_DIR ?= $(CURRENT_DIR)/vendor +VENDOR_CUIK_DIR ?= $(VENDOR_DIR)/cuik +VENDOR_ISOCLINE_DIR ?= $(VENDOR_DIR)/isocline +VENDOR_TCC_DIR ?= $(VENDOR_DIR)/tcc +VENDOR_TREE_SITTER_DIR ?= $(VENDOR_DIR)/tree-sitter +VENDOR_XXHASH_DIR ?= $(VENDOR_DIR)/xxhash + UNAME_S != uname -s UNAME_O != uname -o @@ -18,38 +27,35 @@ PROG_OBJS = $(PROG_SRCS:%.c=$(OBJ_DIR)/%.o) # GC_SRCS = vendor/bdwgc/alloc.c vendor/bdwgc/allchblk.c vendor/bdwgc/blacklst.c vendor/bdwgc/dbg_mlc.c vendor/bdwgc/dyn_load.c vendor/bdwgc/finalize.c vendor/bdwgc/headers.c vendor/bdwgc/malloc.c vendor/bdwgc/mallocx.c vendor/bdwgc/mark.c vendor/bdwgc/mach_dep.c vendor/bdwgc/mark_rts.c vendor/bdwgc/misc.c vendor/bdwgc/new_hblk.c vendor/bdwgc/obj_map.c vendor/bdwgc/os_dep.c vendor/bdwgc/ptr_chck.c vendor/bdwgc/reclaim.c GC_OBJS = $(GC_SRCS:%.c=$(OBJ_DIR)/%.o) -TREES_SRCS = vendor/trees/alloc.c vendor/trees/get_changed_ranges.c vendor/trees/language.c vendor/trees/lexer.c vendor/trees/node.c vendor/trees/parser.c vendor/trees/query.c vendor/trees/stack.c vendor/trees/subtree.c vendor/trees/tree_cursor.c vendor/trees/tree.c +TREES_SRCS = vendor/tree-sitter/lib/src/alloc.c vendor/tree-sitter/lib/src/get_changed_ranges.c vendor/tree-sitter/lib/src/language.c vendor/tree-sitter/lib/src/lexer.c vendor/tree-sitter/lib/src/node.c vendor/tree-sitter/lib/src/parser.c vendor/tree-sitter/lib/src/query.c vendor/tree-sitter/lib/src/stack.c vendor/tree-sitter/lib/src/subtree.c vendor/tree-sitter/lib/src/tree_cursor.c vendor/tree-sitter/lib/src/tree.c vendor/tree-sitter/lib/src/wasm_store.c STD_SRCS = vm/std/io.c vm/std/std.c -ISOCLINE_SRCS = vendor/isocline/isocline.c +ISOCLINE_SRCS = vendor/isocline/src/isocline.c XXH_SRCS = vendor/xxhash/xxhash.c -VM_SRCS = vm/ir/ir.c vm/lib.c vm/ir/type.c vm/ast/build.c vm/ast/ast.c vm/ast/comp.c vm/ast/print.c vm/obj.c vm/backend/tb.c vm/backend/exec.c vm/ir/check.c vm/ir/rblock.c vm/lua/parser.c vm/lua/scan.c vm/lua/ast.c vm/lua/repl.c $(ISOCLINE_SRCS) $(XXH_SRCS) - +VM_SRCS = vm/ir/ir.c vm/lib.c vm/ir/type.c vm/ast/build.c vm/ast/ast.c vm/ast/comp.c vm/ast/print.c vm/obj.c vm/backend/tb.c vm/backend/exec.c vm/ir/check.c vm/ir/rblock.c vm/lua/parser/parser.c vm/lua/parser/scan.c vm/lua/ast.c vm/lua/repl.c $(ISOCLINE_SRCS) $(XXH_SRCS) $(TREES_SRCS) -ALL_SRCS = $(VM_SRCS) $(STD_SRCS) $(EXTRA_SRCS) $(TREES_SRCS) +ALL_SRCS = $(VM_SRCS) $(STD_SRCS) $(EXTRA_SRCS) ALL_OBJS = $(ALL_SRCS:%.c=$(OBJ_DIR)/%.o) TCC_SRCS ?= vendor/tcc/libtcc.c vendor/tcc/lib/libtcc1.c TCC_OBJS = $(TCC_SRCS:%.c=$(OBJ_DIR)/%.o) -TB_SRCS = vendor/common/common.c vendor/common/perf.c vendor/tb/src/libtb.c vendor/tb/src/x64/x64_target.c +TB_SRCS_BASE = vendor/cuik/common/common.c vendor/cuik/common/perf.c vendor/cuik/tb/src/libtb.c vendor/cuik/tb/src/x64/x64_target.c +TB_SRCS_FreeBSD = vendor/cuik/c11threads/threads_posix.c +TB_SRCS = $(TB_SRCS_BASE) $(TB_SRCS_$(UNAME_S)) TB_OBJS = $(TB_SRCS:%.c=$(OBJ_DIR)/%.o) BASE_OBJS = $(ALL_OBJS) $(GC_OBJS) $(TB_OBJS) $(TCC_OBJS) -CFLAGS += $(FLAGS) +CFLAGS += -I vendor/tree-sitter/lib/include -I vendor/tree-sitter/lib/src $(FLAGS) LDFLAGS += $(FLAGS) -RUNNER ?= $(BIN_DIR)/minivm - -OBJS_FreeBSD = - -OBJS = $(BASE_OBJS) $(OBJS_$(UNAME_S)) +OBJS = $(BASE_OBJS) LDFLAGS_S_Darwin = -w -Wl,-pagezero_size,0x4000 LDFLAGS_S_Linux = -lm -ldl LDFLAGS_O_Cygwin = -LDFLAGS_S_FreeBSD = -lm -ldl -lstdthreads +LDFLAGS_S_FreeBSD = -lm -ldl -lpthread LDFLAGS := $(LDFLAGS_S_$(UNAME_S)) $(LDFLAGS_O_$(UNAME_O)) $(LDFLAGS) @@ -61,11 +67,28 @@ default: all all: bins +# tree sitter + +VM_LUA_GRAMMAR_DIR := $(TMP_DIR)/grammar + +pre: $(VM_LUA_GRAMMAR_DIR) + +$(VM_LUA_GRAMMAR_DIR): vm/lua/parser/grammar.js + mkdir -p $(VM_LUA_GRAMMAR_DIR) + cp vm/lua/parser/grammar.js $(VM_LUA_GRAMMAR_DIR) + cd $(VM_LUA_GRAMMAR_DIR) && cargo run --manifest-path $(VENDOR_TREE_SITTER_DIR)/Cargo.toml -- generate + +vm/lua/parser/parser.c: $(VM_LUA_GRAMMAR_DIR) vm/lua/parser/tree_sitter + cp $(VM_LUA_GRAMMAR_DIR)/src/parser.c $(@) + +vm/lua/parser/tree_sitter: $(VM_LUA_GRAMMAR_DIR) + cp -r $(VM_LUA_GRAMMAR_DIR)/src/tree_sitter $(@) + # windows clang-windows: .dummy rm -rf build - $(MAKE) -Bj$(J) CC=clang EXE=.exe OPT="$(OPT)" CFLAGS="-Icuik/c11threads $(CFLAGS)" LDFLAGS="$(LDFLAGS)" EXTRA_SRCS="vendor/c11threads/threads_msvc.c" + $(MAKE) -Bj$(J) CC=clang EXE=.exe OPT="$(OPT)" CFLAGS="-Icuik/c11threads $(CFLAGS)" LDFLAGS="$(LDFLAGS)" EXTRA_SRCS="vendor/cuik/c11threads/threads_msvc.c" gcc-windows: .dummy rm -rf build @@ -83,7 +106,7 @@ minivm$(EXE) $(BIN_DIR)/minivm$(EXE): $(OBJ_DIR)/main/minivm.o $(OBJS) $(TB_OBJS): $(@:$(OBJ_DIR)/%.o=%.c) @mkdir -p $$(dirname $(@)) - $(CC) -Wno-unused-value -c $(OPT) $(@:$(OBJ_DIR)/%.o=%.c) -o $(@) $(CFLAGS) -I vendor/tb/include -I vendor/common -DCUIK_USE_TB -DLOG_SUPPRESS -DTB_HAS_X64 + $(CC) -Wno-unused-value -c $(OPT) $(@:$(OBJ_DIR)/%.o=%.c) -o $(@) $(CFLAGS) -I vendor/cuik/tb/include -I vendor/cuik/common -DCUIK_USE_TB -DLOG_SUPPRESS -DTB_HAS_X64 $(PROG_OBJS) $(ALL_OBJS) $(GC_OBJS) $(TCC_OBJS): $(@:$(OBJ_DIR)/%.o=%.c) @mkdir -p $$(dirname $(@)) diff --git a/pre.sh b/pre.sh deleted file mode 100755 index 572141c0..00000000 --- a/pre.sh +++ /dev/null @@ -1,19 +0,0 @@ -#!/usr/bin/env bash - -set -e - -cd $(dirname $0) - -cd vm/lua - -mkdir -p trees -trap "cd $(pwd) && rm -r trees" EXIT - -cp grammar.js trees - -cd trees - -tree-sitter generate - -cp src/parser.c .. -cp src/tree_sitter/parser.h .. diff --git a/tcc.mak b/tcc.mak new file mode 100644 index 00000000..b8d6707b --- /dev/null +++ b/tcc.mak @@ -0,0 +1,7 @@ + +CFLAGS := -DCONFIG_TCC_PREDEFS -DVM_USE_TCC $(CFLAGS) + +IGNORE != cd vendor/tcc && ./configure +IGNORE != echo '""' > vendor/tcc/tccdefs_.h + +include makefile diff --git a/vendor/bdwgc/allchblk.c b/vendor/bdwgc/allchblk.c deleted file mode 100644 index 8535f7e1..00000000 --- a/vendor/bdwgc/allchblk.c +++ /dev/null @@ -1,990 +0,0 @@ -/* - * Copyright 1988, 1989 Hans-J. Boehm, Alan J. Demers - * Copyright (c) 1991-1994 by Xerox Corporation. All rights reserved. - * Copyright (c) 1998-1999 by Silicon Graphics. All rights reserved. - * Copyright (c) 1999 by Hewlett-Packard Company. All rights reserved. - * Copyright (c) 2008-2022 Ivan Maidanski - * - * THIS MATERIAL IS PROVIDED AS IS, WITH ABSOLUTELY NO WARRANTY EXPRESSED - * OR IMPLIED. ANY USE IS AT YOUR OWN RISK. - * - * Permission is hereby granted to use or copy this program - * for any purpose, provided the above notices are retained on all copies. - * Permission to modify the code and to distribute modified code is granted, - * provided the above notices are retained, and a notice that the code was - * modified is included with the above copyright notice. - */ - -#ifdef _WIN32 -#define WIN32_LEAN_AND_MEAN -#include -#endif -#include "private/gc_priv.h" - -#ifdef GC_USE_ENTIRE_HEAP -int GC_use_entire_heap = TRUE; -#else -int GC_use_entire_heap = FALSE; -#endif - -/* - * Free heap blocks are kept on one of several free lists, - * depending on the size of the block. Each free list is doubly linked. - * Adjacent free blocks are coalesced. - */ - -#define MAX_BLACK_LIST_ALLOC (2 * HBLKSIZE) -/* largest block we will allocate starting on a black */ -/* listed block. Must be >= HBLKSIZE. */ - -#define UNIQUE_THRESHOLD 32 -/* Sizes up to this many HBLKs each have their own free list */ -#define HUGE_THRESHOLD 256 -/* Sizes of at least this many heap blocks are mapped to a */ -/* single free list. */ -#define FL_COMPRESSION 8 -/* In between sizes map this many distinct sizes to a single */ -/* bin. */ - -#define N_HBLK_FLS ((HUGE_THRESHOLD - UNIQUE_THRESHOLD) / FL_COMPRESSION + UNIQUE_THRESHOLD) - -#ifndef GC_GCJ_SUPPORT -STATIC -#endif -struct hblk *GC_hblkfreelist[N_HBLK_FLS + 1] = {0}; -/* List of completely empty heap blocks */ -/* Linked through hb_next field of */ -/* header structure associated with */ -/* block. Remains externally visible */ -/* as used by GNU GCJ currently. */ - -GC_API void GC_CALL GC_iterate_free_hblks(GC_walk_free_blk_fn fn, - GC_word client_data) { - int i; - - for (i = 0; i <= N_HBLK_FLS; ++i) { - struct hblk *h; - - for (h = GC_hblkfreelist[i]; h != NULL; h = HDR(h)->hb_next) { - (*fn)(h, i, client_data); - } - } -} - -#ifndef GC_GCJ_SUPPORT -STATIC -#endif -word GC_free_bytes[N_HBLK_FLS + 1] = {0}; -/* Number of free bytes on each list. Remains visible to GCJ. */ - -/* Return the largest n such that the number of free bytes on lists */ -/* n .. N_HBLK_FLS is greater or equal to GC_max_large_allocd_bytes */ -/* minus GC_large_allocd_bytes. If there is no such n, return 0. */ -GC_INLINE int GC_enough_large_bytes_left(void) { - int n; - word bytes = GC_large_allocd_bytes; - - GC_ASSERT(GC_max_large_allocd_bytes <= GC_heapsize); - for (n = N_HBLK_FLS; n >= 0; --n) { - bytes += GC_free_bytes[n]; - if (bytes >= GC_max_large_allocd_bytes) return n; - } - return 0; -} - -/* Map a number of blocks to the appropriate large block free list index. */ -STATIC int GC_hblk_fl_from_blocks(size_t blocks_needed) { - if (blocks_needed <= UNIQUE_THRESHOLD) return (int)blocks_needed; - if (blocks_needed >= HUGE_THRESHOLD) return N_HBLK_FLS; - return (int)(blocks_needed - UNIQUE_THRESHOLD) / FL_COMPRESSION + UNIQUE_THRESHOLD; -} - -#define PHDR(hhdr) HDR((hhdr)->hb_prev) -#define NHDR(hhdr) HDR((hhdr)->hb_next) - -#ifdef USE_MUNMAP -#define IS_MAPPED(hhdr) (((hhdr)->hb_flags & WAS_UNMAPPED) == 0) -#else -#define IS_MAPPED(hhdr) TRUE -#endif /* !USE_MUNMAP */ - -#if !defined(NO_DEBUGGING) || defined(GC_ASSERTIONS) -static void GC_CALLBACK add_hb_sz(struct hblk *h, int i, GC_word client_data) { - UNUSED_ARG(i); - *(word *)client_data += HDR(h)->hb_sz; -} - -/* Should return the same value as GC_large_free_bytes. */ -GC_INNER word GC_compute_large_free_bytes(void) { - word total_free = 0; - - GC_iterate_free_hblks(add_hb_sz, (word)&total_free); - return total_free; -} -#endif /* !NO_DEBUGGING || GC_ASSERTIONS */ - -#if !defined(NO_DEBUGGING) -static void GC_CALLBACK print_hblkfreelist_item(struct hblk *h, int i, - GC_word prev_index_ptr) { - hdr *hhdr = HDR(h); - - if (i != *(int *)prev_index_ptr) { - GC_printf("Free list %d (total size %lu):\n", - i, (unsigned long)GC_free_bytes[i]); - *(int *)prev_index_ptr = i; - } - - GC_printf("\t%p size %lu %s black listed\n", - (void *)h, (unsigned long)(hhdr->hb_sz), - GC_is_black_listed(h, HBLKSIZE) != NULL ? "start" - : GC_is_black_listed(h, hhdr->hb_sz) != NULL ? "partially" - : "not"); -} - -void GC_print_hblkfreelist(void) { - word total; - int prev_index = -1; - - GC_iterate_free_hblks(print_hblkfreelist_item, (word)&prev_index); - GC_printf("GC_large_free_bytes: %lu\n", - (unsigned long)GC_large_free_bytes); - total = GC_compute_large_free_bytes(); - if (total != GC_large_free_bytes) - GC_err_printf("GC_large_free_bytes INCONSISTENT!! Should be: %lu\n", - (unsigned long)total); -} - -/* Return the free list index on which the block described by the header */ -/* appears, or -1 if it appears nowhere. */ -static int free_list_index_of(const hdr *wanted) { - int i; - - for (i = 0; i <= N_HBLK_FLS; ++i) { - struct hblk *h; - hdr *hhdr; - - for (h = GC_hblkfreelist[i]; h != 0; h = hhdr->hb_next) { - hhdr = HDR(h); - if (hhdr == wanted) return i; - } - } - return -1; -} - -GC_API void GC_CALL GC_dump_regions(void) { - unsigned i; - - for (i = 0; i < GC_n_heap_sects; ++i) { - ptr_t start = GC_heap_sects[i].hs_start; - size_t bytes = GC_heap_sects[i].hs_bytes; - ptr_t end = start + bytes; - ptr_t p; - - /* Merge in contiguous sections. */ - while (i + 1 < GC_n_heap_sects && GC_heap_sects[i + 1].hs_start == end) { - ++i; - end = GC_heap_sects[i].hs_start + GC_heap_sects[i].hs_bytes; - } - GC_printf("***Section from %p to %p\n", (void *)start, (void *)end); - for (p = start; (word)p < (word)end;) { - hdr *hhdr = HDR(p); - - if (IS_FORWARDING_ADDR_OR_NIL(hhdr)) { - GC_printf("\t%p Missing header!!(%p)\n", - (void *)p, (void *)hhdr); - p += HBLKSIZE; - continue; - } - if (HBLK_IS_FREE(hhdr)) { - int correct_index = GC_hblk_fl_from_blocks( - (size_t)divHBLKSZ(hhdr->hb_sz)); - int actual_index; - - GC_printf("\t%p\tfree block of size 0x%lx bytes%s\n", - (void *)p, (unsigned long)(hhdr->hb_sz), - IS_MAPPED(hhdr) ? "" : " (unmapped)"); - actual_index = free_list_index_of(hhdr); - if (-1 == actual_index) { - GC_printf("\t\tBlock not on free list %d!!\n", - correct_index); - } else if (correct_index != actual_index) { - GC_printf("\t\tBlock on list %d, should be on %d!!\n", - actual_index, correct_index); - } - p += hhdr->hb_sz; - } else { - GC_printf("\t%p\tused for blocks of size 0x%lx bytes\n", - (void *)p, (unsigned long)(hhdr->hb_sz)); - p += HBLKSIZE * OBJ_SZ_TO_BLOCKS(hhdr->hb_sz); - } - } - } -} - -#endif /* NO_DEBUGGING */ - -/* Initialize hdr for a block containing the indicated size and */ -/* kind of objects. Return FALSE on failure. */ -static GC_bool setup_header(hdr *hhdr, struct hblk *block, size_t byte_sz, - int kind, unsigned flags) { - word descr; - - GC_ASSERT(I_HOLD_LOCK()); -#ifdef MARK_BIT_PER_GRANULE - if (byte_sz > MAXOBJBYTES) - flags |= LARGE_BLOCK; -#endif -#ifdef ENABLE_DISCLAIM - if (GC_obj_kinds[kind].ok_disclaim_proc) - flags |= HAS_DISCLAIM; - if (GC_obj_kinds[kind].ok_mark_unconditionally) - flags |= MARK_UNCONDITIONALLY; -#endif - - /* Set size, kind and mark proc fields */ - hhdr->hb_sz = byte_sz; - hhdr->hb_obj_kind = (unsigned char)kind; - hhdr->hb_flags = (unsigned char)flags; - hhdr->hb_block = block; - descr = GC_obj_kinds[kind].ok_descriptor; - if (GC_obj_kinds[kind].ok_relocate_descr) descr += byte_sz; - hhdr->hb_descr = descr; - -#ifdef MARK_BIT_PER_OBJ - /* Set hb_inv_sz as portably as possible. */ - /* We set it to the smallest value such that sz * inv_sz >= 2**32 */ - /* This may be more precision than necessary. */ - if (byte_sz > MAXOBJBYTES) { - hhdr->hb_inv_sz = LARGE_INV_SZ; - } else { - word inv_sz; - -#if CPP_WORDSZ == 64 - inv_sz = ((word)1 << 32) / byte_sz; - if (((inv_sz * byte_sz) >> 32) == 0) ++inv_sz; -#else /* 32 bit words */ - GC_ASSERT(byte_sz >= 4); - inv_sz = ((unsigned)1 << 31) / byte_sz; - inv_sz *= 2; - while (inv_sz * byte_sz > byte_sz) ++inv_sz; -#endif -#ifdef INV_SZ_COMPUTATION_CHECK - GC_ASSERT(((1ULL << 32) + byte_sz - 1) / byte_sz == inv_sz); -#endif - hhdr->hb_inv_sz = inv_sz; - } -#endif -#ifdef MARK_BIT_PER_GRANULE - { - size_t granules = BYTES_TO_GRANULES(byte_sz); - - if (EXPECT(!GC_add_map_entry(granules), FALSE)) { - /* Make it look like a valid block. */ - hhdr->hb_sz = HBLKSIZE; - hhdr->hb_descr = 0; - hhdr->hb_flags |= LARGE_BLOCK; - hhdr->hb_map = 0; - return FALSE; - } - hhdr->hb_map = GC_obj_map[(hhdr->hb_flags & LARGE_BLOCK) != 0 ? 0 : granules]; - } -#endif /* MARK_BIT_PER_GRANULE */ - - /* Clear mark bits */ - GC_clear_hdr_marks(hhdr); - - hhdr->hb_last_reclaimed = (unsigned short)GC_gc_no; - return TRUE; -} - -/* Remove hhdr from the free list (it is assumed to specified by index). */ -STATIC void GC_remove_from_fl_at(hdr *hhdr, int index) { - GC_ASSERT(modHBLKSZ(hhdr->hb_sz) == 0); - if (hhdr->hb_prev == 0) { - GC_ASSERT(HDR(GC_hblkfreelist[index]) == hhdr); - GC_hblkfreelist[index] = hhdr->hb_next; - } else { - hdr *phdr; - GET_HDR(hhdr->hb_prev, phdr); - phdr->hb_next = hhdr->hb_next; - } - /* We always need index to maintain free counts. */ - GC_ASSERT(GC_free_bytes[index] >= hhdr->hb_sz); - GC_free_bytes[index] -= hhdr->hb_sz; - if (0 != hhdr->hb_next) { - hdr *nhdr; - GC_ASSERT(!IS_FORWARDING_ADDR_OR_NIL(NHDR(hhdr))); - GET_HDR(hhdr->hb_next, nhdr); - nhdr->hb_prev = hhdr->hb_prev; - } -} - -/* Remove hhdr from the appropriate free list (we assume it is on the */ -/* size-appropriate free list). */ -GC_INLINE void GC_remove_from_fl(hdr *hhdr) { - GC_remove_from_fl_at(hhdr, GC_hblk_fl_from_blocks( - (size_t)divHBLKSZ(hhdr->hb_sz))); -} - -/* Return a pointer to the block ending just before h, if any. */ -static struct hblk *get_block_ending_at(struct hblk *h) { - struct hblk *p = h - 1; - hdr *phdr; - - GET_HDR(p, phdr); - while (0 != phdr && IS_FORWARDING_ADDR_OR_NIL(phdr)) { - p = FORWARDED_ADDR(p, phdr); - phdr = HDR(p); - } - if (0 != phdr) { - return p; - } - p = GC_prev_block(h - 1); - if (p) { - phdr = HDR(p); - if ((ptr_t)p + phdr->hb_sz == (ptr_t)h) { - return p; - } - } - return NULL; -} - -/* Return a pointer to the free block ending just before h, if any. */ -STATIC struct hblk *GC_free_block_ending_at(struct hblk *h) { - struct hblk *p = get_block_ending_at(h); - - if (p /* != NULL */) { /* CPPCHECK */ - hdr *phdr = HDR(p); - - if (HBLK_IS_FREE(phdr)) { - return p; - } - } - return 0; -} - -/* Add hhdr to the appropriate free list. */ -/* We maintain individual free lists sorted by address. */ -STATIC void GC_add_to_fl(struct hblk *h, hdr *hhdr) { - int index = GC_hblk_fl_from_blocks((size_t)divHBLKSZ(hhdr->hb_sz)); - struct hblk *second = GC_hblkfreelist[index]; - -#if defined(GC_ASSERTIONS) && !defined(USE_MUNMAP) - { - struct hblk *next = (struct hblk *)((word)h + hhdr->hb_sz); - hdr *nexthdr = HDR(next); - struct hblk *prev = GC_free_block_ending_at(h); - hdr *prevhdr = HDR(prev); - - GC_ASSERT(nexthdr == 0 || !HBLK_IS_FREE(nexthdr) || (GC_heapsize & SIGNB) != 0); - /* In the last case, blocks may be too large to merge. */ - GC_ASSERT(NULL == prev || !HBLK_IS_FREE(prevhdr) || (GC_heapsize & SIGNB) != 0); - } -#endif - GC_ASSERT(modHBLKSZ(hhdr->hb_sz) == 0); - GC_hblkfreelist[index] = h; - GC_free_bytes[index] += hhdr->hb_sz; - GC_ASSERT(GC_free_bytes[index] <= GC_large_free_bytes); - hhdr->hb_next = second; - hhdr->hb_prev = 0; - if (second /* != NULL */) { /* CPPCHECK */ - hdr *second_hdr; - - GET_HDR(second, second_hdr); - second_hdr->hb_prev = h; - } - hhdr->hb_flags |= FREE_BLK; -} - -#ifdef USE_MUNMAP - -#ifdef COUNT_UNMAPPED_REGIONS -/* GC_unmap_old will avoid creating more than this many unmapped regions, */ -/* but an unmapped region may be split again so exceeding the limit. */ - -/* Return the change in number of unmapped regions if the block h swaps */ -/* from its current state of mapped/unmapped to the opposite state. */ -static int calc_num_unmapped_regions_delta(struct hblk *h, hdr *hhdr) { - struct hblk *prev = get_block_ending_at(h); - struct hblk *next; - GC_bool prev_unmapped = FALSE; - GC_bool next_unmapped = FALSE; - - next = GC_next_block((struct hblk *)((ptr_t)h + hhdr->hb_sz), TRUE); - /* Ensure next is contiguous with h. */ - if ((ptr_t)next != GC_unmap_end((ptr_t)h, (size_t)hhdr->hb_sz)) { - next = NULL; - } - if (prev != NULL) { - hdr *prevhdr = HDR(prev); - prev_unmapped = !IS_MAPPED(prevhdr); - } - if (next != NULL) { - hdr *nexthdr = HDR(next); - next_unmapped = !IS_MAPPED(nexthdr); - } - - if (prev_unmapped && next_unmapped) { - /* If h unmapped, merge two unmapped regions into one. */ - /* If h remapped, split one unmapped region into two. */ - return IS_MAPPED(hhdr) ? -1 : 1; - } - if (!prev_unmapped && !next_unmapped) { - /* If h unmapped, create an isolated unmapped region. */ - /* If h remapped, remove it. */ - return IS_MAPPED(hhdr) ? 1 : -1; - } - /* If h unmapped, merge it with previous or next unmapped region. */ - /* If h remapped, reduce either previous or next unmapped region. */ - /* In either way, no change to the number of unmapped regions. */ - return 0; -} -#endif /* COUNT_UNMAPPED_REGIONS */ - -/* Update GC_num_unmapped_regions assuming the block h changes */ -/* from its current state of mapped/unmapped to the opposite state. */ -GC_INLINE void GC_adjust_num_unmapped(struct hblk *h, hdr *hhdr) { -#ifdef COUNT_UNMAPPED_REGIONS - GC_num_unmapped_regions += calc_num_unmapped_regions_delta(h, hhdr); -#else - UNUSED_ARG(h); - UNUSED_ARG(hhdr); -#endif -} - -/* Unmap blocks that haven't been recently touched. This is the only */ -/* way blocks are ever unmapped. */ -GC_INNER void GC_unmap_old(unsigned threshold) { - int i; - -#ifdef COUNT_UNMAPPED_REGIONS - /* Skip unmapping if we have already exceeded the soft limit. */ - /* This forgoes any opportunities to merge unmapped regions though. */ - if (GC_num_unmapped_regions >= GC_UNMAPPED_REGIONS_SOFT_LIMIT) - return; -#endif - - for (i = 0; i <= N_HBLK_FLS; ++i) { - struct hblk *h; - hdr *hhdr; - - for (h = GC_hblkfreelist[i]; 0 != h; h = hhdr->hb_next) { - hhdr = HDR(h); - if (!IS_MAPPED(hhdr)) continue; - - /* Check that the interval is not smaller than the threshold. */ - /* The truncated counter value wrapping is handled correctly. */ - if ((unsigned short)(GC_gc_no - hhdr->hb_last_reclaimed) >= (unsigned short)threshold) { -#ifdef COUNT_UNMAPPED_REGIONS - /* Continue with unmapping the block only if it will not */ - /* create too many unmapped regions, or if unmapping */ - /* reduces the number of regions. */ - int delta = calc_num_unmapped_regions_delta(h, hhdr); - signed_word regions = GC_num_unmapped_regions + delta; - - if (delta >= 0 && regions >= GC_UNMAPPED_REGIONS_SOFT_LIMIT) { - GC_COND_LOG_PRINTF("Unmapped regions limit reached!\n"); - return; - } - GC_num_unmapped_regions = regions; -#endif - GC_unmap((ptr_t)h, (size_t)(hhdr->hb_sz)); - hhdr->hb_flags |= WAS_UNMAPPED; - } - } - } -} - -/* Merge all unmapped blocks that are adjacent to other free */ -/* blocks. This may involve remapping, since all blocks are either */ -/* fully mapped or fully unmapped. */ -GC_INNER void GC_merge_unmapped(void) { - int i; - - for (i = 0; i <= N_HBLK_FLS; ++i) { - struct hblk *h = GC_hblkfreelist[i]; - - while (h != 0) { - struct hblk *next; - hdr *hhdr, *nexthdr; - word size, nextsize; - - GET_HDR(h, hhdr); - size = hhdr->hb_sz; - next = (struct hblk *)((word)h + size); - GET_HDR(next, nexthdr); - /* Coalesce with successor, if possible */ - if (nexthdr != NULL && HBLK_IS_FREE(nexthdr) && !((size + (nextsize = nexthdr->hb_sz)) & SIGNB) - /* no overflow */) { - /* Note that we usually try to avoid adjacent free blocks */ - /* that are either both mapped or both unmapped. But that */ - /* isn't guaranteed to hold since we remap blocks when we */ - /* split them, and don't merge at that point. It may also */ - /* not hold if the merged block would be too big. */ - if (IS_MAPPED(hhdr) && !IS_MAPPED(nexthdr)) { - /* make both consistent, so that we can merge */ - if (size > nextsize) { - GC_adjust_num_unmapped(next, nexthdr); - GC_remap((ptr_t)next, nextsize); - } else { - GC_adjust_num_unmapped(h, hhdr); - GC_unmap((ptr_t)h, size); - GC_unmap_gap((ptr_t)h, size, (ptr_t)next, nextsize); - hhdr->hb_flags |= WAS_UNMAPPED; - } - } else if (IS_MAPPED(nexthdr) && !IS_MAPPED(hhdr)) { - if (size > nextsize) { - GC_adjust_num_unmapped(next, nexthdr); - GC_unmap((ptr_t)next, nextsize); - GC_unmap_gap((ptr_t)h, size, (ptr_t)next, nextsize); - } else { - GC_adjust_num_unmapped(h, hhdr); - GC_remap((ptr_t)h, size); - hhdr->hb_flags &= ~WAS_UNMAPPED; - hhdr->hb_last_reclaimed = nexthdr->hb_last_reclaimed; - } - } else if (!IS_MAPPED(hhdr) && !IS_MAPPED(nexthdr)) { - /* Unmap any gap in the middle */ - GC_unmap_gap((ptr_t)h, size, (ptr_t)next, nextsize); - } - /* If they are both unmapped, we merge, but leave unmapped. */ - GC_remove_from_fl_at(hhdr, i); - GC_remove_from_fl(nexthdr); - hhdr->hb_sz += nexthdr->hb_sz; - GC_remove_header(next); - GC_add_to_fl(h, hhdr); - /* Start over at beginning of list */ - h = GC_hblkfreelist[i]; - } else /* not mergeable with successor */ { - h = hhdr->hb_next; - } - } /* while (h != 0) ... */ - } /* for ... */ -} - -#endif /* USE_MUNMAP */ - -/* - * Return a pointer to a block starting at h of length bytes. - * Memory for the block is mapped. - * Remove the block from its free list, and return the remainder (if any) - * to its appropriate free list. - * May fail by returning 0. - * The header for the returned block must be set up by the caller. - * If the return value is not 0, then hhdr is the header for it. - */ -STATIC struct hblk *GC_get_first_part(struct hblk *h, hdr *hhdr, - size_t bytes, int index) { - size_t total_size; - struct hblk *rest; - hdr *rest_hdr; - - GC_ASSERT(I_HOLD_LOCK()); - GC_ASSERT(modHBLKSZ(bytes) == 0); - total_size = (size_t)(hhdr->hb_sz); - GC_ASSERT(modHBLKSZ(total_size) == 0); - GC_remove_from_fl_at(hhdr, index); - if (total_size == bytes) return h; - - rest = (struct hblk *)((word)h + bytes); - rest_hdr = GC_install_header(rest); - if (EXPECT(NULL == rest_hdr, FALSE)) { - /* FIXME: This is likely to be very bad news ... */ - WARN("Header allocation failed: dropping block\n", 0); - return NULL; - } - rest_hdr->hb_sz = total_size - bytes; - rest_hdr->hb_flags = 0; -#ifdef GC_ASSERTIONS - /* Mark h not free, to avoid assertion about adjacent free blocks. */ - hhdr->hb_flags &= ~FREE_BLK; -#endif - GC_add_to_fl(rest, rest_hdr); - return h; -} - -/* - * H is a free block. N points at an address inside it. - * A new header for n has already been set up. Fix up h's header - * to reflect the fact that it is being split, move it to the - * appropriate free list. - * N replaces h in the original free list. - * - * Nhdr is not completely filled in, since it is about to allocated. - * It may in fact end up on the wrong free list for its size. - * That's not a disaster, since n is about to be allocated - * by our caller. - * (Hence adding it to a free list is silly. But this path is hopefully - * rare enough that it doesn't matter. The code is cleaner this way.) - */ -STATIC void GC_split_block(struct hblk *h, hdr *hhdr, struct hblk *n, - hdr *nhdr, int index /* of free list */) { - word total_size = hhdr->hb_sz; - word h_size = (word)n - (word)h; - struct hblk *prev = hhdr->hb_prev; - struct hblk *next = hhdr->hb_next; - - /* Replace h with n on its freelist */ - nhdr->hb_prev = prev; - nhdr->hb_next = next; - nhdr->hb_sz = total_size - h_size; - nhdr->hb_flags = 0; - if (prev /* != NULL */) { /* CPPCHECK */ - HDR(prev)->hb_next = n; - } else { - GC_hblkfreelist[index] = n; - } - if (next /* != NULL */) { - HDR(next)->hb_prev = n; - } - GC_ASSERT(GC_free_bytes[index] > h_size); - GC_free_bytes[index] -= h_size; -#ifdef USE_MUNMAP - hhdr->hb_last_reclaimed = (unsigned short)GC_gc_no; -#endif - hhdr->hb_sz = h_size; - GC_add_to_fl(h, hhdr); - nhdr->hb_flags |= FREE_BLK; -} - -STATIC struct hblk *GC_allochblk_nth(size_t sz /* bytes */, int kind, - unsigned flags, int n, int may_split, - size_t align_m1); - -#ifdef USE_MUNMAP -#define AVOID_SPLIT_REMAPPED 2 -#endif - -GC_INNER struct hblk *GC_allochblk(size_t sz, int kind, - unsigned flags /* IGNORE_OFF_PAGE or 0 */, - size_t align_m1) { - size_t blocks; - int start_list; - struct hblk *result; - int may_split; - int split_limit; /* highest index of free list whose blocks we split */ - - GC_ASSERT(I_HOLD_LOCK()); - GC_ASSERT((sz & (GRANULE_BYTES - 1)) == 0); - blocks = OBJ_SZ_TO_BLOCKS_CHECKED(sz); - if (EXPECT(SIZET_SAT_ADD(blocks * HBLKSIZE, align_m1) >= (GC_SIZE_MAX >> 1), FALSE)) - return NULL; /* overflow */ - - start_list = GC_hblk_fl_from_blocks(blocks); - /* Try for an exact match first. */ - result = GC_allochblk_nth(sz, kind, flags, start_list, FALSE, align_m1); - if (result != NULL) return result; - - may_split = TRUE; - if (GC_use_entire_heap || GC_dont_gc || GC_heapsize - GC_large_free_bytes < GC_requested_heapsize || GC_incremental || !GC_should_collect()) { - /* Should use more of the heap, even if it requires splitting. */ - split_limit = N_HBLK_FLS; - } else if (GC_finalizer_bytes_freed > (GC_heapsize >> 4)) { - /* If we are deallocating lots of memory from */ - /* finalizers, fail and collect sooner rather */ - /* than later. */ - split_limit = 0; - } else { - /* If we have enough large blocks left to cover any */ - /* previous request for large blocks, we go ahead */ - /* and split. Assuming a steady state, that should */ - /* be safe. It means that we can use the full */ - /* heap if we allocate only small objects. */ - split_limit = GC_enough_large_bytes_left(); -#ifdef USE_MUNMAP - if (split_limit > 0) - may_split = AVOID_SPLIT_REMAPPED; -#endif - } - if (start_list < UNIQUE_THRESHOLD && 0 == align_m1) { - /* No reason to try start_list again, since all blocks are exact */ - /* matches. */ - ++start_list; - } - for (; start_list <= split_limit; ++start_list) { - result = GC_allochblk_nth(sz, kind, flags, start_list, may_split, - align_m1); - if (result != NULL) break; - } - return result; -} - -STATIC long GC_large_alloc_warn_suppressed = 0; -/* Number of warnings suppressed so far. */ - -STATIC unsigned GC_drop_blacklisted_count = 0; -/* Counter of the cases when found block by */ -/* GC_allochblk_nth is blacklisted completely. */ - -#define ALIGN_PAD_SZ(p, align_m1) \ - (((align_m1) + 1 - (size_t)(word)(p)) & (align_m1)) - -static GC_bool next_hblk_fits_better(hdr *hhdr, word size_avail, - word size_needed, size_t align_m1) { - hdr *next_hdr; - word next_size; - size_t next_ofs; - struct hblk *next_hbp = hhdr->hb_next; - - if (NULL == next_hbp) return FALSE; /* no next block */ - GET_HDR(next_hbp, next_hdr); - next_size = next_hdr->hb_sz; - if (size_avail <= next_size) return FALSE; /* not enough size */ - - next_ofs = ALIGN_PAD_SZ(next_hbp, align_m1); - return next_size >= size_needed + next_ofs && !GC_is_black_listed(next_hbp + divHBLKSZ(next_ofs), size_needed); -} - -static struct hblk *find_nonbl_hblk(struct hblk *last_hbp, word size_remain, - word eff_size_needed, size_t align_m1) { - word search_end = ((word)last_hbp + size_remain) & ~(word)align_m1; - - do { - struct hblk *next_hbp; - - last_hbp += divHBLKSZ(ALIGN_PAD_SZ(last_hbp, align_m1)); - next_hbp = GC_is_black_listed(last_hbp, eff_size_needed); - if (NULL == next_hbp) return last_hbp; /* not black-listed */ - last_hbp = next_hbp; - } while ((word)last_hbp <= search_end); - return NULL; -} - -/* Allocate and drop the block in small chunks, to maximize the chance */ -/* that we will recover some later. hhdr should correspond to hbp. */ -static void drop_hblk_in_chunks(int n, struct hblk *hbp, hdr *hhdr) { - size_t total_size = (size_t)(hhdr->hb_sz); - struct hblk *limit = hbp + divHBLKSZ(total_size); - - GC_ASSERT(HDR(hbp) == hhdr); - GC_ASSERT(modHBLKSZ(total_size) == 0 && total_size > 0); - GC_large_free_bytes -= total_size; - GC_bytes_dropped += total_size; - GC_remove_from_fl_at(hhdr, n); - do { - (void)setup_header(hhdr, hbp, HBLKSIZE, PTRFREE, 0); /* cannot fail */ - if (GC_debugging_started) BZERO(hbp, HBLKSIZE); - if ((word)(++hbp) >= (word)limit) break; - - hhdr = GC_install_header(hbp); - } while (EXPECT(hhdr != NULL, TRUE)); /* no header allocation failure? */ -} - -/* The same as GC_allochblk, but with search restricted to the n-th */ -/* free list. flags should be IGNORE_OFF_PAGE or zero; may_split */ -/* indicates whether it is OK to split larger blocks; sz is in bytes. */ -/* If may_split is set to AVOID_SPLIT_REMAPPED, then memory remapping */ -/* followed by splitting should be generally avoided. Rounded-up sz */ -/* plus align_m1 value should be less than GC_SIZE_MAX/2. */ -STATIC struct hblk *GC_allochblk_nth(size_t sz, int kind, unsigned flags, - int n, int may_split, size_t align_m1) { - struct hblk *hbp, *last_hbp; - hdr *hhdr; /* header corresponding to hbp */ - word size_needed = HBLKSIZE * OBJ_SZ_TO_BLOCKS_CHECKED(sz); - /* number of bytes in requested objects */ - - GC_ASSERT(I_HOLD_LOCK()); - GC_ASSERT(((align_m1 + 1) & align_m1) == 0 && sz > 0); - GC_ASSERT(0 == align_m1 || modHBLKSZ(align_m1 + 1) == 0); -retry: - /* Search for a big enough block in free list. */ - for (hbp = GC_hblkfreelist[n];; hbp = hhdr->hb_next) { - word size_avail; /* bytes available in this block */ - size_t align_ofs; - - if (hbp /* != NULL */) { - /* CPPCHECK */ - } else { - return NULL; - } - GET_HDR(hbp, hhdr); /* set hhdr value */ - size_avail = hhdr->hb_sz; - if (!may_split && size_avail != size_needed) continue; - - align_ofs = ALIGN_PAD_SZ(hbp, align_m1); - if (size_avail < size_needed + align_ofs) - continue; /* the block is too small */ - - if (size_avail != size_needed) { - /* If the next heap block is obviously better, go on. */ - /* This prevents us from disassembling a single large */ - /* block to get tiny blocks. */ - if (next_hblk_fits_better(hhdr, size_avail, size_needed, align_m1)) - continue; - } - - if (IS_UNCOLLECTABLE(kind) || (kind == PTRFREE && size_needed <= MAX_BLACK_LIST_ALLOC)) { - last_hbp = hbp + divHBLKSZ(align_ofs); - break; - } - - last_hbp = find_nonbl_hblk(hbp, size_avail - size_needed, - (flags & IGNORE_OFF_PAGE) != 0 ? HBLKSIZE : size_needed, - align_m1); - /* Is non-blacklisted part of enough size? */ - if (last_hbp != NULL) { -#ifdef USE_MUNMAP - /* Avoid remapping followed by splitting. */ - if (may_split == AVOID_SPLIT_REMAPPED && last_hbp != hbp && !IS_MAPPED(hhdr)) - continue; -#endif - break; - } - - /* The block is completely blacklisted. If so, we need to */ - /* drop some such blocks, since otherwise we spend all our */ - /* time traversing them if pointer-free blocks are unpopular. */ - /* A dropped block will be reconsidered at next GC. */ - if (size_needed == HBLKSIZE && 0 == align_m1 && !GC_find_leak && IS_MAPPED(hhdr) && (++GC_drop_blacklisted_count & 3) == 0) { - struct hblk *prev = hhdr->hb_prev; - - drop_hblk_in_chunks(n, hbp, hhdr); - if (NULL == prev) goto retry; - /* Restore hhdr to point at free block. */ - hhdr = HDR(prev); - continue; - } - - if (size_needed > BL_LIMIT && size_avail - size_needed > BL_LIMIT) { - /* Punt, since anything else risks unreasonable heap growth. */ - if (++GC_large_alloc_warn_suppressed >= GC_large_alloc_warn_interval) { - WARN( - "Repeated allocation of very large block" - " (appr. size %" WARN_PRIuPTR - " KiB):\n" - "\tMay lead to memory leak and poor performance\n", - size_needed >> 10); - GC_large_alloc_warn_suppressed = 0; - } - last_hbp = hbp + divHBLKSZ(align_ofs); - break; - } - } - - GC_ASSERT(((word)last_hbp & align_m1) == 0); - if (last_hbp != hbp) { - hdr *last_hdr = GC_install_header(last_hbp); - - if (EXPECT(NULL == last_hdr, FALSE)) return NULL; - /* Make sure it's mapped before we mangle it. */ -#ifdef USE_MUNMAP - if (!IS_MAPPED(hhdr)) { - GC_adjust_num_unmapped(hbp, hhdr); - GC_remap((ptr_t)hbp, (size_t)(hhdr->hb_sz)); - hhdr->hb_flags &= ~WAS_UNMAPPED; - } -#endif - /* Split the block at last_hbp. */ - GC_split_block(hbp, hhdr, last_hbp, last_hdr, n); - /* We must now allocate last_hbp, since it may be on the */ - /* wrong free list. */ - hbp = last_hbp; - hhdr = last_hdr; - } - GC_ASSERT(hhdr->hb_sz >= size_needed); - -#ifdef USE_MUNMAP - if (!IS_MAPPED(hhdr)) { - GC_adjust_num_unmapped(hbp, hhdr); - GC_remap((ptr_t)hbp, (size_t)(hhdr->hb_sz)); - hhdr->hb_flags &= ~WAS_UNMAPPED; - /* Note: This may leave adjacent, mapped free blocks. */ - } -#endif - /* hbp may be on the wrong freelist; the parameter n is important. */ - hbp = GC_get_first_part(hbp, hhdr, (size_t)size_needed, n); - if (EXPECT(NULL == hbp, FALSE)) return NULL; - - /* Add it to map of valid blocks. */ - if (EXPECT(!GC_install_counts(hbp, (size_t)size_needed), FALSE)) - return NULL; /* This leaks memory under very rare conditions. */ - - /* Set up the header. */ - GC_ASSERT(HDR(hbp) == hhdr); - if (EXPECT(!setup_header(hhdr, hbp, sz, kind, flags), FALSE)) { - GC_remove_counts(hbp, (size_t)size_needed); - return NULL; /* ditto */ - } - -#ifndef GC_DISABLE_INCREMENTAL - /* Notify virtual dirty bit implementation that we are about to */ - /* write. Ensure that pointer-free objects are not protected */ - /* if it is avoidable. This also ensures that newly allocated */ - /* blocks are treated as dirty. Necessary since we don't */ - /* protect free blocks. */ - GC_ASSERT(modHBLKSZ(size_needed) == 0); - GC_remove_protection(hbp, divHBLKSZ(size_needed), - 0 == hhdr->hb_descr /* pointer-free */); -#endif - /* We just successfully allocated a block. Restart count of */ - /* consecutive failures. */ - GC_fail_count = 0; - - GC_large_free_bytes -= size_needed; - GC_ASSERT(IS_MAPPED(hhdr)); - return hbp; -} - -/* - * Free a heap block. - * - * Coalesce the block with its neighbors if possible. - * - * All mark words are assumed to be cleared. - */ -GC_INNER void GC_freehblk(struct hblk *hbp) { - struct hblk *next, *prev; - hdr *hhdr, *prevhdr, *nexthdr; - word size; - - GET_HDR(hbp, hhdr); - size = HBLKSIZE * OBJ_SZ_TO_BLOCKS(hhdr->hb_sz); - if ((size & SIGNB) != 0) - ABORT("Deallocating excessively large block. Too large an allocation?"); - /* Probably possible if we try to allocate more than half the address */ - /* space at once. If we don't catch it here, strange things happen */ - /* later. */ - GC_remove_counts(hbp, (size_t)size); - hhdr->hb_sz = size; -#ifdef USE_MUNMAP - hhdr->hb_last_reclaimed = (unsigned short)GC_gc_no; -#endif - - /* Check for duplicate deallocation in the easy case */ - if (HBLK_IS_FREE(hhdr)) { - ABORT_ARG1("Duplicate large block deallocation", - " of %p", (void *)hbp); - } - - GC_ASSERT(IS_MAPPED(hhdr)); - hhdr->hb_flags |= FREE_BLK; - next = (struct hblk *)((ptr_t)hbp + size); - GET_HDR(next, nexthdr); - prev = GC_free_block_ending_at(hbp); - /* Coalesce with successor, if possible */ - if (nexthdr != NULL && HBLK_IS_FREE(nexthdr) && IS_MAPPED(nexthdr) && !((hhdr->hb_sz + nexthdr->hb_sz) & SIGNB) /* no overflow */) { - GC_remove_from_fl(nexthdr); - hhdr->hb_sz += nexthdr->hb_sz; - GC_remove_header(next); - } - /* Coalesce with predecessor, if possible. */ - if (prev /* != NULL */) { /* CPPCHECK */ - prevhdr = HDR(prev); - if (IS_MAPPED(prevhdr) && !((hhdr->hb_sz + prevhdr->hb_sz) & SIGNB)) { - GC_remove_from_fl(prevhdr); - prevhdr->hb_sz += hhdr->hb_sz; -#ifdef USE_MUNMAP - prevhdr->hb_last_reclaimed = (unsigned short)GC_gc_no; -#endif - GC_remove_header(hbp); - hbp = prev; - hhdr = prevhdr; - } - } - /* FIXME: It is not clear we really always want to do these merges */ - /* with USE_MUNMAP, since it updates ages and hence prevents */ - /* unmapping. */ - - GC_large_free_bytes += size; - GC_add_to_fl(hbp, hhdr); -} diff --git a/vendor/bdwgc/alloc.c b/vendor/bdwgc/alloc.c deleted file mode 100644 index 658de808..00000000 --- a/vendor/bdwgc/alloc.c +++ /dev/null @@ -1,1697 +0,0 @@ -/* - * Copyright (c) 1988-1989 Hans-J. Boehm, Alan J. Demers - * Copyright (c) 1991-1996 by Xerox Corporation. All rights reserved. - * Copyright (c) 1996-1999 by Silicon Graphics. All rights reserved. - * Copyright (c) 1999-2011 Hewlett-Packard Development Company, L.P. - * Copyright (c) 2008-2022 Ivan Maidanski - * - * THIS MATERIAL IS PROVIDED AS IS, WITH ABSOLUTELY NO WARRANTY EXPRESSED - * OR IMPLIED. ANY USE IS AT YOUR OWN RISK. - * - * Permission is hereby granted to use or copy this program - * for any purpose, provided the above notices are retained on all copies. - * Permission to modify the code and to distribute modified code is granted, - * provided the above notices are retained, and a notice that the code was - * modified is included with the above copyright notice. - * - */ - -#ifdef _WIN32 -#define WIN32_LEAN_AND_MEAN -#include -#endif -#include "private/gc_priv.h" - -#if !defined(MACOS) && !defined(MSWINCE) -#include -#if !defined(GC_NO_TYPES) && !defined(SN_TARGET_PSP2) && !defined(__CC_ARM) -#include -#endif -#endif - -/* - * Separate free lists are maintained for different sized objects - * up to MAXOBJBYTES. - * The call GC_allocobj(i,k) ensures that the freelist for - * kind k objects of size i points to a non-empty - * free list. It returns a pointer to the first entry on the free list. - * In a single-threaded world, GC_allocobj may be called to allocate - * an object of small size lb (and NORMAL kind) as follows - * (GC_generic_malloc_inner is a wrapper over GC_allocobj which also - * fills in GC_size_map if needed): - * - * lg = GC_size_map[lb]; - * op = GC_objfreelist[lg]; - * if (NULL == op) { - * op = GC_generic_malloc_inner(lb, NORMAL, 0); - * } else { - * GC_objfreelist[lg] = obj_link(op); - * GC_bytes_allocd += GRANULES_TO_BYTES((word)lg); - * } - * - * Note that this is very fast if the free list is non-empty; it should - * only involve the execution of 4 or 5 simple instructions. - * All composite objects on freelists are cleared, except for - * their first word. - */ - -/* - * The allocator uses GC_allochblk to allocate large chunks of objects. - * These chunks all start on addresses which are multiples of - * HBLKSZ. Each allocated chunk has an associated header, - * which can be located quickly based on the address of the chunk. - * (See headers.c for details.) - * This makes it possible to check quickly whether an - * arbitrary address corresponds to an object administered by the - * allocator. - */ - -word GC_non_gc_bytes = 0; /* Number of bytes not intended to be collected */ - -word GC_gc_no = 0; - -#ifndef NO_CLOCK -static unsigned long full_gc_total_time = 0; /* in ms, may wrap */ -static unsigned full_gc_total_ns_frac = 0; /* fraction of 1 ms */ -static GC_bool measure_performance = FALSE; -/* Do performance measurements if set to true (e.g., */ -/* accumulation of the total time of full collections). */ - -GC_API void GC_CALL GC_start_performance_measurement(void) { - measure_performance = TRUE; -} - -GC_API unsigned long GC_CALL GC_get_full_gc_total_time(void) { - return full_gc_total_time; -} -#endif /* !NO_CLOCK */ - -#ifndef GC_DISABLE_INCREMENTAL -GC_INNER GC_bool GC_incremental = FALSE; /* By default, stop the world. */ -STATIC GC_bool GC_should_start_incremental_collection = FALSE; -#endif - -GC_API int GC_CALL GC_is_incremental_mode(void) { - return (int)GC_incremental; -} - -#ifdef THREADS -int GC_parallel = FALSE; /* By default, parallel GC is off. */ -#endif - -#if defined(GC_FULL_FREQ) && !defined(CPPCHECK) -int GC_full_freq = GC_FULL_FREQ; -#else -int GC_full_freq = 19; /* Every 20th collection is a full */ - /* collection, whether we need it */ - /* or not. */ -#endif - -STATIC GC_bool GC_need_full_gc = FALSE; -/* Need full GC due to heap growth. */ - -#ifdef THREAD_LOCAL_ALLOC -GC_INNER GC_bool GC_world_stopped = FALSE; -#endif - -STATIC GC_bool GC_disable_automatic_collection = FALSE; - -GC_API void GC_CALL GC_set_disable_automatic_collection(int value) { - LOCK(); - GC_disable_automatic_collection = (GC_bool)value; - UNLOCK(); -} - -GC_API int GC_CALL GC_get_disable_automatic_collection(void) { - int value; - - LOCK(); - value = (int)GC_disable_automatic_collection; - UNLOCK(); - return value; -} - -STATIC word GC_used_heap_size_after_full = 0; - -/* Version macros are now defined in gc_version.h, which is included by */ -/* gc.h, which is included by gc_priv.h. */ -#ifndef GC_NO_VERSION_VAR -EXTERN_C_BEGIN -extern const unsigned GC_version; -EXTERN_C_END -const unsigned GC_version = ((GC_VERSION_MAJOR << 16) | - (GC_VERSION_MINOR << 8) | GC_VERSION_MICRO); -#endif - -GC_API unsigned GC_CALL GC_get_version(void) { - return (GC_VERSION_MAJOR << 16) | (GC_VERSION_MINOR << 8) | - GC_VERSION_MICRO; -} - -/* some more variables */ - -#ifdef GC_DONT_EXPAND -int GC_dont_expand = TRUE; -#else -int GC_dont_expand = FALSE; -#endif - -#if defined(GC_FREE_SPACE_DIVISOR) && !defined(CPPCHECK) -word GC_free_space_divisor = GC_FREE_SPACE_DIVISOR; /* must be > 0 */ -#else -word GC_free_space_divisor = 3; -#endif - -GC_INNER int GC_CALLBACK GC_never_stop_func(void) { - return FALSE; -} - -#if defined(GC_TIME_LIMIT) && !defined(CPPCHECK) -unsigned long GC_time_limit = GC_TIME_LIMIT; -/* We try to keep pause times from exceeding */ -/* this by much. In milliseconds. */ -#elif defined(PARALLEL_MARK) -unsigned long GC_time_limit = GC_TIME_UNLIMITED; -/* The parallel marker cannot be interrupted for */ -/* now, so the time limit is absent by default. */ -#else -unsigned long GC_time_limit = 15; -#endif - -#ifndef NO_CLOCK -STATIC unsigned long GC_time_lim_nsec = 0; -/* The nanoseconds add-on to GC_time_limit */ -/* value. Not updated by GC_set_time_limit(). */ -/* Ignored if the value of GC_time_limit is */ -/* GC_TIME_UNLIMITED. */ - -#define TV_NSEC_LIMIT (1000UL * 1000) /* amount of nanoseconds in 1 ms */ - -GC_API void GC_CALL GC_set_time_limit_tv(struct GC_timeval_s tv) { - GC_ASSERT(tv.tv_ms <= GC_TIME_UNLIMITED); - GC_ASSERT(tv.tv_nsec < TV_NSEC_LIMIT); - GC_time_limit = tv.tv_ms; - GC_time_lim_nsec = tv.tv_nsec; -} - -GC_API struct GC_timeval_s GC_CALL GC_get_time_limit_tv(void) { - struct GC_timeval_s tv; - - tv.tv_ms = GC_time_limit; - tv.tv_nsec = GC_time_lim_nsec; - return tv; -} - -STATIC CLOCK_TYPE GC_start_time = CLOCK_TYPE_INITIALIZER; -/* Time at which we stopped world. */ -/* used only in GC_timeout_stop_func. */ -#endif /* !NO_CLOCK */ - -STATIC int GC_n_attempts = 0; /* Number of attempts at finishing */ - /* collection within GC_time_limit. */ - -STATIC GC_stop_func GC_default_stop_func = GC_never_stop_func; -/* Accessed holding the allocator lock. */ - -GC_API void GC_CALL GC_set_stop_func(GC_stop_func stop_func) { - GC_ASSERT(NONNULL_ARG_NOT_NULL(stop_func)); - LOCK(); - GC_default_stop_func = stop_func; - UNLOCK(); -} - -GC_API GC_stop_func GC_CALL GC_get_stop_func(void) { - GC_stop_func stop_func; - - LOCK(); - stop_func = GC_default_stop_func; - UNLOCK(); - return stop_func; -} - -#if defined(GC_DISABLE_INCREMENTAL) || defined(NO_CLOCK) -#define GC_timeout_stop_func GC_default_stop_func -#else -STATIC int GC_CALLBACK GC_timeout_stop_func(void) { - CLOCK_TYPE current_time; - static unsigned count = 0; - unsigned long time_diff, nsec_diff; - - if (GC_default_stop_func()) - return TRUE; - - if (GC_time_limit == GC_TIME_UNLIMITED || (count++ & 3) != 0) - return FALSE; - - GET_TIME(current_time); - time_diff = MS_TIME_DIFF(current_time, GC_start_time); - nsec_diff = NS_FRAC_TIME_DIFF(current_time, GC_start_time); -#if defined(CPPCHECK) - GC_noop1((word)&nsec_diff); -#endif - if (time_diff >= GC_time_limit && (time_diff > GC_time_limit || nsec_diff >= GC_time_lim_nsec)) { - GC_COND_LOG_PRINTF( - "Abandoning stopped marking after %lu ms %lu ns" - " (attempt %d)\n", - time_diff, nsec_diff, GC_n_attempts); - return TRUE; - } - - return FALSE; -} -#endif /* !GC_DISABLE_INCREMENTAL */ - -#ifdef THREADS -GC_INNER word GC_total_stacksize = 0; /* updated on every push_all_stacks */ -#endif - -static size_t min_bytes_allocd_minimum = 1; -/* The lowest value returned by min_bytes_allocd(). */ - -GC_API void GC_CALL GC_set_min_bytes_allocd(size_t value) { - GC_ASSERT(value > 0); - min_bytes_allocd_minimum = value; -} - -GC_API size_t GC_CALL GC_get_min_bytes_allocd(void) { - return min_bytes_allocd_minimum; -} - -/* Return the minimum number of bytes that must be allocated between */ -/* collections to amortize the collection cost. Should be non-zero. */ -static word min_bytes_allocd(void) { - word result; - word stack_size; - word total_root_size; /* includes double stack size, */ - /* since the stack is expensive */ - /* to scan. */ - word scan_size; /* Estimate of memory to be scanned */ - /* during normal GC. */ - -#ifdef THREADS - if (GC_need_to_lock) { - /* We are multi-threaded... */ - stack_size = GC_total_stacksize; - /* For now, we just use the value computed during the latest GC. */ -#ifdef DEBUG_THREADS - GC_log_printf("Total stacks size: %lu\n", - (unsigned long)stack_size); -#endif - } else -#endif - /* else*/ { -#ifdef STACK_NOT_SCANNED - stack_size = 0; -#elif defined(STACK_GROWS_UP) - stack_size = GC_approx_sp() - GC_stackbottom; -#else - stack_size = GC_stackbottom - GC_approx_sp(); -#endif - } - - total_root_size = 2 * stack_size + GC_root_size; - scan_size = 2 * GC_composite_in_use + GC_atomic_in_use / 4 + total_root_size; - result = scan_size / GC_free_space_divisor; - if (GC_incremental) { - result /= 2; - } - return result > min_bytes_allocd_minimum - ? result - : min_bytes_allocd_minimum; -} - -STATIC word GC_non_gc_bytes_at_gc = 0; -/* Number of explicitly managed bytes of storage */ -/* at last collection. */ - -/* Return the number of bytes allocated, adjusted for explicit storage */ -/* management, etc.. This number is used in deciding when to trigger */ -/* collections. */ -STATIC word GC_adj_bytes_allocd(void) { - signed_word result; - signed_word expl_managed = (signed_word)GC_non_gc_bytes - (signed_word)GC_non_gc_bytes_at_gc; - - /* Don't count what was explicitly freed, or newly allocated for */ - /* explicit management. Note that deallocating an explicitly */ - /* managed object should not alter result, assuming the client */ - /* is playing by the rules. */ - result = (signed_word)GC_bytes_allocd + (signed_word)GC_bytes_dropped - (signed_word)GC_bytes_freed + (signed_word)GC_finalizer_bytes_freed - expl_managed; - if (result > (signed_word)GC_bytes_allocd) { - result = GC_bytes_allocd; - /* probably client bug or unfortunate scheduling */ - } - result += GC_bytes_finalized; - /* We count objects enqueued for finalization as though they */ - /* had been reallocated this round. Finalization is user */ - /* visible progress. And if we don't count this, we have */ - /* stability problems for programs that finalize all objects. */ - if (result < (signed_word)(GC_bytes_allocd >> 3)) { - /* Always count at least 1/8 of the allocations. We don't want */ - /* to collect too infrequently, since that would inhibit */ - /* coalescing of free storage blocks. */ - /* This also makes us partially robust against client bugs. */ - result = (signed_word)(GC_bytes_allocd >> 3); - } - return (word)result; -} - -/* Clear up a few frames worth of garbage left at the top of the stack. */ -/* This is used to prevent us from accidentally treating garbage left */ -/* on the stack by other parts of the collector as roots. This */ -/* differs from the code in misc.c, which actually tries to keep the */ -/* stack clear of long-lived, client-generated garbage. */ -STATIC void GC_clear_a_few_frames(void) { -#ifndef CLEAR_NWORDS -#define CLEAR_NWORDS 64 -#endif - volatile word frames[CLEAR_NWORDS]; - BZERO((word *)frames, CLEAR_NWORDS * sizeof(word)); -} - -GC_API void GC_CALL GC_start_incremental_collection(void) { -#ifndef GC_DISABLE_INCREMENTAL - if (!GC_incremental) return; - - LOCK(); - GC_should_start_incremental_collection = TRUE; - ENTER_GC(); - GC_collect_a_little_inner(1); - EXIT_GC(); - UNLOCK(); -#endif -} - -/* Have we allocated enough to amortize a collection? */ -GC_INNER GC_bool GC_should_collect(void) { - static word last_min_bytes_allocd; - static word last_gc_no; - - GC_ASSERT(I_HOLD_LOCK()); - if (last_gc_no != GC_gc_no) { - last_min_bytes_allocd = min_bytes_allocd(); - last_gc_no = GC_gc_no; - } -#ifndef GC_DISABLE_INCREMENTAL - if (GC_should_start_incremental_collection) { - GC_should_start_incremental_collection = FALSE; - return TRUE; - } -#endif - if (GC_disable_automatic_collection) return FALSE; - - if (GC_last_heap_growth_gc_no == GC_gc_no) - return TRUE; /* avoid expanding past limits used by blacklisting */ - - return GC_adj_bytes_allocd() >= last_min_bytes_allocd; -} - -/* STATIC */ GC_start_callback_proc GC_start_call_back = 0; -/* Called at start of full collections. */ -/* Not called if 0. Called with the allocation */ -/* lock held. Not used by GC itself. */ - -GC_API void GC_CALL GC_set_start_callback(GC_start_callback_proc fn) { - LOCK(); - GC_start_call_back = fn; - UNLOCK(); -} - -GC_API GC_start_callback_proc GC_CALL GC_get_start_callback(void) { - GC_start_callback_proc fn; - - LOCK(); - fn = GC_start_call_back; - UNLOCK(); - return fn; -} - -GC_INLINE void GC_notify_full_gc(void) { - if (GC_start_call_back != 0) { - (*GC_start_call_back)(); - } -} - -STATIC GC_bool GC_is_full_gc = FALSE; - -STATIC GC_bool GC_stopped_mark(GC_stop_func stop_func); -STATIC void GC_finish_collection(void); - -/* Initiate a garbage collection if appropriate. Choose judiciously */ -/* between partial, full, and stop-world collections. */ -STATIC void GC_maybe_gc(void) { - static int n_partial_gcs = 0; - - GC_ASSERT(I_HOLD_LOCK()); - ASSERT_CANCEL_DISABLED(); - if (!GC_should_collect()) return; - - if (!GC_incremental) { - GC_gcollect_inner(); - return; - } - - GC_ASSERT(!GC_collection_in_progress()); -#ifdef PARALLEL_MARK - if (GC_parallel) - GC_wait_for_reclaim(); -#endif - if (GC_need_full_gc || n_partial_gcs >= GC_full_freq) { - GC_COND_LOG_PRINTF( - "***>Full mark for collection #%lu after %lu allocd bytes\n", - (unsigned long)GC_gc_no + 1, (unsigned long)GC_bytes_allocd); - GC_promote_black_lists(); - (void)GC_reclaim_all((GC_stop_func)0, TRUE); - GC_notify_full_gc(); - GC_clear_marks(); - n_partial_gcs = 0; - GC_is_full_gc = TRUE; - } else { - n_partial_gcs++; - } - - /* Try to mark with the world stopped. If we run out of */ - /* time, this turns into an incremental marking. */ -#ifndef NO_CLOCK - if (GC_time_limit != GC_TIME_UNLIMITED) GET_TIME(GC_start_time); -#endif - if (GC_stopped_mark(GC_timeout_stop_func)) { -#ifdef SAVE_CALL_CHAIN - GC_save_callers(GC_last_stack); -#endif - GC_finish_collection(); - } else if (!GC_is_full_gc) { - /* Count this as the first attempt. */ - GC_n_attempts++; - } -} - -STATIC GC_on_collection_event_proc GC_on_collection_event = 0; - -GC_API void GC_CALL GC_set_on_collection_event(GC_on_collection_event_proc fn) { - /* fn may be 0 (means no event notifier). */ - LOCK(); - GC_on_collection_event = fn; - UNLOCK(); -} - -GC_API GC_on_collection_event_proc GC_CALL GC_get_on_collection_event(void) { - GC_on_collection_event_proc fn; - - LOCK(); - fn = GC_on_collection_event; - UNLOCK(); - return fn; -} - -/* Stop the world garbage collection. If stop_func is not */ -/* GC_never_stop_func then abort if stop_func returns TRUE. */ -/* Return TRUE if we successfully completed the collection. */ -GC_INNER GC_bool GC_try_to_collect_inner(GC_stop_func stop_func) { -#ifndef NO_CLOCK - CLOCK_TYPE start_time = CLOCK_TYPE_INITIALIZER; - GC_bool start_time_valid; -#endif - - ASSERT_CANCEL_DISABLED(); - GC_ASSERT(I_HOLD_LOCK()); - GC_ASSERT(GC_is_initialized); - if (GC_dont_gc || (*stop_func)()) return FALSE; - if (GC_on_collection_event) - GC_on_collection_event(GC_EVENT_START); - if (GC_incremental && GC_collection_in_progress()) { - GC_COND_LOG_PRINTF( - "GC_try_to_collect_inner: finishing collection in progress\n"); - /* Just finish collection already in progress. */ - while (GC_collection_in_progress()) { - if ((*stop_func)()) { - /* TODO: Notify GC_EVENT_ABANDON */ - return FALSE; - } - ENTER_GC(); - GC_collect_a_little_inner(1); - EXIT_GC(); - } - } - GC_notify_full_gc(); -#ifndef NO_CLOCK - start_time_valid = FALSE; - if ((GC_print_stats | (int)measure_performance) != 0) { - if (GC_print_stats) - GC_log_printf("Initiating full world-stop collection!\n"); - start_time_valid = TRUE; - GET_TIME(start_time); - } -#endif - GC_promote_black_lists(); - /* Make sure all blocks have been reclaimed, so sweep routines */ - /* don't see cleared mark bits. */ - /* If we're guaranteed to finish, then this is unnecessary. */ - /* In the find_leak case, we have to finish to guarantee that */ - /* previously unmarked objects are not reported as leaks. */ -#ifdef PARALLEL_MARK - if (GC_parallel) - GC_wait_for_reclaim(); -#endif - if ((GC_find_leak || stop_func != GC_never_stop_func) && !GC_reclaim_all(stop_func, FALSE)) { - /* Aborted. So far everything is still consistent. */ - /* TODO: Notify GC_EVENT_ABANDON */ - return FALSE; - } - GC_invalidate_mark_state(); /* Flush mark stack. */ - GC_clear_marks(); -#ifdef SAVE_CALL_CHAIN - GC_save_callers(GC_last_stack); -#endif - GC_is_full_gc = TRUE; - if (!GC_stopped_mark(stop_func)) { - if (!GC_incremental) { - /* We're partially done and have no way to complete or use */ - /* current work. Reestablish invariants as cheaply as */ - /* possible. */ - GC_invalidate_mark_state(); - GC_unpromote_black_lists(); - } /* else we claim the world is already still consistent. We'll */ - /* finish incrementally. */ - /* TODO: Notify GC_EVENT_ABANDON */ - return FALSE; - } - GC_finish_collection(); -#ifndef NO_CLOCK - if (start_time_valid) { - CLOCK_TYPE current_time; - unsigned long time_diff, ns_frac_diff; - - GET_TIME(current_time); - time_diff = MS_TIME_DIFF(current_time, start_time); - ns_frac_diff = NS_FRAC_TIME_DIFF(current_time, start_time); - if (measure_performance) { - full_gc_total_time += time_diff; /* may wrap */ - full_gc_total_ns_frac += (unsigned)ns_frac_diff; - if (full_gc_total_ns_frac >= 1000000U) { - /* Overflow of the nanoseconds part. */ - full_gc_total_ns_frac -= 1000000U; - full_gc_total_time++; - } - } - if (GC_print_stats) - GC_log_printf("Complete collection took %lu ms %lu ns\n", - time_diff, ns_frac_diff); - } -#endif - if (GC_on_collection_event) - GC_on_collection_event(GC_EVENT_END); - return TRUE; -} - -/* The number of extra calls to GC_mark_some that we have made. */ -STATIC int GC_deficit = 0; - -/* The default value of GC_rate. */ -#ifndef GC_RATE -#define GC_RATE 10 -#endif - -/* When GC_collect_a_little_inner() performs n units of GC work, a unit */ -/* is intended to touch roughly GC_rate pages. (But, every once in */ -/* a while, we do more than that.) This needs to be a fairly large */ -/* number with our current incremental GC strategy, since otherwise we */ -/* allocate too much during GC, and the cleanup gets expensive. */ -STATIC int GC_rate = GC_RATE; - -GC_API void GC_CALL GC_set_rate(int value) { - GC_ASSERT(value > 0); - GC_rate = value; -} - -GC_API int GC_CALL GC_get_rate(void) { - return GC_rate; -} - -/* The default maximum number of prior attempts at world stop marking. */ -#ifndef MAX_PRIOR_ATTEMPTS -#define MAX_PRIOR_ATTEMPTS 3 -#endif - -/* The maximum number of prior attempts at world stop marking. */ -/* A value of 1 means that we finish the second time, no matter how */ -/* long it takes. Does not count the initial root scan for a full GC. */ -static int max_prior_attempts = MAX_PRIOR_ATTEMPTS; - -GC_API void GC_CALL GC_set_max_prior_attempts(int value) { - GC_ASSERT(value >= 0); - max_prior_attempts = value; -} - -GC_API int GC_CALL GC_get_max_prior_attempts(void) { - return max_prior_attempts; -} - -GC_INNER void GC_collect_a_little_inner(int n) { - IF_CANCEL(int cancel_state;) - - GC_ASSERT(I_HOLD_LOCK()); - GC_ASSERT(GC_is_initialized); - if (GC_dont_gc) return; - - DISABLE_CANCEL(cancel_state); - if (GC_incremental && GC_collection_in_progress()) { - int i; - int max_deficit = GC_rate * n; - -#ifdef PARALLEL_MARK - if (GC_time_limit != GC_TIME_UNLIMITED) - GC_parallel_mark_disabled = TRUE; -#endif - for (i = GC_deficit; i < max_deficit; i++) { - if (GC_mark_some(NULL)) - break; - } -#ifdef PARALLEL_MARK - GC_parallel_mark_disabled = FALSE; -#endif - - if (i < max_deficit) { - GC_ASSERT(!GC_collection_in_progress()); - /* Need to follow up with a full collection. */ -#ifdef SAVE_CALL_CHAIN - GC_save_callers(GC_last_stack); -#endif -#ifdef PARALLEL_MARK - if (GC_parallel) - GC_wait_for_reclaim(); -#endif -#ifndef NO_CLOCK - if (GC_time_limit != GC_TIME_UNLIMITED && GC_n_attempts < max_prior_attempts) - GET_TIME(GC_start_time); -#endif - if (GC_stopped_mark(GC_n_attempts < max_prior_attempts ? GC_timeout_stop_func : GC_never_stop_func)) { - GC_finish_collection(); - } else { - GC_n_attempts++; - } - } - if (GC_deficit > 0) { - GC_deficit -= max_deficit; - if (GC_deficit < 0) - GC_deficit = 0; - } - } else { - GC_maybe_gc(); - } - RESTORE_CANCEL(cancel_state); -} - -GC_INNER void (*GC_check_heap)(void) = 0; -GC_INNER void (*GC_print_all_smashed)(void) = 0; - -GC_API int GC_CALL GC_collect_a_little(void) { - int result; - - if (!EXPECT(GC_is_initialized, TRUE)) GC_init(); - LOCK(); - ENTER_GC(); - GC_collect_a_little_inner(1); - EXIT_GC(); - result = (int)GC_collection_in_progress(); - UNLOCK(); - if (!result && GC_debugging_started) GC_print_all_smashed(); - return result; -} - -#ifndef NO_CLOCK -/* Variables for world-stop average delay time statistic computation. */ -/* "divisor" is incremented every world-stop and halved when reached */ -/* its maximum (or upon "total_time" overflow). */ -static unsigned world_stopped_total_time = 0; -static unsigned world_stopped_total_divisor = 0; -#ifndef MAX_TOTAL_TIME_DIVISOR -/* We shall not use big values here (so "outdated" delay time */ -/* values would have less impact on "average" delay time value than */ -/* newer ones). */ -#define MAX_TOTAL_TIME_DIVISOR 1000 -#endif -#endif /* !NO_CLOCK */ - -#ifdef USE_MUNMAP -#ifndef MUNMAP_THRESHOLD -#define MUNMAP_THRESHOLD 7 -#endif -GC_INNER unsigned GC_unmap_threshold = MUNMAP_THRESHOLD; - -#define IF_USE_MUNMAP(x) x -#define COMMA_IF_USE_MUNMAP(x) /* comma */ , x -#else -#define IF_USE_MUNMAP(x) /* empty */ -#define COMMA_IF_USE_MUNMAP(x) /* empty */ -#endif - -/* We stop the world and mark from all roots. If stop_func() ever */ -/* returns TRUE, we may fail and return FALSE. Increment GC_gc_no if */ -/* we succeed. */ -STATIC GC_bool GC_stopped_mark(GC_stop_func stop_func) { - int i; - ptr_t cold_gc_frame = GC_approx_sp(); -#ifndef NO_CLOCK - CLOCK_TYPE start_time = CLOCK_TYPE_INITIALIZER; -#endif - - GC_ASSERT(I_HOLD_LOCK()); - GC_ASSERT(GC_is_initialized); -#if !defined(REDIRECT_MALLOC) && defined(USE_WINALLOC) - GC_add_current_malloc_heap(); -#endif -#if defined(REGISTER_LIBRARIES_EARLY) - GC_cond_register_dynamic_libraries(); -#endif - -#ifndef NO_CLOCK - if (GC_PRINT_STATS_FLAG) - GET_TIME(start_time); -#endif - -#if !defined(GC_NO_FINALIZATION) && !defined(GC_TOGGLE_REFS_NOT_NEEDED) - GC_process_togglerefs(); -#endif -#ifdef THREADS - if (GC_on_collection_event) - GC_on_collection_event(GC_EVENT_PRE_STOP_WORLD); -#endif - STOP_WORLD(); -#ifdef THREADS - if (GC_on_collection_event) - GC_on_collection_event(GC_EVENT_POST_STOP_WORLD); -#endif - -#ifdef THREAD_LOCAL_ALLOC - GC_world_stopped = TRUE; -#endif - /* Output blank line for convenience here */ - GC_COND_LOG_PRINTF( - "\n--> Marking for collection #%lu after %lu allocated bytes\n", - (unsigned long)GC_gc_no + 1, (unsigned long)GC_bytes_allocd); -#ifdef MAKE_BACK_GRAPH - if (GC_print_back_height) { - GC_build_back_graph(); - } -#endif - - /* Mark from all roots. */ - if (GC_on_collection_event) - GC_on_collection_event(GC_EVENT_MARK_START); - - /* Minimize junk left in my registers and on the stack */ - GC_clear_a_few_frames(); - GC_noop6(0, 0, 0, 0, 0, 0); - - GC_initiate_gc(); -#ifdef PARALLEL_MARK - if (stop_func != GC_never_stop_func) - GC_parallel_mark_disabled = TRUE; -#endif - for (i = 0; !(*stop_func)(); i++) { - if (GC_mark_some(cold_gc_frame)) { -#ifdef PARALLEL_MARK - if (GC_parallel && GC_parallel_mark_disabled) { - GC_COND_LOG_PRINTF( - "Stopped marking done after %d iterations" - " with disabled parallel marker\n", - i); - } -#endif - i = -1; - break; - } - } -#ifdef PARALLEL_MARK - GC_parallel_mark_disabled = FALSE; -#endif - - if (i >= 0) { - GC_COND_LOG_PRINTF( - "Abandoned stopped marking after" - " %d iterations\n", - i); - GC_deficit = i; /* Give the mutator a chance. */ -#ifdef THREAD_LOCAL_ALLOC - GC_world_stopped = FALSE; -#endif - -#ifdef THREADS - if (GC_on_collection_event) - GC_on_collection_event(GC_EVENT_PRE_START_WORLD); -#endif - - START_WORLD(); - -#ifdef THREADS - if (GC_on_collection_event) - GC_on_collection_event(GC_EVENT_POST_START_WORLD); -#endif - - /* TODO: Notify GC_EVENT_MARK_ABANDON */ - return FALSE; - } - - GC_gc_no++; - /* Check all debugged objects for consistency */ - if (GC_debugging_started) { - (*GC_check_heap)(); - } - if (GC_on_collection_event) { - GC_on_collection_event(GC_EVENT_MARK_END); -#ifdef THREADS - GC_on_collection_event(GC_EVENT_PRE_START_WORLD); -#endif - } -#ifdef THREAD_LOCAL_ALLOC - GC_world_stopped = FALSE; -#endif - - START_WORLD(); - -#ifdef THREADS - if (GC_on_collection_event) - GC_on_collection_event(GC_EVENT_POST_START_WORLD); -#endif - -#ifndef NO_CLOCK - if (GC_PRINT_STATS_FLAG) { - unsigned long time_diff; - unsigned total_time, divisor; - CLOCK_TYPE current_time; - - GET_TIME(current_time); - time_diff = MS_TIME_DIFF(current_time, start_time); - - /* Compute new world-stop delay total time */ - total_time = world_stopped_total_time; - divisor = world_stopped_total_divisor; - if (total_time > (((unsigned)-1) >> 1) || divisor >= MAX_TOTAL_TIME_DIVISOR) { - /* Halve values if overflow occurs */ - total_time >>= 1; - divisor >>= 1; - } - total_time += time_diff < (((unsigned)-1) >> 1) ? (unsigned)time_diff : ((unsigned)-1) >> 1; - /* Update old world_stopped_total_time and its divisor */ - world_stopped_total_time = total_time; - world_stopped_total_divisor = ++divisor; - - GC_ASSERT(divisor != 0); - GC_log_printf( - "World-stopped marking took %lu ms %lu ns" - " (%u ms in average)\n", - time_diff, NS_FRAC_TIME_DIFF(current_time, start_time), - total_time / divisor); - } -#endif - return TRUE; -} - -/* Set all mark bits for the free list whose first entry is q */ -GC_INNER void GC_set_fl_marks(ptr_t q) { - if (q /* != NULL */) { /* CPPCHECK */ - struct hblk *h = HBLKPTR(q); - struct hblk *last_h = h; - hdr *hhdr = HDR(h); - IF_PER_OBJ(word sz = hhdr->hb_sz;) - - for (;;) { - word bit_no = MARK_BIT_NO((ptr_t)q - (ptr_t)h, sz); - - if (!mark_bit_from_hdr(hhdr, bit_no)) { - set_mark_bit_from_hdr(hhdr, bit_no); - ++hhdr->hb_n_marks; - } - - q = (ptr_t)obj_link(q); - if (q == NULL) - break; - - h = HBLKPTR(q); - if (h != last_h) { - last_h = h; - hhdr = HDR(h); - IF_PER_OBJ(sz = hhdr->hb_sz;) - } - } - } -} - -#if defined(GC_ASSERTIONS) && defined(THREAD_LOCAL_ALLOC) -/* Check that all mark bits for the free list whose first entry is */ -/* (*pfreelist) are set. Check skipped if points to a special value. */ -void GC_check_fl_marks(void **pfreelist) { - /* TODO: There is a data race with GC_FAST_MALLOC_GRANS (which does */ - /* not do atomic updates to the free-list). The race seems to be */ - /* harmless, and for now we just skip this check in case of TSan. */ -#if defined(AO_HAVE_load_acquire_read) && !defined(THREAD_SANITIZER) - AO_t *list = (AO_t *)AO_load_acquire_read((AO_t *)pfreelist); - /* Atomic operations are used because the world is running. */ - AO_t *prev; - AO_t *p; - - if ((word)list <= HBLKSIZE) return; - - prev = (AO_t *)pfreelist; - for (p = list; p != NULL;) { - AO_t *next; - - if (!GC_is_marked(p)) { - ABORT_ARG2("Unmarked local free list entry", - ": object %p on list %p", (void *)p, (void *)list); - } - - /* While traversing the free-list, it re-reads the pointer to */ - /* the current node before accepting its next pointer and */ - /* bails out if the latter has changed. That way, it won't */ - /* try to follow the pointer which might be been modified */ - /* after the object was returned to the client. It might */ - /* perform the mark-check on the just allocated object but */ - /* that should be harmless. */ - next = (AO_t *)AO_load_acquire_read(p); - if (AO_load(prev) != (AO_t)p) - break; - prev = p; - p = next; - } -#else - /* FIXME: Not implemented (just skipped). */ - (void)pfreelist; -#endif -} -#endif /* GC_ASSERTIONS && THREAD_LOCAL_ALLOC */ - -/* Clear all mark bits for the free list whose first entry is q */ -/* Decrement GC_bytes_found by number of bytes on free list. */ -STATIC void GC_clear_fl_marks(ptr_t q) { - struct hblk *h = HBLKPTR(q); - struct hblk *last_h = h; - hdr *hhdr = HDR(h); - word sz = hhdr->hb_sz; /* Normally set only once. */ - - for (;;) { - word bit_no = MARK_BIT_NO((ptr_t)q - (ptr_t)h, sz); - - if (mark_bit_from_hdr(hhdr, bit_no)) { - size_t n_marks = hhdr->hb_n_marks; - - GC_ASSERT(n_marks != 0); - clear_mark_bit_from_hdr(hhdr, bit_no); - n_marks--; -#ifdef PARALLEL_MARK - /* Appr. count, don't decrement to zero! */ - if (0 != n_marks || !GC_parallel) { - hhdr->hb_n_marks = n_marks; - } -#else - hhdr->hb_n_marks = n_marks; -#endif - } - GC_bytes_found -= sz; - - q = (ptr_t)obj_link(q); - if (q == NULL) - break; - - h = HBLKPTR(q); - if (h != last_h) { - last_h = h; - hhdr = HDR(h); - sz = hhdr->hb_sz; - } - } -} - -#if defined(GC_ASSERTIONS) && defined(THREAD_LOCAL_ALLOC) -void GC_check_tls(void); -#endif - -GC_on_heap_resize_proc GC_on_heap_resize = 0; - -/* Used for logging only. */ -GC_INLINE int GC_compute_heap_usage_percent(void) { - word used = GC_composite_in_use + GC_atomic_in_use + GC_bytes_allocd; - word heap_sz = GC_heapsize - GC_unmapped_bytes; -#if defined(CPPCHECK) - word limit = (GC_WORD_MAX >> 1) / 50; /* to avoid a false positive */ -#else - const word limit = GC_WORD_MAX / 100; -#endif - - return used >= heap_sz ? 0 : used < limit ? (int)((used * 100) / heap_sz) - : (int)(used / (heap_sz / 100)); -} - -#define GC_DBGLOG_PRINT_HEAP_IN_USE() \ - GC_DBGLOG_PRINTF("In-use heap: %d%% (%lu KiB pointers + %lu KiB other)\n", \ - GC_compute_heap_usage_percent(), \ - TO_KiB_UL(GC_composite_in_use), \ - TO_KiB_UL(GC_atomic_in_use + GC_bytes_allocd)) - -/* Finish up a collection. Assumes mark bits are consistent, but the */ -/* world is otherwise running. */ -STATIC void GC_finish_collection(void) { -#ifndef NO_CLOCK - CLOCK_TYPE start_time = CLOCK_TYPE_INITIALIZER; - CLOCK_TYPE finalize_time = CLOCK_TYPE_INITIALIZER; -#endif - - GC_ASSERT(I_HOLD_LOCK()); -#if defined(GC_ASSERTIONS) && defined(THREAD_LOCAL_ALLOC) && !defined(DBG_HDRS_ALL) - /* Check that we marked some of our own data. */ - /* TODO: Add more checks. */ - GC_check_tls(); -#endif - -#ifndef NO_CLOCK - if (GC_print_stats) - GET_TIME(start_time); -#endif - if (GC_on_collection_event) - GC_on_collection_event(GC_EVENT_RECLAIM_START); - -#ifndef GC_GET_HEAP_USAGE_NOT_NEEDED - if (GC_bytes_found > 0) - GC_reclaimed_bytes_before_gc += (word)GC_bytes_found; -#endif - GC_bytes_found = 0; -#if defined(LINUX) && defined(__ELF__) && !defined(SMALL_CONFIG) - if (GETENV("GC_PRINT_ADDRESS_MAP") != 0) { - GC_print_address_map(); - } -#endif - COND_DUMP; - if (GC_find_leak) { - /* Mark all objects on the free list. All objects should be */ - /* marked when we're done. */ - word size; /* current object size */ - unsigned kind; - ptr_t q; - - for (kind = 0; kind < GC_n_kinds; kind++) { - for (size = 1; size <= MAXOBJGRANULES; size++) { - q = (ptr_t)GC_obj_kinds[kind].ok_freelist[size]; - if (q != NULL) - GC_set_fl_marks(q); - } - } - GC_start_reclaim(TRUE); - /* The above just checks; it doesn't really reclaim anything. */ - } - -#ifndef GC_NO_FINALIZATION - GC_finalize(); -#endif -#ifndef NO_CLOCK - if (GC_print_stats) - GET_TIME(finalize_time); -#endif - - if (GC_print_back_height) { -#ifdef MAKE_BACK_GRAPH - GC_traverse_back_graph(); -#elif !defined(SMALL_CONFIG) - GC_err_printf( - "Back height not available: " - "Rebuild collector with -DMAKE_BACK_GRAPH\n"); -#endif - } - - /* Clear free list mark bits, in case they got accidentally marked */ - /* (or GC_find_leak is set and they were intentionally marked). */ - /* Also subtract memory remaining from GC_bytes_found count. */ - /* Note that composite objects on free list are cleared. */ - /* Thus accidentally marking a free list is not a problem; only */ - /* objects on the list itself will be marked, and that's fixed here. */ - { - word size; /* current object size */ - ptr_t q; /* pointer to current object */ - unsigned kind; - - for (kind = 0; kind < GC_n_kinds; kind++) { - for (size = 1; size <= MAXOBJGRANULES; size++) { - q = (ptr_t)GC_obj_kinds[kind].ok_freelist[size]; - if (q != NULL) - GC_clear_fl_marks(q); - } - } - } - - GC_VERBOSE_LOG_PRINTF("Bytes recovered before sweep - f.l. count = %ld\n", - (long)GC_bytes_found); - - /* Reconstruct free lists to contain everything not marked */ - GC_start_reclaim(FALSE); - -#ifdef USE_MUNMAP - if (GC_unmap_threshold > 0 /* unmapping enabled? */ - && EXPECT(GC_gc_no != 1, TRUE)) /* do not unmap during GC init */ - GC_unmap_old(GC_unmap_threshold); - - GC_ASSERT(GC_heapsize >= GC_unmapped_bytes); -#endif - GC_ASSERT(GC_our_mem_bytes >= GC_heapsize); - GC_DBGLOG_PRINTF("GC #%lu freed %ld bytes, heap %lu KiB (" IF_USE_MUNMAP("+ %lu KiB unmapped ") "+ %lu KiB internal)\n", - (unsigned long)GC_gc_no, (long)GC_bytes_found, - TO_KiB_UL(GC_heapsize - GC_unmapped_bytes) /*, */ - COMMA_IF_USE_MUNMAP(TO_KiB_UL(GC_unmapped_bytes)), - TO_KiB_UL(GC_our_mem_bytes - GC_heapsize + sizeof(GC_arrays))); - GC_DBGLOG_PRINT_HEAP_IN_USE(); - if (GC_is_full_gc) { - GC_used_heap_size_after_full = GC_heapsize - GC_large_free_bytes; - GC_need_full_gc = FALSE; - } else { - GC_need_full_gc = GC_heapsize - GC_used_heap_size_after_full > min_bytes_allocd() + GC_large_free_bytes; - } - - /* Reset or increment counters for next cycle */ - GC_n_attempts = 0; - GC_is_full_gc = FALSE; - GC_bytes_allocd_before_gc += GC_bytes_allocd; - GC_non_gc_bytes_at_gc = GC_non_gc_bytes; - GC_bytes_allocd = 0; - GC_bytes_dropped = 0; - GC_bytes_freed = 0; - GC_finalizer_bytes_freed = 0; - - if (GC_on_collection_event) - GC_on_collection_event(GC_EVENT_RECLAIM_END); -#ifndef NO_CLOCK - if (GC_print_stats) { - CLOCK_TYPE done_time; - - GET_TIME(done_time); -#if !defined(SMALL_CONFIG) && !defined(GC_NO_FINALIZATION) - /* A convenient place to output finalization statistics. */ - GC_print_finalization_stats(); -#endif - GC_log_printf( - "Finalize and initiate sweep took %lu ms %lu ns" - " + %lu ms %lu ns\n", - MS_TIME_DIFF(finalize_time, start_time), - NS_FRAC_TIME_DIFF(finalize_time, start_time), - MS_TIME_DIFF(done_time, finalize_time), - NS_FRAC_TIME_DIFF(done_time, finalize_time)); - } -#elif !defined(SMALL_CONFIG) && !defined(GC_NO_FINALIZATION) - if (GC_print_stats) - GC_print_finalization_stats(); -#endif -} - -STATIC word GC_heapsize_at_forced_unmap = 0; -/* accessed with the allocation lock held */ - -/* If stop_func == 0 then GC_default_stop_func is used instead. */ -STATIC GC_bool GC_try_to_collect_general(GC_stop_func stop_func, - GC_bool force_unmap) { - GC_bool result; - IF_USE_MUNMAP(int old_unmap_threshold;) - IF_CANCEL(int cancel_state;) - - if (!EXPECT(GC_is_initialized, TRUE)) GC_init(); - if (GC_debugging_started) GC_print_all_smashed(); - GC_INVOKE_FINALIZERS(); - LOCK(); - if (force_unmap) { - /* Record current heap size to make heap growth more conservative */ - /* afterwards (as if the heap is growing from zero size again). */ - GC_heapsize_at_forced_unmap = GC_heapsize; - } - DISABLE_CANCEL(cancel_state); -#ifdef USE_MUNMAP - old_unmap_threshold = GC_unmap_threshold; - if (force_unmap || - (GC_force_unmap_on_gcollect && old_unmap_threshold > 0)) - GC_unmap_threshold = 1; /* unmap as much as possible */ -#endif - ENTER_GC(); - /* Minimize junk left in my registers */ - GC_noop6(0, 0, 0, 0, 0, 0); - result = GC_try_to_collect_inner(stop_func != 0 ? stop_func : GC_default_stop_func); - EXIT_GC(); - IF_USE_MUNMAP(GC_unmap_threshold = old_unmap_threshold); /* restore */ - RESTORE_CANCEL(cancel_state); - UNLOCK(); - if (result) { - if (GC_debugging_started) GC_print_all_smashed(); - GC_INVOKE_FINALIZERS(); - } - return result; -} - -/* Externally callable routines to invoke full, stop-the-world collection. */ - -GC_API int GC_CALL GC_try_to_collect(GC_stop_func stop_func) { - GC_ASSERT(NONNULL_ARG_NOT_NULL(stop_func)); - return (int)GC_try_to_collect_general(stop_func, FALSE); -} - -GC_API void GC_CALL GC_gcollect(void) { - /* 0 is passed as stop_func to get GC_default_stop_func value */ - /* while holding the allocation lock (to prevent data races). */ - (void)GC_try_to_collect_general(0, FALSE); - if (get_have_errors()) - GC_print_all_errors(); -} - -GC_API void GC_CALL GC_gcollect_and_unmap(void) { - /* Collect and force memory unmapping to OS. */ - (void)GC_try_to_collect_general(GC_never_stop_func, TRUE); -} - -#ifdef USE_PROC_FOR_LIBRARIES -/* Add HBLKSIZE aligned, GET_MEM-generated block to GC_our_memory. */ -GC_INNER void GC_add_to_our_memory(ptr_t p, size_t bytes) { - GC_ASSERT(I_HOLD_LOCK()); - GC_ASSERT(p != NULL); - if (GC_n_memory >= MAX_HEAP_SECTS) - ABORT("Too many GC-allocated memory sections: Increase MAX_HEAP_SECTS"); - GC_our_memory[GC_n_memory].hs_start = p; - GC_our_memory[GC_n_memory].hs_bytes = bytes; - GC_n_memory++; - GC_our_mem_bytes += bytes; -} -#endif - -/* Use the chunk of memory starting at p of size bytes as part of the heap. */ -/* Assumes p is HBLKSIZE aligned, bytes argument is a multiple of HBLKSIZE. */ -STATIC void GC_add_to_heap(struct hblk *p, size_t bytes) { - hdr *phdr; - word endp; - size_t old_capacity = 0; - void *old_heap_sects = NULL; -#ifdef GC_ASSERTIONS - unsigned i; -#endif - - GC_ASSERT(I_HOLD_LOCK()); - GC_ASSERT((word)p % HBLKSIZE == 0); - GC_ASSERT(bytes % HBLKSIZE == 0); - GC_ASSERT(bytes > 0); - GC_ASSERT(GC_all_nils != NULL); - - if (GC_n_heap_sects == GC_capacity_heap_sects) { - /* Allocate new GC_heap_sects with sufficient capacity. */ -#ifndef INITIAL_HEAP_SECTS -#define INITIAL_HEAP_SECTS 32 -#endif - size_t new_capacity = GC_n_heap_sects > 0 ? (size_t)GC_n_heap_sects * 2 : INITIAL_HEAP_SECTS; - void *new_heap_sects = - GC_scratch_alloc(new_capacity * sizeof(struct HeapSect)); - - if (EXPECT(NULL == new_heap_sects, FALSE)) { - /* Retry with smaller yet sufficient capacity. */ - new_capacity = (size_t)GC_n_heap_sects + INITIAL_HEAP_SECTS; - new_heap_sects = - GC_scratch_alloc(new_capacity * sizeof(struct HeapSect)); - if (NULL == new_heap_sects) - ABORT("Insufficient memory for heap sections"); - } - old_capacity = GC_capacity_heap_sects; - old_heap_sects = GC_heap_sects; - /* Transfer GC_heap_sects contents to the newly allocated array. */ - if (GC_n_heap_sects > 0) - BCOPY(old_heap_sects, new_heap_sects, - GC_n_heap_sects * sizeof(struct HeapSect)); - GC_capacity_heap_sects = new_capacity; - GC_heap_sects = (struct HeapSect *)new_heap_sects; - GC_COND_LOG_PRINTF("Grew heap sections array to %lu elements\n", - (unsigned long)new_capacity); - } - - while ((word)p <= HBLKSIZE) { - /* Can't handle memory near address zero. */ - ++p; - bytes -= HBLKSIZE; - if (0 == bytes) return; - } - endp = (word)p + bytes; - if (endp <= (word)p) { - /* Address wrapped. */ - bytes -= HBLKSIZE; - if (0 == bytes) return; - endp -= HBLKSIZE; - } - phdr = GC_install_header(p); - if (EXPECT(NULL == phdr, FALSE)) { - /* This is extremely unlikely. Can't add it. This will */ - /* almost certainly result in a 0 return from the allocator, */ - /* which is entirely appropriate. */ - return; - } - GC_ASSERT(endp > (word)p && endp == (word)p + bytes); -#ifdef GC_ASSERTIONS - /* Ensure no intersection between sections. */ - for (i = 0; i < GC_n_heap_sects; i++) { - word hs_start = (word)GC_heap_sects[i].hs_start; - word hs_end = hs_start + GC_heap_sects[i].hs_bytes; - word p_e = (word)p + bytes; - - GC_ASSERT(!((hs_start <= (word)p && (word)p < hs_end) || (hs_start < p_e && p_e <= hs_end) || ((word)p < hs_start && hs_end < p_e))); - } -#endif - GC_heap_sects[GC_n_heap_sects].hs_start = (ptr_t)p; - GC_heap_sects[GC_n_heap_sects].hs_bytes = bytes; - GC_n_heap_sects++; - phdr->hb_sz = bytes; - phdr->hb_flags = 0; - GC_freehblk(p); - GC_heapsize += bytes; - - if ((word)p <= (word)GC_least_plausible_heap_addr || GC_least_plausible_heap_addr == 0) { - GC_least_plausible_heap_addr = (void *)((ptr_t)p - sizeof(word)); - /* Making it a little smaller than necessary prevents */ - /* us from getting a false hit from the variable */ - /* itself. There's some unintentional reflection */ - /* here. */ - } - if ((word)p + bytes >= (word)GC_greatest_plausible_heap_addr) { - GC_greatest_plausible_heap_addr = (void *)endp; - } - - if (old_capacity > 0) { -#ifndef GWW_VDB - /* Recycling may call GC_add_to_heap() again but should not */ - /* cause resizing of GC_heap_sects. */ - GC_scratch_recycle_no_gww(old_heap_sects, - old_capacity * sizeof(struct HeapSect)); -#else - /* TODO: implement GWW-aware recycling as in alloc_mark_stack */ - GC_noop1((word)old_heap_sects); -#endif - } -} - -#if !defined(NO_DEBUGGING) -void GC_print_heap_sects(void) { - unsigned i; - - GC_printf("Total heap size: %lu" IF_USE_MUNMAP(" (%lu unmapped)") "\n", - (unsigned long)GC_heapsize /*, */ - COMMA_IF_USE_MUNMAP((unsigned long)GC_unmapped_bytes)); - - for (i = 0; i < GC_n_heap_sects; i++) { - ptr_t start = GC_heap_sects[i].hs_start; - size_t len = GC_heap_sects[i].hs_bytes; - struct hblk *h; - unsigned nbl = 0; - - for (h = (struct hblk *)start; (word)h < (word)(start + len); h++) { - if (GC_is_black_listed(h, HBLKSIZE)) nbl++; - } - GC_printf("Section %d from %p to %p %u/%lu blacklisted\n", - i, (void *)start, (void *)&start[len], - nbl, (unsigned long)divHBLKSZ(len)); - } -} -#endif - -void *GC_least_plausible_heap_addr = (void *)GC_WORD_MAX; -void *GC_greatest_plausible_heap_addr = 0; - -STATIC word GC_max_heapsize = 0; - -GC_API void GC_CALL GC_set_max_heap_size(GC_word n) { - GC_max_heapsize = n; -} - -word GC_max_retries = 0; - -GC_INNER void GC_scratch_recycle_inner(void *ptr, size_t bytes) { - size_t page_offset; - size_t displ = 0; - size_t recycled_bytes; - - GC_ASSERT(I_HOLD_LOCK()); - if (NULL == ptr) return; - - GC_ASSERT(bytes != 0); - GC_ASSERT(GC_page_size != 0); - /* TODO: Assert correct memory flags if GWW_VDB */ - page_offset = (word)ptr & (GC_page_size - 1); - if (page_offset != 0) - displ = GC_page_size - page_offset; - recycled_bytes = bytes > displ ? (bytes - displ) & ~(GC_page_size - 1) : 0; - GC_COND_LOG_PRINTF("Recycle %lu/%lu scratch-allocated bytes at %p\n", - (unsigned long)recycled_bytes, (unsigned long)bytes, ptr); - if (recycled_bytes > 0) - GC_add_to_heap((struct hblk *)((word)ptr + displ), recycled_bytes); -} - -/* This explicitly increases the size of the heap. It is used */ -/* internally, but may also be invoked from GC_expand_hp by the user. */ -/* The argument is in units of HBLKSIZE (zero is treated as 1). */ -/* Returns FALSE on failure. */ -GC_INNER GC_bool GC_expand_hp_inner(word n) { - size_t bytes; - struct hblk *space; - word expansion_slop; /* Number of bytes by which we expect */ - /* the heap to expand soon. */ - - GC_ASSERT(I_HOLD_LOCK()); - GC_ASSERT(GC_page_size != 0); - if (0 == n) n = 1; - bytes = ROUNDUP_PAGESIZE((size_t)n * HBLKSIZE); - GC_DBGLOG_PRINT_HEAP_IN_USE(); - if (GC_max_heapsize != 0 && (GC_max_heapsize < (word)bytes || GC_heapsize > GC_max_heapsize - (word)bytes)) { - /* Exceeded self-imposed limit */ - return FALSE; - } - space = GET_MEM(bytes); - if (EXPECT(NULL == space, FALSE)) { - WARN("Failed to expand heap by %" WARN_PRIuPTR " KiB\n", bytes >> 10); - return FALSE; - } - GC_add_to_our_memory((ptr_t)space, bytes); - GC_last_heap_growth_gc_no = GC_gc_no; - GC_INFOLOG_PRINTF("Grow heap to %lu KiB after %lu bytes allocated\n", - TO_KiB_UL(GC_heapsize + bytes), - (unsigned long)GC_bytes_allocd); - - /* Adjust heap limits generously for blacklisting to work better. */ - /* GC_add_to_heap performs minimal adjustment needed for */ - /* correctness. */ - expansion_slop = min_bytes_allocd() + 4 * MAXHINCR * HBLKSIZE; - if ((GC_last_heap_addr == 0 && !((word)space & SIGNB)) || (GC_last_heap_addr != 0 && (word)GC_last_heap_addr < (word)space)) { - /* Assume the heap is growing up. */ - word new_limit = (word)space + (word)bytes + expansion_slop; - if (new_limit > (word)space && (word)GC_greatest_plausible_heap_addr < new_limit) - GC_greatest_plausible_heap_addr = (void *)new_limit; - } else { - /* Heap is growing down. */ - word new_limit = (word)space - expansion_slop; - if (new_limit < (word)space && (word)GC_least_plausible_heap_addr > new_limit) - GC_least_plausible_heap_addr = (void *)new_limit; - } - GC_last_heap_addr = (ptr_t)space; - - GC_add_to_heap(space, bytes); - if (GC_on_heap_resize) - (*GC_on_heap_resize)(GC_heapsize); - - return TRUE; -} - -/* Really returns a bool, but it's externally visible, so that's clumsy. */ -GC_API int GC_CALL GC_expand_hp(size_t bytes) { - word n_blocks = OBJ_SZ_TO_BLOCKS_CHECKED(bytes); - word old_heapsize; - GC_bool result; - - if (!EXPECT(GC_is_initialized, TRUE)) GC_init(); - LOCK(); - old_heapsize = GC_heapsize; - result = GC_expand_hp_inner(n_blocks); - if (result) { - GC_requested_heapsize += bytes; - if (GC_dont_gc) { - /* Do not call WARN if the heap growth is intentional. */ - GC_ASSERT(GC_heapsize >= old_heapsize); - GC_heapsize_on_gc_disable += GC_heapsize - old_heapsize; - } - } - UNLOCK(); - return (int)result; -} - -GC_INNER unsigned GC_fail_count = 0; -/* How many consecutive GC/expansion failures? */ -/* Reset by GC_allochblk. */ - -/* The minimum value of the ratio of allocated bytes since the latest */ -/* GC to the amount of finalizers created since that GC which triggers */ -/* the collection instead heap expansion. Has no effect in the */ -/* incremental mode. */ -#if defined(GC_ALLOCD_BYTES_PER_FINALIZER) && !defined(CPPCHECK) -STATIC word GC_allocd_bytes_per_finalizer = GC_ALLOCD_BYTES_PER_FINALIZER; -#else -STATIC word GC_allocd_bytes_per_finalizer = 10000; -#endif - -GC_API void GC_CALL GC_set_allocd_bytes_per_finalizer(GC_word value) { - GC_allocd_bytes_per_finalizer = value; -} - -GC_API GC_word GC_CALL GC_get_allocd_bytes_per_finalizer(void) { - return GC_allocd_bytes_per_finalizer; -} - -static word last_fo_entries = 0; -static word last_bytes_finalized = 0; - -/* Collect or expand heap in an attempt make the indicated number of */ -/* free blocks available. Should be called until the blocks are */ -/* available (setting retry value to TRUE unless this is the first call */ -/* in a loop) or until it fails by returning FALSE. The flags argument */ -/* should be IGNORE_OFF_PAGE or 0. */ -GC_INNER GC_bool GC_collect_or_expand(word needed_blocks, - unsigned flags, - GC_bool retry) { - GC_bool gc_not_stopped = TRUE; - word blocks_to_get; - IF_CANCEL(int cancel_state;) - - GC_ASSERT(I_HOLD_LOCK()); - GC_ASSERT(GC_is_initialized); - DISABLE_CANCEL(cancel_state); - if (!GC_incremental && !GC_dont_gc && - ((GC_dont_expand && GC_bytes_allocd > 0) || (GC_fo_entries > last_fo_entries && (last_bytes_finalized | GC_bytes_finalized) != 0 && (GC_fo_entries - last_fo_entries) * GC_allocd_bytes_per_finalizer > GC_bytes_allocd) || GC_should_collect())) { - /* Try to do a full collection using 'default' stop_func (unless */ - /* nothing has been allocated since the latest collection or heap */ - /* expansion is disabled). */ - gc_not_stopped = GC_try_to_collect_inner( - GC_bytes_allocd > 0 && (!GC_dont_expand || !retry) ? GC_default_stop_func : GC_never_stop_func); - if (gc_not_stopped == TRUE || !retry) { - /* Either the collection hasn't been aborted or this is the */ - /* first attempt (in a loop). */ - last_fo_entries = GC_fo_entries; - last_bytes_finalized = GC_bytes_finalized; - RESTORE_CANCEL(cancel_state); - return TRUE; - } - } - - blocks_to_get = (GC_heapsize - GC_heapsize_at_forced_unmap) / (HBLKSIZE * GC_free_space_divisor) + needed_blocks; - if (blocks_to_get > MAXHINCR) { - word slop; - - /* Get the minimum required to make it likely that we can satisfy */ - /* the current request in the presence of black-listing. */ - /* This will probably be more than MAXHINCR. */ - if ((flags & IGNORE_OFF_PAGE) != 0) { - slop = 4; - } else { - slop = 2 * divHBLKSZ(BL_LIMIT); - if (slop > needed_blocks) slop = needed_blocks; - } - if (needed_blocks + slop > MAXHINCR) { - blocks_to_get = needed_blocks + slop; - } else { - blocks_to_get = MAXHINCR; - } - if (blocks_to_get > divHBLKSZ(GC_WORD_MAX)) - blocks_to_get = divHBLKSZ(GC_WORD_MAX); - } else if (blocks_to_get < MINHINCR) { - blocks_to_get = MINHINCR; - } - - if (GC_max_heapsize > GC_heapsize) { - word max_get_blocks = divHBLKSZ(GC_max_heapsize - GC_heapsize); - if (blocks_to_get > max_get_blocks) - blocks_to_get = max_get_blocks > needed_blocks - ? max_get_blocks - : needed_blocks; - } - -#ifdef USE_MUNMAP - if (GC_unmap_threshold > 1) { - /* Return as much memory to the OS as possible before */ - /* trying to get memory from it. */ - GC_unmap_old(0); - } -#endif - if (!GC_expand_hp_inner(blocks_to_get) && (blocks_to_get == needed_blocks || !GC_expand_hp_inner(needed_blocks))) { - if (gc_not_stopped == FALSE) { - /* Don't increment GC_fail_count here (and no warning). */ - GC_gcollect_inner(); - GC_ASSERT(GC_bytes_allocd == 0); - } else if (GC_fail_count++ < GC_max_retries) { - WARN("Out of Memory! Trying to continue...\n", 0); - GC_gcollect_inner(); - } else { -#if !defined(AMIGA) || !defined(GC_AMIGA_FASTALLOC) -#ifdef USE_MUNMAP - GC_ASSERT(GC_heapsize >= GC_unmapped_bytes); -#endif - WARN("Out of Memory! Heap size: %" WARN_PRIuPTR - " MiB." - " Returning NULL!\n", - (GC_heapsize - GC_unmapped_bytes) >> 20); -#endif - RESTORE_CANCEL(cancel_state); - return FALSE; - } - } else if (GC_fail_count) { - GC_COND_LOG_PRINTF("Memory available again...\n"); - } - RESTORE_CANCEL(cancel_state); - return TRUE; -} - -/* - * Make sure the object free list for size gran (in granules) is not empty. - * Return a pointer to the first object on the free list. - * The object MUST BE REMOVED FROM THE FREE LIST BY THE CALLER. - */ -GC_INNER ptr_t GC_allocobj(size_t gran, int kind) { - void **flh = &(GC_obj_kinds[kind].ok_freelist[gran]); - GC_bool tried_minor = FALSE; - GC_bool retry = FALSE; - - GC_ASSERT(I_HOLD_LOCK()); - GC_ASSERT(GC_is_initialized); - if (0 == gran) return NULL; - - while (NULL == *flh) { - ENTER_GC(); -#ifndef GC_DISABLE_INCREMENTAL - if (GC_incremental && GC_time_limit != GC_TIME_UNLIMITED) { - /* True incremental mode, not just generational. */ - /* Do our share of marking work. */ - GC_collect_a_little_inner(1); - } -#endif - /* Sweep blocks for objects of this size */ - GC_ASSERT(!GC_is_full_gc || NULL == GC_obj_kinds[kind].ok_reclaim_list || NULL == GC_obj_kinds[kind].ok_reclaim_list[gran]); - GC_continue_reclaim(gran, kind); - EXIT_GC(); -#if defined(CPPCHECK) - GC_noop1((word)&flh); -#endif - if (NULL == *flh) { - GC_new_hblk(gran, kind); -#if defined(CPPCHECK) - GC_noop1((word)&flh); -#endif - if (NULL == *flh) { - ENTER_GC(); - if (GC_incremental && GC_time_limit == GC_TIME_UNLIMITED && !tried_minor) { - GC_collect_a_little_inner(1); - tried_minor = TRUE; - } else { - if (!GC_collect_or_expand(1, 0 /* flags */, retry)) { - EXIT_GC(); - return NULL; - } - retry = TRUE; - } - EXIT_GC(); - } - } - } - /* Successful allocation; reset failure count. */ - GC_fail_count = 0; - - return (ptr_t)(*flh); -} diff --git a/vendor/bdwgc/blacklst.c b/vendor/bdwgc/blacklst.c deleted file mode 100644 index c9a96500..00000000 --- a/vendor/bdwgc/blacklst.c +++ /dev/null @@ -1,296 +0,0 @@ -/* - * Copyright 1988, 1989 Hans-J. Boehm, Alan J. Demers - * Copyright (c) 1991-1994 by Xerox Corporation. All rights reserved. - * - * THIS MATERIAL IS PROVIDED AS IS, WITH ABSOLUTELY NO WARRANTY EXPRESSED - * OR IMPLIED. ANY USE IS AT YOUR OWN RISK. - * - * Permission is hereby granted to use or copy this program - * for any purpose, provided the above notices are retained on all copies. - * Permission to modify the code and to distribute modified code is granted, - * provided the above notices are retained, and a notice that the code was - * modified is included with the above copyright notice. - */ - -#ifdef _WIN32 -#define WIN32_LEAN_AND_MEAN -#include -#endif -#include "private/gc_priv.h" - -/* - * We maintain several hash tables of hblks that have had false hits. - * Each contains one bit per hash bucket; If any page in the bucket - * has had a false hit, we assume that all of them have. - * See the definition of page_hash_table in gc_private.h. - * False hits from the stack(s) are much more dangerous than false hits - * from elsewhere, since the former can pin a large object that spans the - * block, even though it does not start on the dangerous block. - */ - -/* Externally callable routines are: */ -/* - GC_add_to_black_list_normal, */ -/* - GC_add_to_black_list_stack, */ -/* - GC_promote_black_lists. */ - -/* Pointers to individual tables. We replace one table by another by */ -/* switching these pointers. */ -STATIC word *GC_old_normal_bl = NULL; -/* Nonstack false references seen at last full */ -/* collection. */ -STATIC word *GC_incomplete_normal_bl = NULL; -/* Nonstack false references seen since last */ -/* full collection. */ -STATIC word *GC_old_stack_bl = NULL; -STATIC word *GC_incomplete_stack_bl = NULL; - -STATIC word GC_total_stack_black_listed = 0; -/* Number of bytes on stack blacklist. */ - -GC_INNER word GC_black_list_spacing = MINHINCR * HBLKSIZE; -/* Initial rough guess. */ - -STATIC void GC_clear_bl(word *); - -GC_INNER void GC_default_print_heap_obj_proc(ptr_t p) { - ptr_t base = (ptr_t)GC_base(p); - int kind = HDR(base)->hb_obj_kind; - - GC_err_printf("object at %p of appr. %lu bytes (%s)\n", - (void *)base, (unsigned long)GC_size(base), - kind == PTRFREE ? "atomic" : IS_UNCOLLECTABLE(kind) ? "uncollectable" - : "composite"); -} - -GC_INNER void (*GC_print_heap_obj)(ptr_t p) = GC_default_print_heap_obj_proc; - -#ifdef PRINT_BLACK_LIST -STATIC void GC_print_blacklisted_ptr(word p, ptr_t source, - const char *kind_str) { - ptr_t base = (ptr_t)GC_base(source); - - if (0 == base) { - GC_err_printf("Black listing (%s) %p referenced from %p in %s\n", - kind_str, (void *)p, (void *)source, - NULL != source ? "root set" : "register"); - } else { - /* FIXME: We can't call the debug version of GC_print_heap_obj */ - /* (with PRINT_CALL_CHAIN) here because the lock is held and */ - /* the world is stopped. */ - GC_err_printf( - "Black listing (%s) %p referenced from %p in" - " object at %p of appr. %lu bytes\n", - kind_str, (void *)p, (void *)source, - (void *)base, (unsigned long)GC_size(base)); - } -} -#endif /* PRINT_BLACK_LIST */ - -GC_INNER void GC_bl_init_no_interiors(void) { - GC_ASSERT(I_HOLD_LOCK()); - if (GC_incomplete_normal_bl == 0) { - GC_old_normal_bl = (word *)GC_scratch_alloc(sizeof(page_hash_table)); - GC_incomplete_normal_bl = (word *)GC_scratch_alloc( - sizeof(page_hash_table)); - if (GC_old_normal_bl == 0 || GC_incomplete_normal_bl == 0) { - GC_err_printf("Insufficient memory for black list\n"); - EXIT(); - } - GC_clear_bl(GC_old_normal_bl); - GC_clear_bl(GC_incomplete_normal_bl); - } -} - -GC_INNER void GC_bl_init(void) { - GC_ASSERT(I_HOLD_LOCK()); - if (!GC_all_interior_pointers) { - GC_bl_init_no_interiors(); - } - GC_ASSERT(NULL == GC_old_stack_bl && NULL == GC_incomplete_stack_bl); - GC_old_stack_bl = (word *)GC_scratch_alloc(sizeof(page_hash_table)); - GC_incomplete_stack_bl = (word *)GC_scratch_alloc(sizeof(page_hash_table)); - if (GC_old_stack_bl == 0 || GC_incomplete_stack_bl == 0) { - GC_err_printf("Insufficient memory for black list\n"); - EXIT(); - } - GC_clear_bl(GC_old_stack_bl); - GC_clear_bl(GC_incomplete_stack_bl); -} - -STATIC void GC_clear_bl(word *doomed) { - BZERO(doomed, sizeof(page_hash_table)); -} - -STATIC void GC_copy_bl(word *old, word *dest) { - BCOPY(old, dest, sizeof(page_hash_table)); -} - -static word total_stack_black_listed(void); - -/* Signal the completion of a collection. Turn the incomplete black */ -/* lists into new black lists, etc. */ -GC_INNER void GC_promote_black_lists(void) { - word *very_old_normal_bl = GC_old_normal_bl; - word *very_old_stack_bl = GC_old_stack_bl; - - GC_ASSERT(I_HOLD_LOCK()); - GC_old_normal_bl = GC_incomplete_normal_bl; - GC_old_stack_bl = GC_incomplete_stack_bl; - if (!GC_all_interior_pointers) { - GC_clear_bl(very_old_normal_bl); - } - GC_clear_bl(very_old_stack_bl); - GC_incomplete_normal_bl = very_old_normal_bl; - GC_incomplete_stack_bl = very_old_stack_bl; - GC_total_stack_black_listed = total_stack_black_listed(); - GC_VERBOSE_LOG_PRINTF( - "%lu bytes in heap blacklisted for interior pointers\n", - (unsigned long)GC_total_stack_black_listed); - if (GC_total_stack_black_listed != 0) { - GC_black_list_spacing = - HBLKSIZE * (GC_heapsize / GC_total_stack_black_listed); - } - if (GC_black_list_spacing < 3 * HBLKSIZE) { - GC_black_list_spacing = 3 * HBLKSIZE; - } - if (GC_black_list_spacing > MAXHINCR * HBLKSIZE) { - GC_black_list_spacing = MAXHINCR * HBLKSIZE; - /* Makes it easier to allocate really huge blocks, which otherwise */ - /* may have problems with nonuniform blacklist distributions. */ - /* This way we should always succeed immediately after growing the */ - /* heap. */ - } -} - -GC_INNER void GC_unpromote_black_lists(void) { - if (!GC_all_interior_pointers) { - GC_copy_bl(GC_old_normal_bl, GC_incomplete_normal_bl); - } - GC_copy_bl(GC_old_stack_bl, GC_incomplete_stack_bl); -} - -#if defined(PARALLEL_MARK) && defined(THREAD_SANITIZER) -#define backlist_set_pht_entry_from_index(db, index) \ - set_pht_entry_from_index_concurrent(db, index) -#else -/* It is safe to set a bit in a blacklist even without */ -/* synchronization, the only drawback is that we might have */ -/* to redo blacklisting sometimes. */ -#define backlist_set_pht_entry_from_index(bl, index) \ - set_pht_entry_from_index(bl, index) -#endif - -/* P is not a valid pointer reference, but it falls inside */ -/* the plausible heap bounds. */ -/* Add it to the normal incomplete black list if appropriate. */ -#ifdef PRINT_BLACK_LIST -GC_INNER void GC_add_to_black_list_normal(word p, ptr_t source) -#else -GC_INNER void GC_add_to_black_list_normal(word p) -#endif -{ -#ifndef PARALLEL_MARK - GC_ASSERT(I_HOLD_LOCK()); -#endif - if (GC_modws_valid_offsets[p & (sizeof(word) - 1)]) { - word index = PHT_HASH((word)p); - - if (HDR(p) == 0 || get_pht_entry_from_index(GC_old_normal_bl, index)) { -#ifdef PRINT_BLACK_LIST - if (!get_pht_entry_from_index(GC_incomplete_normal_bl, index)) { - GC_print_blacklisted_ptr(p, source, "normal"); - } -#endif - backlist_set_pht_entry_from_index(GC_incomplete_normal_bl, index); - } /* else this is probably just an interior pointer to an allocated */ - /* object, and isn't worth black listing. */ - } -} - -/* And the same for false pointers from the stack. */ -#ifdef PRINT_BLACK_LIST -GC_INNER void GC_add_to_black_list_stack(word p, ptr_t source) -#else -GC_INNER void GC_add_to_black_list_stack(word p) -#endif -{ - word index = PHT_HASH((word)p); - -#ifndef PARALLEL_MARK - GC_ASSERT(I_HOLD_LOCK()); -#endif - if (HDR(p) == 0 || get_pht_entry_from_index(GC_old_stack_bl, index)) { -#ifdef PRINT_BLACK_LIST - if (!get_pht_entry_from_index(GC_incomplete_stack_bl, index)) { - GC_print_blacklisted_ptr(p, source, "stack"); - } -#endif - backlist_set_pht_entry_from_index(GC_incomplete_stack_bl, index); - } -} - -/* - * Is the block starting at h of size len bytes black listed? If so, - * return the address of the next plausible r such that (r, len) might not - * be black listed. (R may not actually be in the heap. We guarantee only - * that every smaller value of r after h is also black listed.) - * If (h,len) is not black listed, return 0. - * Knows about the structure of the black list hash tables. - * Assumes the allocation lock is held but no assertion about it by design. - */ -GC_API struct GC_hblk_s *GC_CALL GC_is_black_listed(struct GC_hblk_s *h, - GC_word len) { - word index = PHT_HASH((word)h); - word i; - word nblocks; - - if (!GC_all_interior_pointers && (get_pht_entry_from_index(GC_old_normal_bl, index) || get_pht_entry_from_index(GC_incomplete_normal_bl, index))) { - return h + 1; - } - - nblocks = divHBLKSZ(len); - for (i = 0;;) { - if (GC_old_stack_bl[divWORDSZ(index)] == 0 && GC_incomplete_stack_bl[divWORDSZ(index)] == 0) { - /* An easy case */ - i += WORDSZ - modWORDSZ(index); - } else { - if (get_pht_entry_from_index(GC_old_stack_bl, index) || get_pht_entry_from_index(GC_incomplete_stack_bl, index)) { - return h + (i + 1); - } - i++; - } - if (i >= nblocks) break; - index = PHT_HASH((word)(h + i)); - } - return NULL; -} - -/* Return the number of blacklisted blocks in a given range. */ -/* Used only for statistical purposes. */ -/* Looks only at the GC_incomplete_stack_bl. */ -STATIC word GC_number_stack_black_listed(struct hblk *start, - struct hblk *endp1) { - struct hblk *h; - word result = 0; - - for (h = start; (word)h < (word)endp1; h++) { - word index = PHT_HASH((word)h); - - if (get_pht_entry_from_index(GC_old_stack_bl, index)) result++; - } - return result; -} - -/* Return the total number of (stack) black-listed bytes. */ -static word total_stack_black_listed(void) { - unsigned i; - word total = 0; - - for (i = 0; i < GC_n_heap_sects; i++) { - struct hblk *start = (struct hblk *)GC_heap_sects[i].hs_start; - struct hblk *endp1 = start + divHBLKSZ(GC_heap_sects[i].hs_bytes); - - total += GC_number_stack_black_listed(start, endp1); - } - return total * HBLKSIZE; -} diff --git a/vendor/bdwgc/dbg_mlc.c b/vendor/bdwgc/dbg_mlc.c deleted file mode 100644 index 5fec25be..00000000 --- a/vendor/bdwgc/dbg_mlc.c +++ /dev/null @@ -1,1168 +0,0 @@ -/* - * Copyright 1988, 1989 Hans-J. Boehm, Alan J. Demers - * Copyright (c) 1991-1995 by Xerox Corporation. All rights reserved. - * Copyright (c) 1997 by Silicon Graphics. All rights reserved. - * Copyright (c) 1999-2004 Hewlett-Packard Development Company, L.P. - * Copyright (c) 2007 Free Software Foundation, Inc. - * Copyright (c) 2008-2022 Ivan Maidanski - * - * THIS MATERIAL IS PROVIDED AS IS, WITH ABSOLUTELY NO WARRANTY EXPRESSED - * OR IMPLIED. ANY USE IS AT YOUR OWN RISK. - * - * Permission is hereby granted to use or copy this program - * for any purpose, provided the above notices are retained on all copies. - * Permission to modify the code and to distribute modified code is granted, - * provided the above notices are retained, and a notice that the code was - * modified is included with the above copyright notice. - */ - -#ifdef _WIN32 -#define WIN32_LEAN_AND_MEAN -#include -#endif -#include "private/dbg_mlc.h" - -#ifndef MSWINCE -#include -#endif -#include - -#ifndef SHORT_DBG_HDRS -/* Check whether object with base pointer p has debugging info. */ -/* p is assumed to point to a legitimate object in our part */ -/* of the heap. */ -/* This excludes the check as to whether the back pointer is */ -/* odd, which is added by the GC_HAS_DEBUG_INFO macro. */ -/* Note that if DBG_HDRS_ALL is set, uncollectible objects */ -/* on free lists may not have debug information set. Thus it's */ -/* not always safe to return TRUE (1), even if the client does */ -/* its part. Return -1 if the object with debug info has been */ -/* marked as deallocated. */ -GC_INNER int GC_has_other_debug_info(ptr_t p) { - ptr_t body = (ptr_t)((oh *)p + 1); - word sz = GC_size(p); - - if (HBLKPTR(p) != HBLKPTR((ptr_t)body) || sz < DEBUG_BYTES + EXTRA_BYTES) { - return 0; - } - if (((oh *)p)->oh_sf != (START_FLAG ^ (word)body) && ((word *)p)[BYTES_TO_WORDS(sz) - 1] != (END_FLAG ^ (word)body)) { - return 0; - } - if (((oh *)p)->oh_sz == sz) { - /* Object may have had debug info, but has been deallocated */ - return -1; - } - return 1; -} -#endif /* !SHORT_DBG_HDRS */ - -#ifdef KEEP_BACK_PTRS - -#ifdef LINT2 -static int GC_rand(void) { - static GC_RAND_STATE_T seed; - - return GC_RAND_NEXT(&seed); -} - -#define RANDOM() (long)GC_rand() -#else -#undef GC_RAND_MAX -#define GC_RAND_MAX RAND_MAX - -#if defined(__GLIBC__) || defined(SOLARIS) || defined(HPUX) || defined(IRIX5) || defined(OSF1) -#define RANDOM() random() -#else -#define RANDOM() (long)rand() -#endif -#endif /* !LINT2 */ - -/* Store back pointer to source in dest, if that appears to be possible. */ -/* This is not completely safe, since we may mistakenly conclude that */ -/* dest has a debugging wrapper. But the error probability is very */ -/* small, and this shouldn't be used in production code. */ -/* We assume that dest is the real base pointer. Source will usually */ -/* be a pointer to the interior of an object. */ -GC_INNER void GC_store_back_pointer(ptr_t source, ptr_t dest) { - if (GC_HAS_DEBUG_INFO(dest)) { -#ifdef PARALLEL_MARK - AO_store((volatile AO_t *)&((oh *)dest)->oh_back_ptr, - (AO_t)HIDE_BACK_PTR(source)); -#else - ((oh *)dest)->oh_back_ptr = HIDE_BACK_PTR(source); -#endif - } -} - -GC_INNER void GC_marked_for_finalization(ptr_t dest) { - GC_store_back_pointer(MARKED_FOR_FINALIZATION, dest); -} - -/* Store information about the object referencing dest in *base_p */ -/* and *offset_p. */ -/* source is root ==> *base_p = address, *offset_p = 0 */ -/* source is heap object ==> *base_p != 0, *offset_p = offset */ -/* Returns 1 on success, 0 if source couldn't be determined. */ -/* Dest can be any address within a heap object. */ -GC_API GC_ref_kind GC_CALL GC_get_back_ptr_info(void *dest, void **base_p, - size_t *offset_p) { - oh *hdr = (oh *)GC_base(dest); - ptr_t bp; - ptr_t bp_base; - -#ifdef LINT2 - /* Explicitly instruct the code analysis tool that */ - /* GC_get_back_ptr_info is not expected to be called with an */ - /* incorrect "dest" value. */ - if (!hdr) ABORT("Invalid GC_get_back_ptr_info argument"); -#endif - if (!GC_HAS_DEBUG_INFO((ptr_t)hdr)) return GC_NO_SPACE; - bp = (ptr_t)GC_REVEAL_POINTER(hdr->oh_back_ptr); - if (MARKED_FOR_FINALIZATION == bp) return GC_FINALIZER_REFD; - if (MARKED_FROM_REGISTER == bp) return GC_REFD_FROM_REG; - if (NOT_MARKED == bp) return GC_UNREFERENCED; -#if ALIGNMENT == 1 - /* Heuristically try to fix off by 1 errors we introduced by */ - /* insisting on even addresses. */ - { - ptr_t alternate_ptr = bp + 1; - ptr_t target = *(ptr_t *)bp; - ptr_t alternate_target = *(ptr_t *)alternate_ptr; - - if ((word)alternate_target >= (word)GC_least_plausible_heap_addr && (word)alternate_target <= (word)GC_greatest_plausible_heap_addr && ((word)target < (word)GC_least_plausible_heap_addr || (word)target > (word)GC_greatest_plausible_heap_addr)) { - bp = alternate_ptr; - } - } -#endif - bp_base = (ptr_t)GC_base(bp); - if (NULL == bp_base) { - *base_p = bp; - *offset_p = 0; - return GC_REFD_FROM_ROOT; - } else { - if (GC_HAS_DEBUG_INFO(bp_base)) bp_base += sizeof(oh); - *base_p = bp_base; - *offset_p = bp - bp_base; - return GC_REFD_FROM_HEAP; - } -} - -/* Generate a random heap address. */ -/* The resulting address is in the heap, but */ -/* not necessarily inside a valid object. */ -GC_API void *GC_CALL GC_generate_random_heap_address(void) { - size_t i; - word heap_offset = (word)RANDOM(); - - if (GC_heapsize > (word)GC_RAND_MAX) { - heap_offset *= GC_RAND_MAX; - heap_offset += (word)RANDOM(); - } - heap_offset %= GC_heapsize; - /* This doesn't yield a uniform distribution, especially if */ - /* e.g. RAND_MAX is 1.5*GC_heapsize. But for typical cases, */ - /* it's not too bad. */ - - for (i = 0;; ++i) { - size_t size; - - if (i >= GC_n_heap_sects) - ABORT("GC_generate_random_heap_address: size inconsistency"); - - size = GC_heap_sects[i].hs_bytes; - if (heap_offset < size) { - break; - } else { - heap_offset -= size; - } - } - return GC_heap_sects[i].hs_start + heap_offset; -} - -/* Generate a random address inside a valid marked heap object. */ -GC_API void *GC_CALL GC_generate_random_valid_address(void) { - ptr_t result; - ptr_t base; - do { - result = (ptr_t)GC_generate_random_heap_address(); - base = (ptr_t)GC_base(result); - } while (NULL == base || !GC_is_marked(base)); - return result; -} - -/* Print back trace for p */ -GC_API void GC_CALL GC_print_backtrace(void *p) { - void *current = p; - int i; - GC_ref_kind source; - size_t offset; - void *base; - - GC_ASSERT(I_DONT_HOLD_LOCK()); - GC_print_heap_obj((ptr_t)GC_base(current)); - - for (i = 0;; ++i) { - source = GC_get_back_ptr_info(current, &base, &offset); - if (GC_UNREFERENCED == source) { - GC_err_printf("Reference could not be found\n"); - goto out; - } - if (GC_NO_SPACE == source) { - GC_err_printf("No debug info in object: Can't find reference\n"); - goto out; - } - GC_err_printf("Reachable via %d levels of pointers from ", i); - switch (source) { - case GC_REFD_FROM_ROOT: - GC_err_printf("root at %p\n\n", base); - goto out; - case GC_REFD_FROM_REG: - GC_err_printf("root in register\n\n"); - goto out; - case GC_FINALIZER_REFD: - GC_err_printf("list of finalizable objects\n\n"); - goto out; - case GC_REFD_FROM_HEAP: - GC_err_printf("offset %ld in object:\n", (long)offset); - /* Take GC_base(base) to get real base, i.e. header. */ - GC_print_heap_obj((ptr_t)GC_base(base)); - break; - default: - GC_err_printf("INTERNAL ERROR: UNEXPECTED SOURCE!!!!\n"); - goto out; - } - current = base; - } -out:; -} - -GC_API void GC_CALL GC_generate_random_backtrace(void) { - void *current; - - GC_ASSERT(I_DONT_HOLD_LOCK()); - if (GC_try_to_collect(GC_never_stop_func) == 0) { - GC_err_printf( - "Cannot generate a backtrace: " - "garbage collection is disabled!\n"); - return; - } - - /* Generate/print a backtrace from a random heap address. */ - LOCK(); - current = GC_generate_random_valid_address(); - UNLOCK(); - GC_printf("\n****Chosen address %p in object\n", current); - GC_print_backtrace(current); -} - -#endif /* KEEP_BACK_PTRS */ - -#define CROSSES_HBLK(p, sz) \ - (((word)((p) + sizeof(oh) + (sz)-1) ^ (word)(p)) >= HBLKSIZE) - -GC_INNER void *GC_store_debug_info_inner(void *p, word sz, - const char *string, int linenum) { - word *result = (word *)((oh *)p + 1); - - GC_ASSERT(I_HOLD_LOCK()); - GC_ASSERT(GC_size(p) >= sizeof(oh) + sz); - GC_ASSERT(!(SMALL_OBJ(sz) && CROSSES_HBLK((ptr_t)p, sz))); -#ifdef KEEP_BACK_PTRS - ((oh *)p)->oh_back_ptr = HIDE_BACK_PTR(NOT_MARKED); -#endif -#ifdef MAKE_BACK_GRAPH - ((oh *)p)->oh_bg_ptr = HIDE_BACK_PTR((ptr_t)0); -#endif - ((oh *)p)->oh_string = string; - ((oh *)p)->oh_int = linenum; -#ifdef SHORT_DBG_HDRS - UNUSED_ARG(sz); -#else - ((oh *)p)->oh_sz = sz; - ((oh *)p)->oh_sf = START_FLAG ^ (word)result; - ((word *)p)[BYTES_TO_WORDS(GC_size(p)) - 1] = - result[SIMPLE_ROUNDED_UP_WORDS(sz)] = END_FLAG ^ (word)result; -#endif - return result; -} - -/* Check the allocation is successful, store debugging info into p, */ -/* start the debugging mode (if not yet), and return displaced pointer. */ -static void *store_debug_info(void *p, size_t lb, - const char *fn, GC_EXTRA_PARAMS) { - void *result; - - if (NULL == p) { - GC_err_printf("%s(%lu) returning NULL (%s:%d)\n", - fn, (unsigned long)lb, s, i); - return NULL; - } - LOCK(); - if (!GC_debugging_started) - GC_start_debugging_inner(); - ADD_CALL_CHAIN(p, ra); - result = GC_store_debug_info_inner(p, (word)lb, s, i); - UNLOCK(); - return result; -} - -#ifndef SHORT_DBG_HDRS -/* Check the object with debugging info at ohdr. */ -/* Return NULL if it's OK. Else return clobbered */ -/* address. */ -STATIC ptr_t GC_check_annotated_obj(oh *ohdr) { - ptr_t body = (ptr_t)(ohdr + 1); - word gc_sz = GC_size((ptr_t)ohdr); - if (ohdr->oh_sz + DEBUG_BYTES > gc_sz) { - return (ptr_t)(&(ohdr->oh_sz)); - } - if (ohdr->oh_sf != (START_FLAG ^ (word)body)) { - return (ptr_t)(&(ohdr->oh_sf)); - } - if (((word *)ohdr)[BYTES_TO_WORDS(gc_sz) - 1] != (END_FLAG ^ (word)body)) { - return (ptr_t)(&((word *)ohdr)[BYTES_TO_WORDS(gc_sz) - 1]); - } - if (((word *)body)[SIMPLE_ROUNDED_UP_WORDS(ohdr->oh_sz)] != (END_FLAG ^ (word)body)) { - return (ptr_t)(&((word *)body)[SIMPLE_ROUNDED_UP_WORDS(ohdr->oh_sz)]); - } - return NULL; -} -#endif /* !SHORT_DBG_HDRS */ - -STATIC GC_describe_type_fn GC_describe_type_fns[MAXOBJKINDS] = {0}; - -GC_API void GC_CALL GC_register_describe_type_fn(int kind, - GC_describe_type_fn fn) { - GC_ASSERT((unsigned)kind < MAXOBJKINDS); - GC_describe_type_fns[kind] = fn; -} - -#define GET_OH_LINENUM(ohdr) ((int)(ohdr)->oh_int) - -#ifndef SHORT_DBG_HDRS -#define IF_NOT_SHORTDBG_HDRS(x) x -#define COMMA_IFNOT_SHORTDBG_HDRS(x) /* comma */ , x -#else -#define IF_NOT_SHORTDBG_HDRS(x) /* empty */ -#define COMMA_IFNOT_SHORTDBG_HDRS(x) /* empty */ -#endif - -/* Print a human-readable description of the object to stderr. */ -/* p points to somewhere inside an object with the debugging info. */ -STATIC void GC_print_obj(ptr_t p) { - oh *ohdr = (oh *)GC_base(p); - ptr_t q; - hdr *hhdr; - int kind; - const char *kind_str; - char buffer[GC_TYPE_DESCR_LEN + 1]; - - GC_ASSERT(I_DONT_HOLD_LOCK()); -#ifdef LINT2 - if (!ohdr) ABORT("Invalid GC_print_obj argument"); -#endif - - q = (ptr_t)(ohdr + 1); - /* Print a type description for the object whose client-visible */ - /* address is q. */ - hhdr = GC_find_header(q); - kind = hhdr->hb_obj_kind; - if (0 != GC_describe_type_fns[kind] && GC_is_marked(ohdr)) { - /* This should preclude free list objects except with */ - /* thread-local allocation. */ - buffer[GC_TYPE_DESCR_LEN] = 0; - (GC_describe_type_fns[kind])(q, buffer); - GC_ASSERT(buffer[GC_TYPE_DESCR_LEN] == 0); - kind_str = buffer; - } else { - switch (kind) { - case PTRFREE: - kind_str = "PTRFREE"; - break; - case NORMAL: - kind_str = "NORMAL"; - break; - case UNCOLLECTABLE: - kind_str = "UNCOLLECTABLE"; - break; -#ifdef GC_ATOMIC_UNCOLLECTABLE - case AUNCOLLECTABLE: - kind_str = "ATOMIC_UNCOLLECTABLE"; - break; -#endif - default: - kind_str = NULL; - /* The alternative is to use snprintf(buffer) but it is */ - /* not quite portable (see vsnprintf in misc.c). */ - } - } - - if (NULL != kind_str) { - GC_err_printf("%p (%s:%d," IF_NOT_SHORTDBG_HDRS(" sz= %lu,") " %s)\n", - (void *)((ptr_t)ohdr + sizeof(oh)), - ohdr->oh_string, GET_OH_LINENUM(ohdr) /*, */ - COMMA_IFNOT_SHORTDBG_HDRS((unsigned long)ohdr->oh_sz), - kind_str); - } else { - GC_err_printf("%p (%s:%d," IF_NOT_SHORTDBG_HDRS(" sz= %lu,") " kind= %d, descr= 0x%lx)\n", - (void *)((ptr_t)ohdr + sizeof(oh)), - ohdr->oh_string, GET_OH_LINENUM(ohdr) /*, */ - COMMA_IFNOT_SHORTDBG_HDRS((unsigned long)ohdr->oh_sz), - kind, (unsigned long)hhdr->hb_descr); - } - PRINT_CALL_CHAIN(ohdr); -} - -STATIC void GC_debug_print_heap_obj_proc(ptr_t p) { - GC_ASSERT(I_DONT_HOLD_LOCK()); - if (GC_HAS_DEBUG_INFO(p)) { - GC_print_obj(p); - } else { - GC_default_print_heap_obj_proc(p); - } -} - -#ifndef SHORT_DBG_HDRS -/* Use GC_err_printf and friends to print a description of the object */ -/* whose client-visible address is p, and which was smashed at */ -/* clobbered_addr. */ -STATIC void GC_print_smashed_obj(const char *msg, void *p, - ptr_t clobbered_addr) { - oh *ohdr = (oh *)GC_base(p); - - GC_ASSERT(I_DONT_HOLD_LOCK()); -#ifdef LINT2 - if (!ohdr) ABORT("Invalid GC_print_smashed_obj argument"); -#endif - if ((word)clobbered_addr <= (word)(&ohdr->oh_sz) || ohdr->oh_string == 0) { - GC_err_printf( - "%s %p in or near object at %p(, appr. sz= %lu)\n", - msg, (void *)clobbered_addr, p, - (unsigned long)(GC_size((ptr_t)ohdr) - DEBUG_BYTES)); - } else { - GC_err_printf("%s %p in or near object at %p (%s:%d, sz= %lu)\n", - msg, (void *)clobbered_addr, p, - (word)(ohdr->oh_string) < HBLKSIZE ? "(smashed string)" : ohdr->oh_string[0] == '\0' ? "EMPTY(smashed?)" - : ohdr->oh_string, - GET_OH_LINENUM(ohdr), (unsigned long)(ohdr->oh_sz)); - PRINT_CALL_CHAIN(ohdr); - } -} - -STATIC void GC_check_heap_proc(void); -STATIC void GC_print_all_smashed_proc(void); -#else -STATIC void GC_do_nothing(void) {} -#endif /* SHORT_DBG_HDRS */ - -GC_INNER void GC_start_debugging_inner(void) { - GC_ASSERT(I_HOLD_LOCK()); -#ifndef SHORT_DBG_HDRS - GC_check_heap = GC_check_heap_proc; - GC_print_all_smashed = GC_print_all_smashed_proc; -#else - GC_check_heap = GC_do_nothing; - GC_print_all_smashed = GC_do_nothing; -#endif - GC_print_heap_obj = GC_debug_print_heap_obj_proc; - GC_debugging_started = TRUE; - GC_register_displacement_inner((word)sizeof(oh)); -#if defined(CPPCHECK) - GC_noop1(GC_debug_header_size); -#endif -} - -const size_t GC_debug_header_size = sizeof(oh); - -GC_API size_t GC_CALL GC_get_debug_header_size(void) { - return sizeof(oh); -} - -GC_API void GC_CALL GC_debug_register_displacement(size_t offset) { - LOCK(); - GC_register_displacement_inner(offset); - GC_register_displacement_inner((word)sizeof(oh) + offset); - UNLOCK(); -} - -#ifdef GC_ADD_CALLER -#if defined(HAVE_DLADDR) && defined(GC_HAVE_RETURN_ADDR_PARENT) -#include - -STATIC void GC_caller_func_offset(word ad, const char **symp, int *offp) { - Dl_info caller; - - if (ad && dladdr((void *)ad, &caller) && caller.dli_sname != NULL) { - *symp = caller.dli_sname; - *offp = (int)((char *)ad - (char *)caller.dli_saddr); - } - if (NULL == *symp) { - *symp = "unknown"; - } -} -#else -#define GC_caller_func_offset(ad, symp, offp) (void)(*(symp) = "unknown") -#endif -#endif /* GC_ADD_CALLER */ - -GC_API GC_ATTR_MALLOC void *GC_CALL GC_debug_malloc(size_t lb, - GC_EXTRA_PARAMS) { - void *result; - - /* Note that according to malloc() specification, if size is 0 then */ - /* malloc() returns either NULL, or a unique pointer value that can */ - /* later be successfully passed to free(). We always do the latter. */ -#if defined(_FORTIFY_SOURCE) && !defined(__clang__) - /* Workaround to avoid "exceeds maximum object size" gcc warning. */ - result = GC_malloc(lb < GC_SIZE_MAX - DEBUG_BYTES ? lb + DEBUG_BYTES - : GC_SIZE_MAX >> 1); -#else - result = GC_malloc(SIZET_SAT_ADD(lb, DEBUG_BYTES)); -#endif -#ifdef GC_ADD_CALLER - if (s == NULL) { - GC_caller_func_offset(ra, &s, &i); - } -#endif - return store_debug_info(result, lb, "GC_debug_malloc", OPT_RA s, i); -} - -GC_API GC_ATTR_MALLOC void *GC_CALL -GC_debug_malloc_ignore_off_page(size_t lb, GC_EXTRA_PARAMS) { - void *result = GC_malloc_ignore_off_page(SIZET_SAT_ADD(lb, DEBUG_BYTES)); - - return store_debug_info(result, lb, "GC_debug_malloc_ignore_off_page", - OPT_RA s, i); -} - -GC_API GC_ATTR_MALLOC void *GC_CALL -GC_debug_malloc_atomic_ignore_off_page(size_t lb, GC_EXTRA_PARAMS) { - void *result = GC_malloc_atomic_ignore_off_page( - SIZET_SAT_ADD(lb, DEBUG_BYTES)); - - return store_debug_info(result, lb, - "GC_debug_malloc_atomic_ignore_off_page", - OPT_RA s, i); -} - -STATIC void *GC_debug_generic_malloc(size_t lb, int k, GC_EXTRA_PARAMS) { - void *result = GC_generic_malloc_aligned(SIZET_SAT_ADD(lb, DEBUG_BYTES), - k, 0 /* flags */, - 0 /* align_m1 */); - - return store_debug_info(result, lb, "GC_debug_generic_malloc", - OPT_RA s, i); -} - -#ifdef DBG_HDRS_ALL -/* An allocation function for internal use. Normally internally */ -/* allocated objects do not have debug information. But in this */ -/* case, we need to make sure that all objects have debug headers. */ -GC_INNER void *GC_debug_generic_malloc_inner(size_t lb, int k, - unsigned flags) { - void *result; - - GC_ASSERT(I_HOLD_LOCK()); - result = GC_generic_malloc_inner(SIZET_SAT_ADD(lb, DEBUG_BYTES), k, flags); - if (NULL == result) { - GC_err_printf("GC internal allocation (%lu bytes) returning NULL\n", - (unsigned long)lb); - return NULL; - } - if (!GC_debugging_started) { - GC_start_debugging_inner(); - } - ADD_CALL_CHAIN(result, GC_RETURN_ADDR); - return GC_store_debug_info_inner(result, (word)lb, "INTERNAL", 0); -} -#endif /* DBG_HDRS_ALL */ - -#ifndef CPPCHECK -GC_API void *GC_CALL GC_debug_malloc_stubborn(size_t lb, GC_EXTRA_PARAMS) { - return GC_debug_malloc(lb, OPT_RA s, i); -} - -GC_API void GC_CALL GC_debug_change_stubborn(const void *p) { - UNUSED_ARG(p); -} -#endif /* !CPPCHECK */ - -GC_API void GC_CALL GC_debug_end_stubborn_change(const void *p) { - const void *q = GC_base_C(p); - - if (NULL == q) { - ABORT_ARG1("GC_debug_end_stubborn_change: bad arg", ": %p", p); - } - GC_end_stubborn_change(q); -} - -GC_API void GC_CALL GC_debug_ptr_store_and_dirty(void *p, const void *q) { - *(void **)GC_is_visible(p) = GC_is_valid_displacement((void *)q); - GC_debug_end_stubborn_change(p); - REACHABLE_AFTER_DIRTY(q); -} - -GC_API GC_ATTR_MALLOC void *GC_CALL GC_debug_malloc_atomic(size_t lb, - GC_EXTRA_PARAMS) { - void *result = GC_malloc_atomic(SIZET_SAT_ADD(lb, DEBUG_BYTES)); - - return store_debug_info(result, lb, "GC_debug_malloc_atomic", - OPT_RA s, i); -} - -GC_API GC_ATTR_MALLOC char *GC_CALL GC_debug_strdup(const char *str, - GC_EXTRA_PARAMS) { - char *copy; - size_t lb; - if (str == NULL) { - if (GC_find_leak) - GC_err_printf("strdup(NULL) behavior is undefined\n"); - return NULL; - } - - lb = strlen(str) + 1; - copy = (char *)GC_debug_malloc_atomic(lb, OPT_RA s, i); - if (copy == NULL) { -#ifndef MSWINCE - errno = ENOMEM; -#endif - return NULL; - } - BCOPY(str, copy, lb); - return copy; -} - -GC_API GC_ATTR_MALLOC char *GC_CALL GC_debug_strndup(const char *str, - size_t size, GC_EXTRA_PARAMS) { - char *copy; - size_t len = strlen(str); /* str is expected to be non-NULL */ - if (len > size) - len = size; - copy = (char *)GC_debug_malloc_atomic(len + 1, OPT_RA s, i); - if (copy == NULL) { -#ifndef MSWINCE - errno = ENOMEM; -#endif - return NULL; - } - if (len > 0) - BCOPY(str, copy, len); - copy[len] = '\0'; - return copy; -} - -#ifdef GC_REQUIRE_WCSDUP -#include /* for wcslen() */ - -GC_API GC_ATTR_MALLOC wchar_t *GC_CALL GC_debug_wcsdup(const wchar_t *str, - GC_EXTRA_PARAMS) { - size_t lb = (wcslen(str) + 1) * sizeof(wchar_t); - wchar_t *copy = (wchar_t *)GC_debug_malloc_atomic(lb, OPT_RA s, i); - if (copy == NULL) { -#ifndef MSWINCE - errno = ENOMEM; -#endif - return NULL; - } - BCOPY(str, copy, lb); - return copy; -} -#endif /* GC_REQUIRE_WCSDUP */ - -GC_API GC_ATTR_MALLOC void *GC_CALL GC_debug_malloc_uncollectable(size_t lb, - GC_EXTRA_PARAMS) { - void *result = GC_malloc_uncollectable( - SIZET_SAT_ADD(lb, UNCOLLECTABLE_DEBUG_BYTES)); - - return store_debug_info(result, lb, "GC_debug_malloc_uncollectable", - OPT_RA s, i); -} - -#ifdef GC_ATOMIC_UNCOLLECTABLE -GC_API GC_ATTR_MALLOC void *GC_CALL -GC_debug_malloc_atomic_uncollectable(size_t lb, GC_EXTRA_PARAMS) { - void *result = GC_malloc_atomic_uncollectable( - SIZET_SAT_ADD(lb, UNCOLLECTABLE_DEBUG_BYTES)); - - return store_debug_info(result, lb, - "GC_debug_malloc_atomic_uncollectable", - OPT_RA s, i); -} -#endif /* GC_ATOMIC_UNCOLLECTABLE */ - -#ifndef GC_FREED_MEM_MARKER -#if CPP_WORDSZ == 32 -#define GC_FREED_MEM_MARKER 0xdeadbeef -#else -#define GC_FREED_MEM_MARKER GC_WORD_C(0xEFBEADDEdeadbeef) -#endif -#endif - -#ifdef LINT2 -#ifdef _WIN32 -#define WIN32_LEAN_AND_MEAN -#include -#endif -#include "private/gc_alloc_ptrs.h" -#endif - -GC_API void GC_CALL GC_debug_free(void *p) { - ptr_t base; - if (0 == p) return; - - base = (ptr_t)GC_base(p); - if (NULL == base) { -#if defined(REDIRECT_MALLOC) && ((defined(NEED_CALLINFO) && defined(GC_HAVE_BUILTIN_BACKTRACE)) || defined(GC_LINUX_THREADS) || defined(GC_SOLARIS_THREADS) || defined(MSWIN32)) - /* In some cases, we should ignore objects that do not belong */ - /* to the GC heap. See the comment in GC_free. */ - if (!GC_is_heap_ptr(p)) return; -#endif - ABORT_ARG1("Invalid pointer passed to free()", ": %p", p); - } - if ((ptr_t)p - (ptr_t)base != sizeof(oh)) { -#if defined(REDIRECT_FREE) && defined(USE_PROC_FOR_LIBRARIES) - /* TODO: Suppress the warning if free() caller is in libpthread */ - /* or libdl. */ -#endif - /* TODO: Suppress the warning for objects allocated by */ - /* GC_memalign and friends (these ones do not have the debugging */ - /* counterpart). */ - GC_err_printf( - "GC_debug_free called on pointer %p w/o debugging info\n", p); - } else { -#ifndef SHORT_DBG_HDRS - ptr_t clobbered = GC_check_annotated_obj((oh *)base); - word sz = GC_size(base); - if (clobbered != 0) { - GC_SET_HAVE_ERRORS(); /* no "release" barrier is needed */ - if (((oh *)base)->oh_sz == sz) { - GC_print_smashed_obj( - "GC_debug_free: found previously deallocated (?) object at", - p, clobbered); - return; /* ignore double free */ - } else { - GC_print_smashed_obj("GC_debug_free: found smashed location at", - p, clobbered); - } - } - /* Invalidate size (mark the object as deallocated) */ - ((oh *)base)->oh_sz = sz; -#endif /* !SHORT_DBG_HDRS */ - } - if (GC_find_leak -#ifndef SHORT_DBG_HDRS - && ((ptr_t)p - (ptr_t)base != sizeof(oh) || !GC_findleak_delay_free) -#endif - ) { - GC_free(base); - } else { - hdr *hhdr = HDR(p); - if (hhdr->hb_obj_kind == UNCOLLECTABLE -#ifdef GC_ATOMIC_UNCOLLECTABLE - || hhdr->hb_obj_kind == AUNCOLLECTABLE -#endif - ) { - GC_free(base); - } else { - word i; - word sz = hhdr->hb_sz; - word obj_sz = BYTES_TO_WORDS(sz - sizeof(oh)); - - for (i = 0; i < obj_sz; ++i) - ((word *)p)[i] = GC_FREED_MEM_MARKER; - GC_ASSERT((word *)p + i == (word *)(base + sz)); - /* Update the counter even though the real deallocation */ - /* is deferred. */ - LOCK(); -#ifdef LINT2 - GC_incr_bytes_freed((size_t)sz); -#else - GC_bytes_freed += sz; -#endif - UNLOCK(); - } - } /* !GC_find_leak */ -} - -#if defined(THREADS) && defined(DBG_HDRS_ALL) -/* Used internally; we assume it's called correctly. */ -GC_INNER void GC_debug_free_inner(void *p) { - ptr_t base = (ptr_t)GC_base(p); - GC_ASSERT((ptr_t)p - (ptr_t)base == sizeof(oh)); -#ifdef LINT2 - if (!base) ABORT("Invalid GC_debug_free_inner argument"); -#endif -#ifndef SHORT_DBG_HDRS - /* Invalidate size */ - ((oh *)base)->oh_sz = GC_size(base); -#endif - GC_free_inner(base); -} -#endif - -GC_API void *GC_CALL GC_debug_realloc(void *p, size_t lb, GC_EXTRA_PARAMS) { - void *base; - void *result; - hdr *hhdr; - - if (p == 0) { - return GC_debug_malloc(lb, OPT_RA s, i); - } - if (0 == lb) /* and p != NULL */ { - GC_debug_free(p); - return NULL; - } - -#ifdef GC_ADD_CALLER - if (s == NULL) { - GC_caller_func_offset(ra, &s, &i); - } -#endif - base = GC_base(p); - if (base == 0) { - ABORT_ARG1("Invalid pointer passed to realloc()", ": %p", p); - } - if ((ptr_t)p - (ptr_t)base != sizeof(oh)) { - GC_err_printf( - "GC_debug_realloc called on pointer %p w/o debugging info\n", p); - return GC_realloc(p, lb); - } - hhdr = HDR(base); - switch (hhdr->hb_obj_kind) { - case NORMAL: - result = GC_debug_malloc(lb, OPT_RA s, i); - break; - case PTRFREE: - result = GC_debug_malloc_atomic(lb, OPT_RA s, i); - break; - case UNCOLLECTABLE: - result = GC_debug_malloc_uncollectable(lb, OPT_RA s, i); - break; -#ifdef GC_ATOMIC_UNCOLLECTABLE - case AUNCOLLECTABLE: - result = GC_debug_malloc_atomic_uncollectable(lb, OPT_RA s, i); - break; -#endif - default: - result = NULL; /* initialized to prevent warning. */ - ABORT_RET("GC_debug_realloc: encountered bad kind"); - } - - if (result != NULL) { - size_t old_sz; -#ifdef SHORT_DBG_HDRS - old_sz = GC_size(base) - sizeof(oh); -#else - old_sz = ((oh *)base)->oh_sz; -#endif - if (old_sz > 0) - BCOPY(p, result, old_sz < lb ? old_sz : lb); - GC_debug_free(p); - } - return result; -} - -GC_API GC_ATTR_MALLOC void *GC_CALL -GC_debug_generic_or_special_malloc(size_t lb, int k, GC_EXTRA_PARAMS) { - switch (k) { - case PTRFREE: - return GC_debug_malloc_atomic(lb, OPT_RA s, i); - case NORMAL: - return GC_debug_malloc(lb, OPT_RA s, i); - case UNCOLLECTABLE: - return GC_debug_malloc_uncollectable(lb, OPT_RA s, i); -#ifdef GC_ATOMIC_UNCOLLECTABLE - case AUNCOLLECTABLE: - return GC_debug_malloc_atomic_uncollectable(lb, OPT_RA s, i); -#endif - default: - return GC_debug_generic_malloc(lb, k, OPT_RA s, i); - } -} - -#ifndef SHORT_DBG_HDRS - -/* List of smashed (clobbered) locations. We defer printing these, */ -/* since we can't always print them nicely with the allocation lock */ -/* held. We put them here instead of in GC_arrays, since it may be */ -/* useful to be able to look at them with the debugger. */ -#ifndef MAX_SMASHED -#define MAX_SMASHED 20 -#endif -STATIC ptr_t GC_smashed[MAX_SMASHED] = {0}; -STATIC unsigned GC_n_smashed = 0; - -STATIC void GC_add_smashed(ptr_t smashed) { - GC_ASSERT(I_HOLD_LOCK()); - GC_ASSERT(GC_is_marked(GC_base(smashed))); - /* FIXME: Prevent adding an object while printing smashed list. */ - GC_smashed[GC_n_smashed] = smashed; - if (GC_n_smashed < MAX_SMASHED - 1) ++GC_n_smashed; - /* In case of overflow, we keep the first MAX_SMASHED-1 */ - /* entries plus the last one. */ - GC_SET_HAVE_ERRORS(); -} - -/* Print all objects on the list. Clear the list. */ -STATIC void GC_print_all_smashed_proc(void) { - unsigned i; - - GC_ASSERT(I_DONT_HOLD_LOCK()); - if (GC_n_smashed == 0) return; - GC_err_printf("GC_check_heap_block: found %u smashed heap objects:\n", - GC_n_smashed); - for (i = 0; i < GC_n_smashed; ++i) { - ptr_t base = (ptr_t)GC_base(GC_smashed[i]); - -#ifdef LINT2 - if (!base) ABORT("Invalid GC_smashed element"); -#endif - GC_print_smashed_obj("", base + sizeof(oh), GC_smashed[i]); - GC_smashed[i] = 0; - } - GC_n_smashed = 0; -} - -/* Check all marked objects in the given block for validity */ -/* Avoid GC_apply_to_each_object for performance reasons. */ -STATIC void GC_CALLBACK GC_check_heap_block(struct hblk *hbp, GC_word dummy) { - struct hblkhdr *hhdr = HDR(hbp); - word sz = hhdr->hb_sz; - word bit_no; - char *p, *plim; - - UNUSED_ARG(dummy); - p = hbp->hb_body; - if (sz > MAXOBJBYTES) { - plim = p; - } else { - plim = hbp->hb_body + HBLKSIZE - sz; - } - /* go through all words in block */ - for (bit_no = 0; (word)p <= (word)plim; - bit_no += MARK_BIT_OFFSET(sz), p += sz) { - if (mark_bit_from_hdr(hhdr, bit_no) && GC_HAS_DEBUG_INFO((ptr_t)p)) { - ptr_t clobbered = GC_check_annotated_obj((oh *)p); - if (clobbered != 0) - GC_add_smashed(clobbered); - } - } -} - -/* This assumes that all accessible objects are marked. */ -/* Normally called by collector. */ -STATIC void GC_check_heap_proc(void) { - GC_ASSERT(I_HOLD_LOCK()); - GC_STATIC_ASSERT((sizeof(oh) & (GRANULE_BYTES - 1)) == 0); - /* FIXME: Should we check for twice that alignment? */ - GC_apply_to_all_blocks(GC_check_heap_block, 0); -} - -GC_INNER GC_bool GC_check_leaked(ptr_t base) { - word i; - word obj_sz; - word *p; - - if ( -#if defined(KEEP_BACK_PTRS) || defined(MAKE_BACK_GRAPH) - (*(word *)base & 1) != 0 && -#endif - GC_has_other_debug_info(base) >= 0) - return TRUE; /* object has leaked */ - - /* Validate freed object's content. */ - p = (word *)(base + sizeof(oh)); - obj_sz = BYTES_TO_WORDS(HDR(base)->hb_sz - sizeof(oh)); - for (i = 0; i < obj_sz; ++i) - if (p[i] != GC_FREED_MEM_MARKER) { - GC_set_mark_bit(base); /* do not reclaim it in this cycle */ - GC_add_smashed((ptr_t)(&p[i])); /* alter-after-free detected */ - break; /* don't report any other smashed locations in the object */ - } - - return FALSE; /* GC_debug_free() has been called */ -} - -#endif /* !SHORT_DBG_HDRS */ - -#ifndef GC_NO_FINALIZATION - -struct closure { - GC_finalization_proc cl_fn; - void *cl_data; -}; - -STATIC void *GC_make_closure(GC_finalization_proc fn, void *data) { - struct closure *result = -#ifdef DBG_HDRS_ALL - (struct closure *)GC_debug_malloc(sizeof(struct closure), - GC_EXTRAS); -#else - (struct closure *)GC_malloc(sizeof(struct closure)); -#endif - if (result != 0) { - result->cl_fn = fn; - result->cl_data = data; - } - return (void *)result; -} - -/* An auxiliary fns to make finalization work correctly with displaced */ -/* pointers introduced by the debugging allocators. */ -STATIC void GC_CALLBACK GC_debug_invoke_finalizer(void *obj, void *data) { - struct closure *cl = (struct closure *)data; - (*(cl->cl_fn))((void *)((char *)obj + sizeof(oh)), cl->cl_data); -} - -/* Special finalizer_proc value to detect GC_register_finalizer() failure. */ -#define OFN_UNSET ((GC_finalization_proc) ~(signed_word)0) - -/* Set ofn and ocd to reflect the values we got back. */ -static void store_old(void *obj, GC_finalization_proc my_old_fn, - struct closure *my_old_cd, GC_finalization_proc *ofn, - void **ocd) { - if (0 != my_old_fn) { - if (my_old_fn == OFN_UNSET) { - /* GC_register_finalizer() failed; (*ofn) and (*ocd) are unchanged. */ - return; - } - if (my_old_fn != GC_debug_invoke_finalizer) { - GC_err_printf("Debuggable object at %p had a non-debug finalizer\n", - obj); - /* This should probably be fatal. */ - } else { - if (ofn) *ofn = my_old_cd->cl_fn; - if (ocd) *ocd = my_old_cd->cl_data; - } - } else { - if (ofn) *ofn = 0; - if (ocd) *ocd = 0; - } -} - -GC_API void GC_CALL GC_debug_register_finalizer(void *obj, - GC_finalization_proc fn, - void *cd, GC_finalization_proc *ofn, - void **ocd) { - GC_finalization_proc my_old_fn = OFN_UNSET; - void *my_old_cd = NULL; /* to avoid "might be uninitialized" warning */ - ptr_t base = (ptr_t)GC_base(obj); - if (NULL == base) { - /* We won't collect it, hence finalizer wouldn't be run. */ - if (ocd) *ocd = 0; - if (ofn) *ofn = 0; - return; - } - if ((ptr_t)obj - base != sizeof(oh)) { - GC_err_printf( - "GC_debug_register_finalizer called with" - " non-base-pointer %p\n", - obj); - } - if (0 == fn) { - GC_register_finalizer(base, 0, 0, &my_old_fn, &my_old_cd); - } else { - cd = GC_make_closure(fn, cd); - if (cd == 0) return; /* out of memory; *ofn and *ocd are unchanged */ - GC_register_finalizer(base, GC_debug_invoke_finalizer, - cd, &my_old_fn, &my_old_cd); - } - store_old(obj, my_old_fn, (struct closure *)my_old_cd, ofn, ocd); -} - -GC_API void GC_CALL GC_debug_register_finalizer_no_order(void *obj, GC_finalization_proc fn, - void *cd, GC_finalization_proc *ofn, - void **ocd) { - GC_finalization_proc my_old_fn = OFN_UNSET; - void *my_old_cd = NULL; - ptr_t base = (ptr_t)GC_base(obj); - if (NULL == base) { - /* We won't collect it, hence finalizer wouldn't be run. */ - if (ocd) *ocd = 0; - if (ofn) *ofn = 0; - return; - } - if ((ptr_t)obj - base != sizeof(oh)) { - GC_err_printf( - "GC_debug_register_finalizer_no_order called with" - " non-base-pointer %p\n", - obj); - } - if (0 == fn) { - GC_register_finalizer_no_order(base, 0, 0, &my_old_fn, &my_old_cd); - } else { - cd = GC_make_closure(fn, cd); - if (cd == 0) return; /* out of memory */ - GC_register_finalizer_no_order(base, GC_debug_invoke_finalizer, - cd, &my_old_fn, &my_old_cd); - } - store_old(obj, my_old_fn, (struct closure *)my_old_cd, ofn, ocd); -} - -GC_API void GC_CALL GC_debug_register_finalizer_unreachable(void *obj, GC_finalization_proc fn, - void *cd, GC_finalization_proc *ofn, - void **ocd) { - GC_finalization_proc my_old_fn = OFN_UNSET; - void *my_old_cd = NULL; - ptr_t base = (ptr_t)GC_base(obj); - if (NULL == base) { - /* We won't collect it, hence finalizer wouldn't be run. */ - if (ocd) *ocd = 0; - if (ofn) *ofn = 0; - return; - } - if ((ptr_t)obj - base != sizeof(oh)) { - GC_err_printf( - "GC_debug_register_finalizer_unreachable called with" - " non-base-pointer %p\n", - obj); - } - if (0 == fn) { - GC_register_finalizer_unreachable(base, 0, 0, &my_old_fn, &my_old_cd); - } else { - cd = GC_make_closure(fn, cd); - if (cd == 0) return; /* out of memory */ - GC_register_finalizer_unreachable(base, GC_debug_invoke_finalizer, - cd, &my_old_fn, &my_old_cd); - } - store_old(obj, my_old_fn, (struct closure *)my_old_cd, ofn, ocd); -} - -GC_API void GC_CALL GC_debug_register_finalizer_ignore_self(void *obj, GC_finalization_proc fn, - void *cd, GC_finalization_proc *ofn, - void **ocd) { - GC_finalization_proc my_old_fn = OFN_UNSET; - void *my_old_cd = NULL; - ptr_t base = (ptr_t)GC_base(obj); - if (NULL == base) { - /* We won't collect it, hence finalizer wouldn't be run. */ - if (ocd) *ocd = 0; - if (ofn) *ofn = 0; - return; - } - if ((ptr_t)obj - base != sizeof(oh)) { - GC_err_printf( - "GC_debug_register_finalizer_ignore_self called with" - " non-base-pointer %p\n", - obj); - } - if (0 == fn) { - GC_register_finalizer_ignore_self(base, 0, 0, &my_old_fn, &my_old_cd); - } else { - cd = GC_make_closure(fn, cd); - if (cd == 0) return; /* out of memory */ - GC_register_finalizer_ignore_self(base, GC_debug_invoke_finalizer, - cd, &my_old_fn, &my_old_cd); - } - store_old(obj, my_old_fn, (struct closure *)my_old_cd, ofn, ocd); -} - -#endif /* !GC_NO_FINALIZATION */ - -GC_API GC_ATTR_MALLOC void *GC_CALL GC_debug_malloc_replacement(size_t lb) { - return GC_debug_malloc(lb, GC_DBG_EXTRAS); -} - -GC_API void *GC_CALL GC_debug_realloc_replacement(void *p, size_t lb) { - return GC_debug_realloc(p, lb, GC_DBG_EXTRAS); -} diff --git a/vendor/bdwgc/dyn_load.c b/vendor/bdwgc/dyn_load.c deleted file mode 100644 index 7d3d9cdf..00000000 --- a/vendor/bdwgc/dyn_load.c +++ /dev/null @@ -1,1516 +0,0 @@ -/* - * Copyright (c) 1991-1994 by Xerox Corporation. All rights reserved. - * Copyright (c) 1997 by Silicon Graphics. All rights reserved. - * Copyright (c) 2009-2022 Ivan Maidanski - * - * THIS MATERIAL IS PROVIDED AS IS, WITH ABSOLUTELY NO WARRANTY EXPRESSED - * OR IMPLIED. ANY USE IS AT YOUR OWN RISK. - * - * Permission is hereby granted to use or copy this program - * for any purpose, provided the above notices are retained on all copies. - * Permission to modify the code and to distribute modified code is granted, - * provided the above notices are retained, and a notice that the code was - * modified is included with the above copyright notice. - */ - -#ifdef _WIN32 -#define WIN32_LEAN_AND_MEAN -#include -#endif -#include "private/gc_priv.h" - -/* - * This is incredibly OS specific code for tracking down data sections in - * dynamic libraries. There appears to be no way of doing this quickly - * without groveling through undocumented data structures. We would argue - * that this is a bug in the design of the dlopen interface. THIS CODE - * MAY BREAK IN FUTURE OS RELEASES. If this matters to you, don't hesitate - * to let your vendor know ... - * - * None of this is safe with dlclose and incremental collection. - * But then not much of anything is safe in the presence of dlclose. - */ - -#if !defined(MACOS) && !defined(GC_NO_TYPES) && !defined(SN_TARGET_PSP2) && !defined(_WIN32_WCE) && !defined(__CC_ARM) -#include -#endif - -/* BTL: avoid circular redefinition of dlopen if GC_SOLARIS_THREADS defined */ -#undef GC_MUST_RESTORE_REDEFINED_DLOPEN -#if defined(GC_PTHREADS) && !defined(GC_NO_DLOPEN) && !defined(GC_NO_THREAD_REDIRECTS) && !defined(GC_USE_LD_WRAP) -/* To support threads in Solaris, gc.h interposes on dlopen by */ -/* defining "dlopen" to be "GC_dlopen", which is implemented below. */ -/* However, both GC_FirstDLOpenedLinkMap() and GC_dlopen() use the */ -/* real system dlopen() in their implementation. We first remove */ -/* gc.h's dlopen definition and restore it later, after GC_dlopen(). */ -#undef dlopen -#define GC_MUST_RESTORE_REDEFINED_DLOPEN -#endif /* !GC_NO_DLOPEN */ - -/* A user-supplied routine (custom filter) that might be called to */ -/* determine whether a DSO really needs to be scanned by the GC. */ -/* 0 means no filter installed. May be unused on some platforms. */ -/* FIXME: Add filter support for more platforms. */ -STATIC GC_has_static_roots_func GC_has_static_roots = 0; - -#if (defined(DYNAMIC_LOADING) || defined(MSWIN32) || defined(MSWINCE) || defined(CYGWIN32)) && !defined(PCR) - -#if !defined(DARWIN) && !defined(SCO_ELF) && !defined(SOLARISDL) && !defined(AIX) && !defined(DGUX) && !defined(IRIX5) && !defined(HPUX) && !defined(CYGWIN32) && !defined(MSWIN32) && !defined(MSWINCE) && !(defined(ALPHA) && defined(OSF1)) && !(defined(FREEBSD) && defined(__ELF__)) && !(defined(LINUX) && defined(__ELF__)) && !(defined(NETBSD) && defined(__ELF__)) && !(defined(OPENBSD) && (defined(__ELF__) || defined(M68K))) && !defined(HAIKU) && !defined(HURD) && !defined(NACL) && !defined(CPPCHECK) -#error We only know how to find data segments of dynamic libraries for above. -#error Additional SVR4 variants might not be too hard to add. -#endif - -#ifdef SOLARISDL -#include -#include -#include -#endif - -#if defined(NETBSD) -#include -#include -#include -#define ELFSIZE ARCH_ELFSIZE -#endif - -#if defined(OPENBSD) -#include -#if (OpenBSD >= 200519) && !defined(HAVE_DL_ITERATE_PHDR) -#define HAVE_DL_ITERATE_PHDR -#endif -#endif /* OPENBSD */ - -#if defined(SCO_ELF) || defined(DGUX) || defined(HURD) || defined(NACL) || (defined(__ELF__) && (defined(LINUX) || defined(FREEBSD) || defined(NETBSD) || defined(OPENBSD))) -#include -#if !defined(OPENBSD) && !defined(HOST_ANDROID) -/* OpenBSD does not have elf.h file; link.h below is sufficient. */ -/* Exclude Android because linker.h below includes its own version. */ -#include -#endif -#ifdef HOST_ANDROID -/* If you don't need the "dynamic loading" feature, you may build */ -/* the collector with -D IGNORE_DYNAMIC_LOADING. */ -#ifdef BIONIC_ELFDATA_REDEF_BUG -/* Workaround a problem in Bionic (as of Android 4.2) which has */ -/* mismatching ELF_DATA definitions in sys/exec_elf.h and */ -/* asm/elf.h included from linker.h file (similar to EM_ALPHA). */ -#include -#include -#undef ELF_DATA -#undef EM_ALPHA -#endif -#include -#if !defined(GC_DONT_DEFINE_LINK_MAP) && !(__ANDROID_API__ >= 21) -/* link_map and r_debug are defined in link.h of NDK r10+. */ -/* bionic/linker/linker.h defines them too but the header */ -/* itself is a C++ one starting from Android 4.3. */ -struct link_map { - uintptr_t l_addr; - char *l_name; - uintptr_t l_ld; - struct link_map *l_next; - struct link_map *l_prev; -}; -struct r_debug { - int32_t r_version; - struct link_map *r_map; - void (*r_brk)(void); - int32_t r_state; - uintptr_t r_ldbase; -}; -#endif -#else -EXTERN_C_BEGIN /* Workaround missing extern "C" around _DYNAMIC */ - /* symbol in link.h of some Linux hosts. */ -#include - EXTERN_C_END -#endif -#endif - -/* Newer versions of GNU/Linux define this macro. We - * define it similarly for any ELF systems that don't. */ -#ifndef ElfW -#if defined(FREEBSD) -#if __ELF_WORD_SIZE == 32 -#define ElfW(type) Elf32_##type -#else -#define ElfW(type) Elf64_##type -#endif -#elif defined(NETBSD) || defined(OPENBSD) -#if ELFSIZE == 32 -#define ElfW(type) Elf32_##type -#elif ELFSIZE == 64 -#define ElfW(type) Elf64_##type -#else -#error Missing ELFSIZE define -#endif -#else -#if !defined(ELF_CLASS) || ELF_CLASS == ELFCLASS32 -#define ElfW(type) Elf32_##type -#else -#define ElfW(type) Elf64_##type -#endif -#endif -#endif - -#if defined(SOLARISDL) && !defined(USE_PROC_FOR_LIBRARIES) - -EXTERN_C_BEGIN -extern ElfW(Dyn) _DYNAMIC; -EXTERN_C_END - -STATIC struct link_map * -GC_FirstDLOpenedLinkMap(void) { - ElfW(Dyn) * dp; - static struct link_map *cachedResult = 0; - static ElfW(Dyn) *dynStructureAddr = 0; - /* BTL: added to avoid Solaris 5.3 ld.so _DYNAMIC bug */ - -#ifdef SUNOS53_SHARED_LIB - /* BTL: Avoid the Solaris 5.3 bug that _DYNAMIC isn't being set */ - /* up properly in dynamically linked .so's. This means we have */ - /* to use its value in the set of original object files loaded */ - /* at program startup. */ - if (dynStructureAddr == 0) { - void *startupSyms = dlopen(0, RTLD_LAZY); - dynStructureAddr = (ElfW(Dyn) *)(word)dlsym(startupSyms, "_DYNAMIC"); - } -#else - dynStructureAddr = &_DYNAMIC; -#endif - - if (0 == COVERT_DATAFLOW(dynStructureAddr)) { - /* _DYNAMIC symbol not resolved. */ - return NULL; - } - if (cachedResult == 0) { - int tag; - for (dp = ((ElfW(Dyn) *)(&_DYNAMIC)); (tag = dp->d_tag) != 0; dp++) { - if (tag == DT_DEBUG) { - struct r_debug *rd = (struct r_debug *)dp->d_un.d_ptr; - if (rd != NULL) { - struct link_map *lm = rd->r_map; - if (lm != NULL) - cachedResult = lm->l_next; /* might be NULL */ - } - break; - } - } - } - return cachedResult; -} - -#endif /* SOLARISDL ... */ - -/* BTL: added to fix circular dlopen definition if GC_SOLARIS_THREADS defined */ -#ifdef GC_MUST_RESTORE_REDEFINED_DLOPEN -#define dlopen GC_dlopen -#endif - -#if defined(SOLARISDL) - -/* Add dynamic library data sections to the root set. */ -#if !defined(PCR) && !defined(GC_SOLARIS_THREADS) && defined(THREADS) && !defined(CPPCHECK) -#error Fix mutual exclusion with dlopen -#endif - -#ifndef USE_PROC_FOR_LIBRARIES -GC_INNER void GC_register_dynamic_libraries(void) { - struct link_map *lm; - - GC_ASSERT(I_HOLD_LOCK()); - for (lm = GC_FirstDLOpenedLinkMap(); lm != 0; lm = lm->l_next) { - ElfW(Ehdr) * e; - ElfW(Phdr) * p; - unsigned long offset; - char *start; - int i; - - e = (ElfW(Ehdr) *)lm->l_addr; - p = ((ElfW(Phdr) *)(((char *)(e)) + e->e_phoff)); - offset = ((unsigned long)(lm->l_addr)); - for (i = 0; i < (int)e->e_phnum; i++, p++) { - switch (p->p_type) { - case PT_LOAD: { - if (!(p->p_flags & PF_W)) break; - start = ((char *)(p->p_vaddr)) + offset; - GC_add_roots_inner(start, start + p->p_memsz, TRUE); - } break; - default: - break; - } - } - } -} - -#endif /* !USE_PROC_FOR_LIBRARIES */ -#endif /* SOLARISDL */ - -#if defined(SCO_ELF) || defined(DGUX) || defined(HURD) || defined(NACL) || (defined(__ELF__) && (defined(LINUX) || defined(FREEBSD) || defined(NETBSD) || defined(OPENBSD))) - -#ifdef USE_PROC_FOR_LIBRARIES - -#include -#include -#include -#include - -#define MAPS_BUF_SIZE (32 * 1024) - -/* Sort an array of HeapSects by start address. */ -/* Unfortunately at least some versions of */ -/* Linux qsort end up calling malloc by way of sysconf, and hence can't */ -/* be used in the collector. Hence we roll our own. Should be */ -/* reasonably fast if the array is already mostly sorted, as we expect */ -/* it to be. */ -static void sort_heap_sects(struct HeapSect *base, size_t number_of_elements) { - signed_word n = (signed_word)number_of_elements; - signed_word nsorted = 1; - - while (nsorted < n) { - signed_word i; - - while (nsorted < n && - (word)base[nsorted - 1].hs_start < (word)base[nsorted].hs_start) - ++nsorted; - if (nsorted == n) break; - GC_ASSERT((word)base[nsorted - 1].hs_start > (word)base[nsorted].hs_start); - i = nsorted - 1; - while (i >= 0 && (word)base[i].hs_start > (word)base[i + 1].hs_start) { - struct HeapSect tmp = base[i]; - base[i] = base[i + 1]; - base[i + 1] = tmp; - --i; - } - GC_ASSERT((word)base[nsorted - 1].hs_start < (word)base[nsorted].hs_start); - ++nsorted; - } -} - -STATIC void GC_register_map_entries(const char *maps) { - const char *prot, *path; - ptr_t start, end; - unsigned int maj_dev; - ptr_t least_ha, greatest_ha; - unsigned i; - - GC_ASSERT(I_HOLD_LOCK()); - sort_heap_sects(GC_our_memory, GC_n_memory); - least_ha = GC_our_memory[0].hs_start; - greatest_ha = GC_our_memory[GC_n_memory - 1].hs_start + GC_our_memory[GC_n_memory - 1].hs_bytes; - - for (;;) { - maps = GC_parse_map_entry(maps, &start, &end, &prot, &maj_dev, &path); - if (NULL == maps) break; - - if (prot[1] == 'w') { - /* This is a writable mapping. Add it to */ - /* the root set unless it is already otherwise */ - /* accounted for. */ -#ifndef THREADS - if ((word)start <= (word)GC_stackbottom && (word)end >= (word)GC_stackbottom) { - /* Stack mapping; discard */ - continue; - } -#endif -#if defined(E2K) && defined(__ptr64__) - /* TODO: avoid hard-coded addresses */ - if ((word)start == 0xc2fffffff000UL && (word)end == 0xc30000000000UL && path[0] == '\n') - continue; /* discard some special mapping */ -#endif - if (path[0] == '[' && strncmp(path + 1, "heap]", 5) != 0) - continue; /* discard if a pseudo-path unless "[heap]" */ - -#ifdef THREADS - /* This may fail, since a thread may already be */ - /* unregistered, but its thread stack may still be there. */ - /* That can fail because the stack may disappear while */ - /* we're marking. Thus the marker is, and has to be */ - /* prepared to recover from segmentation faults. */ - - if (GC_segment_is_thread_stack(start, end)) continue; - - /* FIXME: NPTL squirrels */ - /* away pointers in pieces of the stack segment that we */ - /* don't scan. We work around this */ - /* by treating anything allocated by libpthread as */ - /* uncollectible, as we do in some other cases. */ - /* A specifically identified problem is that */ - /* thread stacks contain pointers to dynamic thread */ - /* vectors, which may be reused due to thread caching. */ - /* They may not be marked if the thread is still live. */ - /* This specific instance should be addressed by */ - /* INCLUDE_LINUX_THREAD_DESCR, but that doesn't quite */ - /* seem to suffice. */ - /* We currently trace entire thread stacks, if they are */ - /* are currently cached but unused. This is */ - /* very suboptimal for performance reasons. */ -#endif - /* We no longer exclude the main data segment. */ - if ((word)end <= (word)least_ha || (word)start >= (word)greatest_ha) { - /* The easy case; just trace entire segment */ - GC_add_roots_inner(start, end, TRUE); - continue; - } - /* Add sections that don't belong to us. */ - i = 0; - while ((word)(GC_our_memory[i].hs_start + GC_our_memory[i].hs_bytes) < (word)start) - ++i; - GC_ASSERT(i < GC_n_memory); - if ((word)GC_our_memory[i].hs_start <= (word)start) { - start = GC_our_memory[i].hs_start + GC_our_memory[i].hs_bytes; - ++i; - } - while (i < GC_n_memory && (word)GC_our_memory[i].hs_start < (word)end && (word)start < (word)end) { - if ((word)start < (word)GC_our_memory[i].hs_start) - GC_add_roots_inner(start, - GC_our_memory[i].hs_start, TRUE); - start = GC_our_memory[i].hs_start + GC_our_memory[i].hs_bytes; - ++i; - } - if ((word)start < (word)end) - GC_add_roots_inner(start, end, TRUE); - } else if (prot[0] == '-' && prot[1] == '-' && prot[2] == '-') { - /* Even roots added statically might disappear partially */ - /* (e.g. the roots added by INCLUDE_LINUX_THREAD_DESCR). */ - GC_remove_roots_subregion(start, end); - } - } -} - -GC_INNER void GC_register_dynamic_libraries(void) { - GC_register_map_entries(GC_get_maps()); -} - -/* We now take care of the main data segment ourselves: */ -GC_INNER GC_bool GC_register_main_static_data(void) { - return FALSE; -} - -#define HAVE_REGISTER_MAIN_STATIC_DATA - -#else /* !USE_PROC_FOR_LIBRARIES */ - -/* The following is the preferred way to walk dynamic libraries */ -/* for glibc 2.2.4+. Unfortunately, it doesn't work for older */ -/* versions. Thanks to Jakub Jelinek for most of the code. */ - -#if GC_GLIBC_PREREQ(2, 3) || defined(HOST_ANDROID) -/* Are others OK here, too? */ -#ifndef HAVE_DL_ITERATE_PHDR -#define HAVE_DL_ITERATE_PHDR -#endif -#ifdef HOST_ANDROID -/* Android headers might have no such definition for some targets. */ -EXTERN_C_BEGIN -extern int dl_iterate_phdr(int (*cb)(struct dl_phdr_info *, - size_t, void *), - void *data); -EXTERN_C_END -#endif -#endif /* __GLIBC__ >= 2 || HOST_ANDROID */ - -#if defined(__DragonFly__) || defined(__FreeBSD_kernel__) || (defined(FREEBSD) && __FreeBSD__ >= 7) -/* On the FreeBSD system, any target system at major version 7 shall */ -/* have dl_iterate_phdr; therefore, we need not make it weak as below. */ -#ifndef HAVE_DL_ITERATE_PHDR -#define HAVE_DL_ITERATE_PHDR -#endif -#define DL_ITERATE_PHDR_STRONG -#elif defined(HAVE_DL_ITERATE_PHDR) -/* We have the header files for a glibc that includes dl_iterate_phdr.*/ -/* It may still not be available in the library on the target system. */ -/* Thus we also treat it as a weak symbol. */ -EXTERN_C_BEGIN -#pragma weak dl_iterate_phdr -EXTERN_C_END -#endif - -#if defined(HAVE_DL_ITERATE_PHDR) - -#ifdef PT_GNU_RELRO -/* Instead of registering PT_LOAD sections directly, we keep them */ -/* in a temporary list, and filter them by excluding PT_GNU_RELRO */ -/* segments. Processing PT_GNU_RELRO sections with */ -/* GC_exclude_static_roots instead would be superficially cleaner. But */ -/* it runs into trouble if a client registers an overlapping segment, */ -/* which unfortunately seems quite possible. */ - -#define MAX_LOAD_SEGS MAX_ROOT_SETS - -static struct load_segment { - ptr_t start; - ptr_t end; - /* Room for a second segment if we remove a RELRO segment */ - /* from the middle. */ - ptr_t start2; - ptr_t end2; -} load_segs[MAX_LOAD_SEGS]; - -static int n_load_segs; -static GC_bool load_segs_overflow; -#endif /* PT_GNU_RELRO */ - -STATIC int GC_register_dynlib_callback(struct dl_phdr_info *info, - size_t size, void *ptr) { - const ElfW(Phdr) * p; - ptr_t start, end; - int i; - - /* Make sure struct dl_phdr_info is at least as big as we need. */ - if (size < offsetof(struct dl_phdr_info, dlpi_phnum) + sizeof(info->dlpi_phnum)) - return -1; - - p = info->dlpi_phdr; - for (i = 0; i < (int)info->dlpi_phnum; i++, p++) { - if (p->p_type == PT_LOAD) { - GC_has_static_roots_func callback = GC_has_static_roots; - if ((p->p_flags & PF_W) == 0) continue; - - start = (ptr_t)p->p_vaddr + info->dlpi_addr; - end = start + p->p_memsz; - if (callback != 0 && !callback(info->dlpi_name, start, p->p_memsz)) - continue; -#ifdef PT_GNU_RELRO -#if CPP_WORDSZ == 64 - /* TODO: GC_push_all eventually does the correct */ - /* rounding to the next multiple of ALIGNMENT, so, most */ - /* probably, we should remove the corresponding assertion */ - /* check in GC_add_roots_inner along with this code line. */ - /* start pointer value may require aligning. */ - start = (ptr_t)((word)start & ~(word)(sizeof(word) - 1)); -#endif - if (n_load_segs >= MAX_LOAD_SEGS) { - if (!load_segs_overflow) { - WARN( - "Too many PT_LOAD segments;" - " registering as roots directly...\n", - 0); - load_segs_overflow = TRUE; - } - GC_add_roots_inner(start, end, TRUE); - } else { - load_segs[n_load_segs].start = start; - load_segs[n_load_segs].end = end; - load_segs[n_load_segs].start2 = 0; - load_segs[n_load_segs].end2 = 0; - ++n_load_segs; - } -#else - GC_add_roots_inner(start, end, TRUE); -#endif /* !PT_GNU_RELRO */ - } - } - -#ifdef PT_GNU_RELRO - p = info->dlpi_phdr; - for (i = 0; i < (int)info->dlpi_phnum; i++, p++) { - if (p->p_type == PT_GNU_RELRO) { - /* This entry is known to be constant and will eventually be */ - /* remapped as read-only. However, the address range covered */ - /* by this entry is typically a subset of a previously */ - /* encountered "LOAD" segment, so we need to exclude it. */ - int j; - - start = (ptr_t)p->p_vaddr + info->dlpi_addr; - end = start + p->p_memsz; - for (j = n_load_segs; --j >= 0;) { - if ((word)start >= (word)load_segs[j].start && (word)start < (word)load_segs[j].end) { - if (load_segs[j].start2 != 0) { - WARN("More than one GNU_RELRO segment per load one\n", 0); - } else { - GC_ASSERT((word)end <= - (((word)load_segs[j].end + GC_page_size - 1) & - ~(GC_page_size - 1))); - /* Remove from the existing load segment */ - load_segs[j].end2 = load_segs[j].end; - load_segs[j].end = start; - load_segs[j].start2 = end; - /* Note that start2 may be greater than end2 because of */ - /* p->p_memsz value multiple of page size. */ - } - break; - } - if (0 == j && 0 == GC_has_static_roots) - WARN( - "Failed to find PT_GNU_RELRO segment" - " inside PT_LOAD region\n", - 0); - /* No warning reported in case of the callback is present */ - /* because most likely the segment has been excluded. */ - } - } - } -#endif - - *(int *)ptr = 1; /* Signal that we were called */ - return 0; -} - -/* Do we need to separately register the main static data segment? */ -GC_INNER GC_bool GC_register_main_static_data(void) { -#ifdef DL_ITERATE_PHDR_STRONG - /* If dl_iterate_phdr is not a weak symbol then don't test against */ - /* zero (otherwise a compiler might issue a warning). */ - return FALSE; -#else - return 0 == COVERT_DATAFLOW(dl_iterate_phdr); -#endif -} - -/* Return TRUE if we succeed, FALSE if dl_iterate_phdr wasn't there. */ -STATIC GC_bool GC_register_dynamic_libraries_dl_iterate_phdr(void) { - int did_something; - if (GC_register_main_static_data()) - return FALSE; - -#ifdef PT_GNU_RELRO - { - static GC_bool excluded_segs = FALSE; - n_load_segs = 0; - load_segs_overflow = FALSE; - if (!EXPECT(excluded_segs, TRUE)) { - GC_exclude_static_roots_inner((ptr_t)load_segs, - (ptr_t)load_segs + sizeof(load_segs)); - excluded_segs = TRUE; - } - } -#endif - - did_something = 0; - dl_iterate_phdr(GC_register_dynlib_callback, &did_something); - if (did_something) { -#ifdef PT_GNU_RELRO - int i; - - for (i = 0; i < n_load_segs; ++i) { - if ((word)load_segs[i].end > (word)load_segs[i].start) { - GC_add_roots_inner(load_segs[i].start, load_segs[i].end, TRUE); - } - if ((word)load_segs[i].end2 > (word)load_segs[i].start2) { - GC_add_roots_inner(load_segs[i].start2, load_segs[i].end2, TRUE); - } - } -#endif - } else { - ptr_t datastart, dataend; -#ifdef DATASTART_IS_FUNC - static ptr_t datastart_cached = (ptr_t)GC_WORD_MAX; - - /* Evaluate DATASTART only once. */ - if (datastart_cached == (ptr_t)GC_WORD_MAX) { - datastart_cached = DATASTART; - } - datastart = datastart_cached; -#else - datastart = DATASTART; -#endif -#ifdef DATAEND_IS_FUNC - { - static ptr_t dataend_cached = 0; - /* Evaluate DATAEND only once. */ - if (dataend_cached == 0) { - dataend_cached = DATAEND; - } - dataend = dataend_cached; - } -#else - dataend = DATAEND; -#endif - if (NULL == *(char *volatile *)&datastart || (word)datastart > (word)dataend) - ABORT_ARG2("Wrong DATASTART/END pair", - ": %p .. %p", (void *)datastart, (void *)dataend); - - /* dl_iterate_phdr may forget the static data segment in */ - /* statically linked executables. */ - GC_add_roots_inner(datastart, dataend, TRUE); -#ifdef GC_HAVE_DATAREGION2 - if ((word)DATASTART2 - 1U >= (word)DATAEND2) { - /* Subtract one to check also for NULL */ - /* without a compiler warning. */ - ABORT_ARG2("Wrong DATASTART/END2 pair", - ": %p .. %p", (void *)DATASTART2, (void *)DATAEND2); - } - GC_add_roots_inner(DATASTART2, DATAEND2, TRUE); -#endif - } - return TRUE; -} - -#define HAVE_REGISTER_MAIN_STATIC_DATA - -#else /* !HAVE_DL_ITERATE_PHDR */ - -/* Dynamic loading code for Linux running ELF. Somewhat tested on - * Linux/x86, untested but hopefully should work on Linux/Alpha. - * This code was derived from the Solaris/ELF support. Thanks to - * whatever kind soul wrote that. - Patrick Bridges */ - -/* This doesn't necessarily work in all cases, e.g. with preloaded - * dynamic libraries. */ - -#if defined(NETBSD) || defined(OPENBSD) -#include -/* for compatibility with 1.4.x */ -#ifndef DT_DEBUG -#define DT_DEBUG 21 -#endif -#ifndef PT_LOAD -#define PT_LOAD 1 -#endif -#ifndef PF_W -#define PF_W 2 -#endif -#elif !defined(HOST_ANDROID) -#include -#endif - -#ifndef HOST_ANDROID -#include -#endif - -#endif /* !HAVE_DL_ITERATE_PHDR */ - -EXTERN_C_BEGIN -#ifdef __GNUC__ -#pragma weak _DYNAMIC -#endif -extern ElfW(Dyn) _DYNAMIC[]; -EXTERN_C_END - -STATIC struct link_map * -GC_FirstDLOpenedLinkMap(void) { - static struct link_map *cachedResult = 0; - - if (0 == COVERT_DATAFLOW(_DYNAMIC)) { - /* _DYNAMIC symbol not resolved. */ - return NULL; - } - if (NULL == cachedResult) { -#if defined(NETBSD) && defined(RTLD_DI_LINKMAP) -#if defined(CPPCHECK) -#define GC_RTLD_DI_LINKMAP 2 -#else -#define GC_RTLD_DI_LINKMAP RTLD_DI_LINKMAP -#endif - struct link_map *lm = NULL; - if (!dlinfo(RTLD_SELF, GC_RTLD_DI_LINKMAP, &lm) && lm != NULL) { - /* Now lm points link_map object of libgc. Since it */ - /* might not be the first dynamically linked object, */ - /* try to find it (object next to the main object). */ - while (lm->l_prev != NULL) { - lm = lm->l_prev; - } - cachedResult = lm->l_next; - } -#else - ElfW(Dyn) * dp; - int tag; - - for (dp = _DYNAMIC; (tag = dp->d_tag) != 0; dp++) { - if (tag == DT_DEBUG) { - struct r_debug *rd = (struct r_debug *)dp->d_un.d_ptr; - /* d_ptr could be null if libs are linked statically. */ - if (rd != NULL) { - struct link_map *lm = rd->r_map; - if (lm != NULL) - cachedResult = lm->l_next; /* might be NULL */ - } - break; - } - } -#endif /* !NETBSD || !RTLD_DI_LINKMAP */ - } - return cachedResult; -} - -GC_INNER void GC_register_dynamic_libraries(void) { - struct link_map *lm; - - GC_ASSERT(I_HOLD_LOCK()); -#ifdef HAVE_DL_ITERATE_PHDR - if (GC_register_dynamic_libraries_dl_iterate_phdr()) { - return; - } -#endif - for (lm = GC_FirstDLOpenedLinkMap(); lm != 0; lm = lm->l_next) { - ElfW(Ehdr) * e; - ElfW(Phdr) * p; - unsigned long offset; - char *start; - int i; - - e = (ElfW(Ehdr) *)lm->l_addr; -#ifdef HOST_ANDROID - if (e == NULL) - continue; -#endif - p = ((ElfW(Phdr) *)(((char *)(e)) + e->e_phoff)); - offset = ((unsigned long)(lm->l_addr)); - for (i = 0; i < (int)e->e_phnum; i++, p++) { - switch (p->p_type) { - case PT_LOAD: { - if (!(p->p_flags & PF_W)) break; - start = ((char *)(p->p_vaddr)) + offset; - GC_add_roots_inner(start, start + p->p_memsz, TRUE); - } break; - default: - break; - } - } - } -} - -#endif /* !USE_PROC_FOR_LIBRARIES */ - -#endif /* LINUX */ - -#if defined(IRIX5) || (defined(USE_PROC_FOR_LIBRARIES) && !defined(LINUX)) - -#include -#include -#include -#include /* Only for the following test. */ -#include -#include -#ifndef _sigargs -#define IRIX6 -#endif - -/* We use /proc to track down all parts of the address space that are */ -/* mapped by the process, and throw out regions we know we shouldn't */ -/* worry about. This may also work under other SVR4 variants. */ -GC_INNER void GC_register_dynamic_libraries(void) { - static int fd = -1; - static prmap_t *addr_map = 0; - static int current_sz = 0; /* Number of records currently in addr_map */ - - char buf[32]; - int needed_sz = 0; /* Required size of addr_map */ - int i; - long flags; - ptr_t start; - ptr_t limit; - ptr_t heap_start = HEAP_START; - ptr_t heap_end = heap_start; - -#ifdef SOLARISDL -#define MA_PHYS 0 -#endif /* SOLARISDL */ - - GC_ASSERT(I_HOLD_LOCK()); - if (fd < 0) { - (void)snprintf(buf, sizeof(buf), "/proc/%ld", (long)getpid()); - buf[sizeof(buf) - 1] = '\0'; - fd = open(buf, O_RDONLY); - if (fd < 0) { - ABORT("/proc open failed"); - } - } - if (ioctl(fd, PIOCNMAP, &needed_sz) < 0) { - ABORT_ARG2("/proc PIOCNMAP ioctl failed", - ": fd= %d, errno= %d", fd, errno); - } - if (needed_sz >= current_sz) { - GC_scratch_recycle_no_gww(addr_map, - (size_t)current_sz * sizeof(prmap_t)); - current_sz = needed_sz * 2 + 1; - /* Expansion, plus room for 0 record */ - addr_map = (prmap_t *)GC_scratch_alloc( - (size_t)current_sz * sizeof(prmap_t)); - if (addr_map == NULL) - ABORT("Insufficient memory for address map"); - } - if (ioctl(fd, PIOCMAP, addr_map) < 0) { - ABORT_ARG3("/proc PIOCMAP ioctl failed", - ": errcode= %d, needed_sz= %d, addr_map= %p", - errno, needed_sz, (void *)addr_map); - } - if (GC_n_heap_sects > 0) { - heap_end = GC_heap_sects[GC_n_heap_sects - 1].hs_start + GC_heap_sects[GC_n_heap_sects - 1].hs_bytes; - if ((word)heap_end < (word)GC_scratch_last_end_ptr) - heap_end = GC_scratch_last_end_ptr; - } - for (i = 0; i < needed_sz; i++) { - flags = addr_map[i].pr_mflags; - if ((flags & (MA_BREAK | MA_STACK | MA_PHYS | MA_FETCHOP | MA_NOTCACHED)) != 0) goto irrelevant; - if ((flags & (MA_READ | MA_WRITE)) != (MA_READ | MA_WRITE)) - goto irrelevant; - /* The latter test is empirically useless in very old Irix */ - /* versions. Other than the */ - /* main data and stack segments, everything appears to be */ - /* mapped readable, writable, executable, and shared(!!). */ - /* This makes no sense to me. - HB */ - start = (ptr_t)(addr_map[i].pr_vaddr); - if (GC_roots_present(start)) goto irrelevant; - if ((word)start < (word)heap_end && (word)start >= (word)heap_start) - goto irrelevant; - - limit = start + addr_map[i].pr_size; - /* The following seemed to be necessary for very old versions */ - /* of Irix, but it has been reported to discard relevant */ - /* segments under Irix 6.5. */ -#ifndef IRIX6 - if (addr_map[i].pr_off == 0 && strncmp(start, ELFMAG, 4) == 0) { - /* Discard text segments, i.e. 0-offset mappings against */ - /* executable files which appear to have ELF headers. */ - caddr_t arg; - int obj; -#define MAP_IRR_SZ 10 - static ptr_t map_irr[MAP_IRR_SZ]; - /* Known irrelevant map entries */ - static int n_irr = 0; - struct stat buf; - int j; - - for (j = 0; j < n_irr; j++) { - if (map_irr[j] == start) goto irrelevant; - } - arg = (caddr_t)start; - obj = ioctl(fd, PIOCOPENM, &arg); - if (obj >= 0) { - fstat(obj, &buf); - close(obj); - if ((buf.st_mode & 0111) != 0) { - if (n_irr < MAP_IRR_SZ) { - map_irr[n_irr++] = start; - } - goto irrelevant; - } - } - } -#endif /* !IRIX6 */ - GC_add_roots_inner(start, limit, TRUE); - irrelevant:; - } - /* Don't keep cached descriptor, for now. Some kernels don't like us */ - /* to keep a /proc file descriptor around during kill -9. */ - /* Otherwise, it should also require FD_CLOEXEC and proper handling */ - /* at fork (i.e. close because of the pid change). */ - if (close(fd) < 0) ABORT("Couldn't close /proc file"); - fd = -1; -} - -#endif /* USE_PROC_FOR_LIBRARIES || IRIX5 */ - -#if defined(MSWIN32) || defined(MSWINCE) || defined(CYGWIN32) - -/* We traverse the entire address space and register all segments */ -/* that could possibly have been written to. */ -STATIC void GC_cond_add_roots(char *base, char *limit) { -#ifdef GC_WIN32_THREADS - char *curr_base = base; - char *next_stack_lo; - char *next_stack_hi; - - if (base == limit) return; - for (;;) { - GC_get_next_stack(curr_base, limit, &next_stack_lo, &next_stack_hi); - if ((word)next_stack_lo >= (word)limit) break; - if ((word)next_stack_lo > (word)curr_base) - GC_add_roots_inner(curr_base, next_stack_lo, TRUE); - curr_base = next_stack_hi; - } - if ((word)curr_base < (word)limit) - GC_add_roots_inner(curr_base, limit, TRUE); -#else - char *stack_top = (char *)((word)GC_approx_sp() & - ~(word)(GC_sysinfo.dwAllocationGranularity - 1)); - - if (base == limit) return; - if ((word)limit > (word)stack_top && (word)base < (word)GC_stackbottom) { - /* Part of the stack; ignore it. */ - return; - } - GC_add_roots_inner(base, limit, TRUE); -#endif -} - -#ifdef DYNAMIC_LOADING -/* GC_register_main_static_data is not needed unless DYNAMIC_LOADING. */ -GC_INNER GC_bool GC_register_main_static_data(void) { -#if defined(MSWINCE) || defined(CYGWIN32) - /* Do we need to separately register the main static data segment? */ - return FALSE; -#else - return GC_no_win32_dlls; -#endif -} -#define HAVE_REGISTER_MAIN_STATIC_DATA -#endif /* DYNAMIC_LOADING */ - -#ifdef DEBUG_VIRTUALQUERY -void GC_dump_meminfo(MEMORY_BASIC_INFORMATION *buf) { - GC_printf( - "BaseAddress= 0x%lx, AllocationBase= 0x%lx," - " RegionSize= 0x%lx(%lu)\n", - buf->BaseAddress, buf->AllocationBase, - buf->RegionSize, buf->RegionSize); - GC_printf( - "\tAllocationProtect= 0x%lx, State= 0x%lx, Protect= 0x%lx, " - "Type= 0x%lx\n", - buf->AllocationProtect, buf->State, - buf->Protect, buf->Type); -} -#endif /* DEBUG_VIRTUALQUERY */ - -#if defined(MSWINCE) || defined(CYGWIN32) -/* FIXME: Should we really need to scan MEM_PRIVATE sections? */ -/* For now, we don't add MEM_PRIVATE sections to the data roots for */ -/* WinCE because otherwise SEGV fault sometimes happens to occur in */ -/* GC_mark_from() (and, even if we use WRAP_MARK_SOME, WinCE prints */ -/* a "Data Abort" message to the debugging console). */ -/* To workaround that, use -DGC_REGISTER_MEM_PRIVATE. */ -#define GC_wnt TRUE -#endif - -GC_INNER void GC_register_dynamic_libraries(void) { - MEMORY_BASIC_INFORMATION buf; - DWORD protect; - LPVOID p; - char *base; - char *limit, *new_limit; - - GC_ASSERT(I_HOLD_LOCK()); -#ifdef MSWIN32 - if (GC_no_win32_dlls) return; -#endif - p = GC_sysinfo.lpMinimumApplicationAddress; - base = limit = (char *)p; - while ((word)p < (word)GC_sysinfo.lpMaximumApplicationAddress) { - size_t result = VirtualQuery(p, &buf, sizeof(buf)); - -#ifdef MSWINCE - if (result == 0) { - /* Page is free; advance to the next possible allocation base */ - new_limit = (char *)(((DWORD)p + GC_sysinfo.dwAllocationGranularity) & ~(GC_sysinfo.dwAllocationGranularity - 1)); - } else -#endif - /* else */ { - if (result != sizeof(buf)) { - ABORT("Weird VirtualQuery result"); - } - new_limit = (char *)p + buf.RegionSize; - protect = buf.Protect; - if (buf.State == MEM_COMMIT && (protect == PAGE_EXECUTE_READWRITE || protect == PAGE_EXECUTE_WRITECOPY || protect == PAGE_READWRITE || protect == PAGE_WRITECOPY) && (buf.Type == MEM_IMAGE -#ifdef GC_REGISTER_MEM_PRIVATE - || (protect == PAGE_READWRITE && buf.Type == MEM_PRIVATE) -#else - /* There is some evidence that we cannot always */ - /* ignore MEM_PRIVATE sections under Windows ME */ - /* and predecessors. Hence we now also check for */ - /* that case. */ - || (!GC_wnt && buf.Type == MEM_PRIVATE) -#endif - ) && - !GC_is_heap_base(buf.AllocationBase)) { -#ifdef DEBUG_VIRTUALQUERY - GC_dump_meminfo(&buf); -#endif - if ((char *)p != limit) { - GC_cond_add_roots(base, limit); - base = (char *)p; - } - limit = new_limit; - } - } - if ((word)p > (word)new_limit /* overflow */) break; - p = (LPVOID)new_limit; - } - GC_cond_add_roots(base, limit); -} - -#endif /* MSWIN32 || MSWINCE || CYGWIN32 */ - -#if defined(ALPHA) && defined(OSF1) - -#include - -EXTERN_C_BEGIN -extern char *sys_errlist[]; -extern int sys_nerr; -extern int errno; -EXTERN_C_END - -GC_INNER void GC_register_dynamic_libraries(void) { - ldr_module_t moduleid = LDR_NULL_MODULE; - ldr_process_t mypid = ldr_my_process(); /* obtain id of this process */ - - /* For each module */ - while (TRUE) { - ldr_module_info_t moduleinfo; - size_t modulereturnsize; - ldr_region_t region; - ldr_region_info_t regioninfo; - size_t regionreturnsize; - int status = ldr_next_module(mypid, &moduleid); - /* Get the next (first) module */ - - /* Any more modules? */ - if (moduleid == LDR_NULL_MODULE) - break; /* No more modules */ - - /* Check status AFTER checking moduleid because */ - /* of a bug in the non-shared ldr_next_module stub. */ - if (status != 0) { - ABORT_ARG3("ldr_next_module failed", - ": status= %d, errcode= %d (%s)", status, errno, - errno < sys_nerr ? sys_errlist[errno] : ""); - } - - /* Get the module information */ - status = ldr_inq_module(mypid, moduleid, &moduleinfo, - sizeof(moduleinfo), &modulereturnsize); - if (status != 0) - ABORT("ldr_inq_module failed"); - - /* is module for the main program (i.e. nonshared portion)? */ - if (moduleinfo.lmi_flags & LDR_MAIN) - continue; /* skip the main module */ - -#ifdef DL_VERBOSE - GC_log_printf("---Module---\n"); - GC_log_printf("Module ID: %ld\n", moduleinfo.lmi_modid); - GC_log_printf("Count of regions: %d\n", moduleinfo.lmi_nregion); - GC_log_printf("Flags for module: %016lx\n", moduleinfo.lmi_flags); - GC_log_printf("Module pathname: \"%s\"\n", moduleinfo.lmi_name); -#endif - - /* For each region in this module */ - for (region = 0; region < moduleinfo.lmi_nregion; region++) { - /* Get the region information */ - status = ldr_inq_region(mypid, moduleid, region, ®ioninfo, - sizeof(regioninfo), ®ionreturnsize); - if (status != 0) - ABORT("ldr_inq_region failed"); - - /* only process writable (data) regions */ - if (!(regioninfo.lri_prot & LDR_W)) - continue; - -#ifdef DL_VERBOSE - GC_log_printf("--- Region ---\n"); - GC_log_printf("Region number: %ld\n", regioninfo.lri_region_no); - GC_log_printf("Protection flags: %016x\n", regioninfo.lri_prot); - GC_log_printf("Virtual address: %p\n", regioninfo.lri_vaddr); - GC_log_printf("Mapped address: %p\n", regioninfo.lri_mapaddr); - GC_log_printf("Region size: %ld\n", regioninfo.lri_size); - GC_log_printf("Region name: \"%s\"\n", regioninfo.lri_name); -#endif - - /* register region as a garbage collection root */ - GC_add_roots_inner((char *)regioninfo.lri_mapaddr, - (char *)regioninfo.lri_mapaddr + regioninfo.lri_size, - TRUE); - } - } -} -#endif - -#if defined(HPUX) - -#include -#include - -EXTERN_C_BEGIN -extern char *sys_errlist[]; -extern int sys_nerr; -EXTERN_C_END - -GC_INNER void GC_register_dynamic_libraries(void) { - int index = 1; /* Ordinal position in shared library search list */ - - /* For each dynamic library loaded */ - while (TRUE) { - struct shl_descriptor *shl_desc; /* Shared library info, see dl.h */ - int status = shl_get(index, &shl_desc); - /* Get info about next shared library */ - - /* Check if this is the end of the list or if some error occurred */ - if (status != 0) { -#ifdef GC_HPUX_THREADS - /* I've seen errno values of 0. The man page is not clear */ - /* as to whether errno should get set on a -1 return. */ - break; -#else - if (errno == EINVAL) { - break; /* Moved past end of shared library list --> finished */ - } else { - ABORT_ARG3("shl_get failed", - ": status= %d, errcode= %d (%s)", status, errno, - errno < sys_nerr ? sys_errlist[errno] : ""); - } -#endif - } - -#ifdef DL_VERBOSE - GC_log_printf("---Shared library---\n"); - GC_log_printf("filename= \"%s\"\n", shl_desc->filename); - GC_log_printf("index= %d\n", index); - GC_log_printf("handle= %08x\n", (unsigned long)shl_desc->handle); - GC_log_printf("text seg.start= %08x\n", shl_desc->tstart); - GC_log_printf("text seg.end= %08x\n", shl_desc->tend); - GC_log_printf("data seg.start= %08x\n", shl_desc->dstart); - GC_log_printf("data seg.end= %08x\n", shl_desc->dend); - GC_log_printf("ref.count= %lu\n", shl_desc->ref_count); -#endif - - /* register shared library's data segment as a garbage collection root */ - GC_add_roots_inner((char *)shl_desc->dstart, - (char *)shl_desc->dend, TRUE); - - index++; - } -} -#endif /* HPUX */ - -#ifdef AIX -#include -#include -#include -GC_INNER void GC_register_dynamic_libraries(void) { - int ldibuflen = 8192; - - for (;;) { - int len; - struct ld_info *ldi; -#if defined(CPPCHECK) - char ldibuf[ldibuflen]; -#else - char *ldibuf = alloca(ldibuflen); -#endif - - len = loadquery(L_GETINFO, ldibuf, ldibuflen); - if (len < 0) { - if (errno != ENOMEM) { - ABORT("loadquery failed"); - } - ldibuflen *= 2; - continue; - } - - ldi = (struct ld_info *)ldibuf; - while (ldi) { - len = ldi->ldinfo_next; - GC_add_roots_inner( - ldi->ldinfo_dataorg, - (ptr_t)(unsigned long)ldi->ldinfo_dataorg + ldi->ldinfo_datasize, - TRUE); - ldi = len ? (struct ld_info *)((char *)ldi + len) : 0; - } - break; - } -} -#endif /* AIX */ - -#ifdef DARWIN - -/* __private_extern__ hack required for pre-3.4 gcc versions. */ -#ifndef __private_extern__ -#define __private_extern__ extern -#include -#undef __private_extern__ -#else -#include -#endif -#include - -/*#define DARWIN_DEBUG*/ - -/* Writable sections generally available on Darwin. */ -STATIC const struct dyld_sections_s { - const char *seg; - const char *sect; -} GC_dyld_sections[] = { - {SEG_DATA, SECT_DATA}, - /* Used by FSF GCC, but not by OS X system tools, so far. */ - {SEG_DATA, "__static_data"}, - {SEG_DATA, SECT_BSS}, - {SEG_DATA, SECT_COMMON}, - /* FSF GCC - zero-sized object sections for targets */ - /*supporting section anchors. */ - {SEG_DATA, "__zobj_data"}, - {SEG_DATA, "__zobj_bss"}}; - -/* Additional writable sections: */ -/* GCC on Darwin constructs aligned sections "on demand", where */ -/* the alignment size is embedded in the section name. */ -/* Furthermore, there are distinctions between sections */ -/* containing private vs. public symbols. It also constructs */ -/* sections specifically for zero-sized objects, when the */ -/* target supports section anchors. */ -STATIC const char *const GC_dyld_add_sect_fmts[] = { - "__bss%u", - "__pu_bss%u", - "__zo_bss%u", - "__zo_pu_bss%u"}; - -/* Currently, mach-o will allow up to the max of 2^15 alignment */ -/* in an object file. */ -#ifndef L2_MAX_OFILE_ALIGNMENT -#define L2_MAX_OFILE_ALIGNMENT 15 -#endif - -STATIC const char *GC_dyld_name_for_hdr(const struct GC_MACH_HEADER *hdr) { - unsigned long i, c; - c = _dyld_image_count(); - for (i = 0; i < c; i++) - if ((const struct GC_MACH_HEADER *)_dyld_get_image_header(i) == hdr) - return _dyld_get_image_name(i); - return NULL; -} - -STATIC void GC_dyld_image_add(const struct GC_MACH_HEADER *hdr, - intptr_t slide) { - unsigned long start, end; - unsigned i, j; - const struct GC_MACH_SECTION *sec; - const char *name; - GC_has_static_roots_func callback = GC_has_static_roots; - - GC_ASSERT(I_DONT_HOLD_LOCK()); - if (GC_no_dls) return; -#ifdef DARWIN_DEBUG - name = GC_dyld_name_for_hdr(hdr); -#else - name = callback != 0 ? GC_dyld_name_for_hdr(hdr) : NULL; -#endif - for (i = 0; i < sizeof(GC_dyld_sections) / sizeof(GC_dyld_sections[0]); i++) { - sec = GC_GETSECTBYNAME(hdr, GC_dyld_sections[i].seg, - GC_dyld_sections[i].sect); - if (sec == NULL || sec->size < sizeof(word)) - continue; - start = slide + sec->addr; - end = start + sec->size; - LOCK(); - /* The user callback is called holding the lock. */ - if (callback == 0 || callback(name, (void *)start, (size_t)sec->size)) { -#ifdef DARWIN_DEBUG - GC_log_printf( - "Adding section __DATA,%s at %p-%p (%lu bytes) from image %s\n", - GC_dyld_sections[i].sect, (void *)start, (void *)end, - (unsigned long)sec->size, name); -#endif - GC_add_roots_inner((ptr_t)start, (ptr_t)end, FALSE); - } - UNLOCK(); - } - - /* Sections constructed on demand. */ - for (j = 0; j < sizeof(GC_dyld_add_sect_fmts) / sizeof(char *); j++) { - const char *fmt = GC_dyld_add_sect_fmts[j]; - - /* Add our manufactured aligned BSS sections. */ - for (i = 0; i <= L2_MAX_OFILE_ALIGNMENT; i++) { - char secnam[16]; - - (void)snprintf(secnam, sizeof(secnam), fmt, (unsigned)i); - secnam[sizeof(secnam) - 1] = '\0'; - sec = GC_GETSECTBYNAME(hdr, SEG_DATA, secnam); - if (sec == NULL || sec->size == 0) - continue; - start = slide + sec->addr; - end = start + sec->size; -#ifdef DARWIN_DEBUG - GC_log_printf( - "Adding on-demand section __DATA,%s at" - " %p-%p (%lu bytes) from image %s\n", - secnam, (void *)start, (void *)end, - (unsigned long)sec->size, name); -#endif - GC_add_roots((char *)start, (char *)end); - } - } - -#if defined(DARWIN_DEBUG) && !defined(NO_DEBUGGING) - LOCK(); - GC_print_static_roots(); - UNLOCK(); -#endif -} - -STATIC void GC_dyld_image_remove(const struct GC_MACH_HEADER *hdr, - intptr_t slide) { - unsigned long start, end; - unsigned i, j; - const struct GC_MACH_SECTION *sec; - - GC_ASSERT(I_DONT_HOLD_LOCK()); - for (i = 0; i < sizeof(GC_dyld_sections) / sizeof(GC_dyld_sections[0]); i++) { - sec = GC_GETSECTBYNAME(hdr, GC_dyld_sections[i].seg, - GC_dyld_sections[i].sect); - if (sec == NULL || sec->size == 0) - continue; - start = slide + sec->addr; - end = start + sec->size; -#ifdef DARWIN_DEBUG - GC_log_printf( - "Removing section __DATA,%s at %p-%p (%lu bytes) from image %s\n", - GC_dyld_sections[i].sect, (void *)start, (void *)end, - (unsigned long)sec->size, GC_dyld_name_for_hdr(hdr)); -#endif - GC_remove_roots((char *)start, (char *)end); - } - - /* Remove our on-demand sections. */ - for (j = 0; j < sizeof(GC_dyld_add_sect_fmts) / sizeof(char *); j++) { - const char *fmt = GC_dyld_add_sect_fmts[j]; - - for (i = 0; i <= L2_MAX_OFILE_ALIGNMENT; i++) { - char secnam[16]; - - (void)snprintf(secnam, sizeof(secnam), fmt, (unsigned)i); - secnam[sizeof(secnam) - 1] = '\0'; - sec = GC_GETSECTBYNAME(hdr, SEG_DATA, secnam); - if (sec == NULL || sec->size == 0) - continue; - start = slide + sec->addr; - end = start + sec->size; -#ifdef DARWIN_DEBUG - GC_log_printf( - "Removing on-demand section __DATA,%s at" - " %p-%p (%lu bytes) from image %s\n", - secnam, - (void *)start, (void *)end, (unsigned long)sec->size, - GC_dyld_name_for_hdr(hdr)); -#endif - GC_remove_roots((char *)start, (char *)end); - } - } - -#if defined(DARWIN_DEBUG) && !defined(NO_DEBUGGING) - LOCK(); - GC_print_static_roots(); - UNLOCK(); -#endif -} - -GC_INNER void GC_register_dynamic_libraries(void) { - /* Currently does nothing. The callbacks are setup by GC_init_dyld() - The dyld library takes it from there. */ -} - -/* The _dyld_* functions have an internal lock so no _dyld functions - can be called while the world is stopped without the risk of a deadlock. - Because of this we MUST setup callbacks BEFORE we ever stop the world. - This should be called BEFORE any thread in created and WITHOUT the - allocation lock held. */ - -GC_INNER void GC_init_dyld(void) { - static GC_bool initialized = FALSE; - - GC_ASSERT(I_DONT_HOLD_LOCK()); - if (initialized) return; - -#ifdef DARWIN_DEBUG - GC_log_printf("Registering dyld callbacks...\n"); -#endif - - /* Apple's Documentation: - When you call _dyld_register_func_for_add_image, the dynamic linker - runtime calls the specified callback (func) once for each of the images - that is currently loaded into the program. When a new image is added to - the program, your callback is called again with the mach_header for the - new image, and the virtual memory slide amount of the new image. - - This WILL properly register already linked libraries and libraries - linked in the future. - */ - _dyld_register_func_for_add_image( - (void (*)(const struct mach_header *, intptr_t))GC_dyld_image_add); - _dyld_register_func_for_remove_image( - (void (*)(const struct mach_header *, intptr_t))GC_dyld_image_remove); - /* Structure mach_header64 has the same fields */ - /* as mach_header except for the reserved one */ - /* at the end, so these casts are OK. */ - - /* Set this early to avoid reentrancy issues. */ - initialized = TRUE; - -#ifdef NO_DYLD_BIND_FULLY_IMAGE - /* FIXME: What should we do in this case? */ -#else - if (GC_no_dls) return; /* skip main data segment registration */ - - /* When the environment variable is set, the dynamic linker binds */ - /* all undefined symbols the application needs at launch time. */ - /* This includes function symbols that are normally bound lazily at */ - /* the time of their first invocation. */ - if (GETENV("DYLD_BIND_AT_LAUNCH") == 0) { - /* The environment variable is unset, so we should bind manually. */ -#ifdef DARWIN_DEBUG - GC_log_printf("Forcing full bind of GC code...\n"); -#endif - /* FIXME: '_dyld_bind_fully_image_containing_address' is deprecated. */ - if (!_dyld_bind_fully_image_containing_address( - (unsigned long *)GC_malloc)) - ABORT("_dyld_bind_fully_image_containing_address failed"); - } -#endif -} - -#define HAVE_REGISTER_MAIN_STATIC_DATA -GC_INNER GC_bool GC_register_main_static_data(void) { - /* Already done through dyld callbacks */ - return FALSE; -} - -#endif /* DARWIN */ - -#if defined(HAIKU) -#include - -GC_INNER void GC_register_dynamic_libraries(void) { - image_info info; - int32 cookie = 0; - - while (get_next_image_info(0, &cookie, &info) == B_OK) { - ptr_t data = (ptr_t)info.data; - GC_add_roots_inner(data, data + info.data_size, TRUE); - } -} -#endif /* HAIKU */ - -#elif defined(PCR) - -#include "il/PCR_IL.h" -#include "mm/PCR_MM.h" -#include "th/PCR_ThCtl.h" - -GC_INNER void GC_register_dynamic_libraries(void) { - /* Add new static data areas of dynamically loaded modules. */ - PCR_IL_LoadedFile* p = PCR_IL_GetLastLoadedFile(); - PCR_IL_LoadedSegment* q; - - /* Skip uncommitted files */ - while (p != NIL && !(p->lf_commitPoint)) { - /* The loading of this file has not yet been committed */ - /* Hence its description could be inconsistent. */ - /* Furthermore, it hasn't yet been run. Hence its data */ - /* segments can't possibly reference heap allocated */ - /* objects. */ - p = p->lf_prev; - } - for (; p != NIL; p = p->lf_prev) { - for (q = p->lf_ls; q != NIL; q = q->ls_next) { - if ((q->ls_flags & PCR_IL_SegFlags_Traced_MASK) == PCR_IL_SegFlags_Traced_on) { - GC_add_roots_inner((ptr_t)q->ls_addr, - (ptr_t)q->ls_addr + q->ls_bytes, TRUE); - } - } - } -} -#endif /* PCR && !DYNAMIC_LOADING && !MSWIN32 */ - -#if !defined(HAVE_REGISTER_MAIN_STATIC_DATA) && defined(DYNAMIC_LOADING) -/* Do we need to separately register the main static data segment? */ -GC_INNER GC_bool GC_register_main_static_data(void) { - return TRUE; -} -#endif /* HAVE_REGISTER_MAIN_STATIC_DATA */ - -/* Register a routine to filter dynamic library registration. */ -GC_API void GC_CALL GC_register_has_static_roots_callback( - GC_has_static_roots_func callback) { - GC_has_static_roots = callback; -} diff --git a/vendor/bdwgc/finalize.c b/vendor/bdwgc/finalize.c deleted file mode 100644 index 4ecbc8c6..00000000 --- a/vendor/bdwgc/finalize.c +++ /dev/null @@ -1,1376 +0,0 @@ -/* - * Copyright 1988, 1989 Hans-J. Boehm, Alan J. Demers - * Copyright (c) 1991-1996 by Xerox Corporation. All rights reserved. - * Copyright (c) 1996-1999 by Silicon Graphics. All rights reserved. - * Copyright (c) 2007 Free Software Foundation, Inc. - * Copyright (c) 2008-2022 Ivan Maidanski - * - * THIS MATERIAL IS PROVIDED AS IS, WITH ABSOLUTELY NO WARRANTY EXPRESSED - * OR IMPLIED. ANY USE IS AT YOUR OWN RISK. - * - * Permission is hereby granted to use or copy this program - * for any purpose, provided the above notices are retained on all copies. - * Permission to modify the code and to distribute modified code is granted, - * provided the above notices are retained, and a notice that the code was - * modified is included with the above copyright notice. - */ - -#ifdef _WIN32 -#define WIN32_LEAN_AND_MEAN -#include -#endif -#include "private/gc_pmark.h" - -#ifndef GC_NO_FINALIZATION - -GC_API void GC_CALL GC_finalize_all(void); - -/* Type of mark procedure used for marking from finalizable object. */ -/* This procedure normally does not mark the object, only its */ -/* descendants. */ -typedef void (*finalization_mark_proc)(ptr_t /* finalizable_obj_ptr */); - -#define HASH3(addr, size, log_size) \ - ((((word)(addr) >> 3) ^ ((word)(addr) >> (3 + (log_size)))) & ((size)-1)) -#define HASH2(addr, log_size) HASH3(addr, (word)1 << (log_size), log_size) - -struct hash_chain_entry { - word hidden_key; - struct hash_chain_entry *next; -}; - -struct disappearing_link { - struct hash_chain_entry prolog; -#define dl_hidden_link prolog.hidden_key - /* Field to be cleared. */ -#define dl_next(x) (struct disappearing_link *)((x)->prolog.next) -#define dl_set_next(x, y) \ - (void)((x)->prolog.next = (struct hash_chain_entry *)(y)) - word dl_hidden_obj; /* Pointer to object base */ -}; - -struct finalizable_object { - struct hash_chain_entry prolog; -#define fo_hidden_base prolog.hidden_key - /* Pointer to object base. */ - /* No longer hidden once object */ - /* is on finalize_now queue. */ -#define fo_next(x) (struct finalizable_object *)((x)->prolog.next) -#define fo_set_next(x, y) ((x)->prolog.next = (struct hash_chain_entry *)(y)) - GC_finalization_proc fo_fn; /* Finalizer. */ - ptr_t fo_client_data; - word fo_object_size; /* In bytes. */ - finalization_mark_proc fo_mark_proc; /* Mark-through procedure */ -}; - -#ifdef AO_HAVE_store -/* Update finalize_now atomically as GC_should_invoke_finalizers does */ -/* not acquire the allocation lock. */ -#define SET_FINALIZE_NOW(fo) \ - AO_store((volatile AO_t *)&GC_fnlz_roots.finalize_now, (AO_t)(fo)) -#else -#define SET_FINALIZE_NOW(fo) (void)(GC_fnlz_roots.finalize_now = (fo)) -#endif /* !THREADS */ - -GC_API void GC_CALL GC_push_finalizer_structures(void) { - GC_ASSERT((word)(&GC_dl_hashtbl.head) % sizeof(word) == 0); - GC_ASSERT((word)(&GC_fnlz_roots) % sizeof(word) == 0); -#ifndef GC_LONG_REFS_NOT_NEEDED - GC_ASSERT((word)(&GC_ll_hashtbl.head) % sizeof(word) == 0); - GC_PUSH_ALL_SYM(GC_ll_hashtbl.head); -#endif - GC_PUSH_ALL_SYM(GC_dl_hashtbl.head); - GC_PUSH_ALL_SYM(GC_fnlz_roots); - /* GC_toggleref_arr is pushed specially by GC_mark_togglerefs. */ -} - -/* Threshold of log_size to initiate full collection before growing */ -/* a hash table. */ -#ifndef GC_ON_GROW_LOG_SIZE_MIN -#define GC_ON_GROW_LOG_SIZE_MIN CPP_LOG_HBLKSIZE -#endif - -/* Double the size of a hash table. *log_size_ptr is the log of its */ -/* current size. May be a no-op. *table is a pointer to an array of */ -/* hash headers. We update both *table and *log_size_ptr on success. */ -STATIC void GC_grow_table(struct hash_chain_entry ***table, - unsigned *log_size_ptr, const word *entries_ptr) { - word i; - struct hash_chain_entry *p; - unsigned log_old_size = *log_size_ptr; - unsigned log_new_size = log_old_size + 1; - word old_size = *table == NULL ? 0 : (word)1 << log_old_size; - word new_size = (word)1 << log_new_size; - /* FIXME: Power of 2 size often gets rounded up to one more page. */ - struct hash_chain_entry **new_table; - - GC_ASSERT(I_HOLD_LOCK()); - /* Avoid growing the table in case of at least 25% of entries can */ - /* be deleted by enforcing a collection. Ignored for small tables. */ - /* In incremental mode we skip this optimization, as we want to */ - /* avoid triggering a full GC whenever possible. */ - if (log_old_size >= GC_ON_GROW_LOG_SIZE_MIN && !GC_incremental) { - IF_CANCEL(int cancel_state;) - - DISABLE_CANCEL(cancel_state); - GC_gcollect_inner(); - RESTORE_CANCEL(cancel_state); - /* GC_finalize might decrease entries value. */ - if (*entries_ptr < ((word)1 << log_old_size) - (*entries_ptr >> 2)) - return; - } - - new_table = (struct hash_chain_entry **) - GC_INTERNAL_MALLOC_IGNORE_OFF_PAGE( - (size_t)new_size * sizeof(struct hash_chain_entry *), - NORMAL); - if (new_table == 0) { - if (*table == 0) { - ABORT("Insufficient space for initial table allocation"); - } else { - return; - } - } - for (i = 0; i < old_size; i++) { - p = (*table)[i]; - while (p != 0) { - ptr_t real_key = (ptr_t)GC_REVEAL_POINTER(p->hidden_key); - struct hash_chain_entry *next = p->next; - size_t new_hash = HASH3(real_key, new_size, log_new_size); - - p->next = new_table[new_hash]; - GC_dirty(p); - new_table[new_hash] = p; - p = next; - } - } - *log_size_ptr = log_new_size; - *table = new_table; - GC_dirty(new_table); /* entire object */ -} - -GC_API int GC_CALL GC_register_disappearing_link(void **link) { - ptr_t base; - - base = (ptr_t)GC_base(link); - if (base == 0) - ABORT("Bad arg to GC_register_disappearing_link"); - return GC_general_register_disappearing_link(link, base); -} - -STATIC int GC_register_disappearing_link_inner( - struct dl_hashtbl_s *dl_hashtbl, void **link, - const void *obj, const char *tbl_log_name) { - struct disappearing_link *curr_dl; - size_t index; - struct disappearing_link *new_dl; - - GC_ASSERT(GC_is_initialized); - if (EXPECT(GC_find_leak, FALSE)) return GC_UNIMPLEMENTED; -#ifdef GC_ASSERTIONS - GC_noop1((word)(*link)); /* check accessibility */ -#endif - LOCK(); - GC_ASSERT(obj != NULL && GC_base_C(obj) == obj); - if (EXPECT(NULL == dl_hashtbl->head, FALSE) || EXPECT(dl_hashtbl->entries > ((word)1 << dl_hashtbl->log_size), FALSE)) { - GC_grow_table((struct hash_chain_entry ***)&dl_hashtbl->head, - &dl_hashtbl->log_size, &dl_hashtbl->entries); - GC_COND_LOG_PRINTF("Grew %s table to %u entries\n", tbl_log_name, - 1U << dl_hashtbl->log_size); - } - index = HASH2(link, dl_hashtbl->log_size); - for (curr_dl = dl_hashtbl->head[index]; curr_dl != 0; - curr_dl = dl_next(curr_dl)) { - if (curr_dl->dl_hidden_link == GC_HIDE_POINTER(link)) { - curr_dl->dl_hidden_obj = GC_HIDE_POINTER(obj); - UNLOCK(); - return GC_DUPLICATE; - } - } - new_dl = (struct disappearing_link *) - GC_INTERNAL_MALLOC(sizeof(struct disappearing_link), NORMAL); - if (EXPECT(NULL == new_dl, FALSE)) { - GC_oom_func oom_fn = GC_oom_fn; - UNLOCK(); - new_dl = (struct disappearing_link *)(*oom_fn)(sizeof(struct disappearing_link)); - if (0 == new_dl) { - return GC_NO_MEMORY; - } - /* It's not likely we'll make it here, but ... */ - LOCK(); - /* Recalculate index since the table may grow. */ - index = HASH2(link, dl_hashtbl->log_size); - /* Check again that our disappearing link not in the table. */ - for (curr_dl = dl_hashtbl->head[index]; curr_dl != 0; - curr_dl = dl_next(curr_dl)) { - if (curr_dl->dl_hidden_link == GC_HIDE_POINTER(link)) { - curr_dl->dl_hidden_obj = GC_HIDE_POINTER(obj); - UNLOCK(); -#ifndef DBG_HDRS_ALL - /* Free unused new_dl returned by GC_oom_fn() */ - GC_free((void *)new_dl); -#endif - return GC_DUPLICATE; - } - } - } - new_dl->dl_hidden_obj = GC_HIDE_POINTER(obj); - new_dl->dl_hidden_link = GC_HIDE_POINTER(link); - dl_set_next(new_dl, dl_hashtbl->head[index]); - GC_dirty(new_dl); - dl_hashtbl->head[index] = new_dl; - dl_hashtbl->entries++; - GC_dirty(dl_hashtbl->head + index); - UNLOCK(); - return GC_SUCCESS; -} - -GC_API int GC_CALL GC_general_register_disappearing_link(void **link, - const void *obj) { - if (((word)link & (ALIGNMENT - 1)) != 0 || !NONNULL_ARG_NOT_NULL(link)) - ABORT("Bad arg to GC_general_register_disappearing_link"); - return GC_register_disappearing_link_inner(&GC_dl_hashtbl, link, obj, - "dl"); -} - -#ifdef DBG_HDRS_ALL -#define FREE_DL_ENTRY(curr_dl) dl_set_next(curr_dl, NULL) -#else -#define FREE_DL_ENTRY(curr_dl) GC_free(curr_dl) -#endif - -/* Unregisters given link and returns the link entry to free. */ -GC_INLINE struct disappearing_link *GC_unregister_disappearing_link_inner( - struct dl_hashtbl_s *dl_hashtbl, void **link) { - struct disappearing_link *curr_dl; - struct disappearing_link *prev_dl = NULL; - size_t index; - - GC_ASSERT(I_HOLD_LOCK()); - if (EXPECT(NULL == dl_hashtbl->head, FALSE)) return NULL; - - index = HASH2(link, dl_hashtbl->log_size); - for (curr_dl = dl_hashtbl->head[index]; curr_dl; - curr_dl = dl_next(curr_dl)) { - if (curr_dl->dl_hidden_link == GC_HIDE_POINTER(link)) { - /* Remove found entry from the table. */ - if (NULL == prev_dl) { - dl_hashtbl->head[index] = dl_next(curr_dl); - GC_dirty(dl_hashtbl->head + index); - } else { - dl_set_next(prev_dl, dl_next(curr_dl)); - GC_dirty(prev_dl); - } - dl_hashtbl->entries--; - break; - } - prev_dl = curr_dl; - } - return curr_dl; -} - -GC_API int GC_CALL GC_unregister_disappearing_link(void **link) { - struct disappearing_link *curr_dl; - - if (((word)link & (ALIGNMENT - 1)) != 0) return 0; /* Nothing to do. */ - - LOCK(); - curr_dl = GC_unregister_disappearing_link_inner(&GC_dl_hashtbl, link); - UNLOCK(); - if (NULL == curr_dl) return 0; - FREE_DL_ENTRY(curr_dl); - return 1; -} - -/* Mark from one finalizable object using the specified mark proc. */ -/* May not mark the object pointed to by real_ptr (i.e, it is the job */ -/* of the caller, if appropriate). Note that this is called with the */ -/* mutator running. This is safe only if the mutator (client) gets */ -/* the allocation lock to reveal hidden pointers. */ -GC_INLINE void GC_mark_fo(ptr_t real_ptr, finalization_mark_proc mark_proc) { - GC_ASSERT(I_HOLD_LOCK()); - mark_proc(real_ptr); - /* Process objects pushed by the mark procedure. */ - while (!GC_mark_stack_empty()) - MARK_FROM_MARK_STACK(); -} - -/* Complete a collection in progress, if any. */ -GC_INLINE void GC_complete_ongoing_collection(void) { - if (EXPECT(GC_collection_in_progress(), FALSE)) { - while (!GC_mark_some(NULL)) { /* empty */ - } - } -} - -/* Toggle-ref support. */ -#ifndef GC_TOGGLE_REFS_NOT_NEEDED -typedef union toggle_ref_u GCToggleRef; - -STATIC GC_toggleref_func GC_toggleref_callback = 0; - -GC_INNER void GC_process_togglerefs(void) { - size_t i; - size_t new_size = 0; - GC_bool needs_barrier = FALSE; - - GC_ASSERT(I_HOLD_LOCK()); - for (i = 0; i < GC_toggleref_array_size; ++i) { - GCToggleRef r = GC_toggleref_arr[i]; - void *obj = r.strong_ref; - - if (((word)obj & 1) != 0) { - obj = GC_REVEAL_POINTER(r.weak_ref); - } - if (NULL == obj) { - continue; - } - switch (GC_toggleref_callback(obj)) { - case GC_TOGGLE_REF_DROP: - break; - case GC_TOGGLE_REF_STRONG: - GC_toggleref_arr[new_size++].strong_ref = obj; - needs_barrier = TRUE; - break; - case GC_TOGGLE_REF_WEAK: - GC_toggleref_arr[new_size++].weak_ref = GC_HIDE_POINTER(obj); - break; - default: - ABORT("Bad toggle-ref status returned by callback"); - } - } - - if (new_size < GC_toggleref_array_size) { - BZERO(&GC_toggleref_arr[new_size], - (GC_toggleref_array_size - new_size) * sizeof(GCToggleRef)); - GC_toggleref_array_size = new_size; - } - if (needs_barrier) - GC_dirty(GC_toggleref_arr); /* entire object */ -} - -STATIC void GC_normal_finalize_mark_proc(ptr_t); - -STATIC void GC_mark_togglerefs(void) { - size_t i; - - GC_ASSERT(I_HOLD_LOCK()); - if (NULL == GC_toggleref_arr) - return; - - GC_set_mark_bit(GC_toggleref_arr); - for (i = 0; i < GC_toggleref_array_size; ++i) { - void *obj = GC_toggleref_arr[i].strong_ref; - if (obj != NULL && ((word)obj & 1) == 0) { - /* Push and mark the object. */ - GC_mark_fo((ptr_t)obj, GC_normal_finalize_mark_proc); - GC_set_mark_bit(obj); - GC_complete_ongoing_collection(); - } - } -} - -STATIC void GC_clear_togglerefs(void) { - size_t i; - for (i = 0; i < GC_toggleref_array_size; ++i) { - if ((GC_toggleref_arr[i].weak_ref & 1) != 0) { - if (!GC_is_marked(GC_REVEAL_POINTER(GC_toggleref_arr[i].weak_ref))) { - GC_toggleref_arr[i].weak_ref = 0; - } else { - /* No need to copy, BDWGC is a non-moving collector. */ - } - } - } -} - -GC_API void GC_CALL GC_set_toggleref_func(GC_toggleref_func fn) { - LOCK(); - GC_toggleref_callback = fn; - UNLOCK(); -} - -GC_API GC_toggleref_func GC_CALL GC_get_toggleref_func(void) { - GC_toggleref_func fn; - - LOCK(); - fn = GC_toggleref_callback; - UNLOCK(); - return fn; -} - -static GC_bool ensure_toggleref_capacity(size_t capacity_inc) { - GC_ASSERT(I_HOLD_LOCK()); - if (NULL == GC_toggleref_arr) { - GC_toggleref_array_capacity = 32; /* initial capacity */ - GC_toggleref_arr = (GCToggleRef *)GC_INTERNAL_MALLOC_IGNORE_OFF_PAGE( - GC_toggleref_array_capacity * sizeof(GCToggleRef), - NORMAL); - if (NULL == GC_toggleref_arr) - return FALSE; - } - if (GC_toggleref_array_size + capacity_inc >= GC_toggleref_array_capacity) { - GCToggleRef *new_array; - while (GC_toggleref_array_capacity < GC_toggleref_array_size + capacity_inc) { - GC_toggleref_array_capacity *= 2; - if ((GC_toggleref_array_capacity & ((size_t)1 << (sizeof(size_t) * 8 - 1))) != 0) - return FALSE; /* overflow */ - } - - new_array = (GCToggleRef *)GC_INTERNAL_MALLOC_IGNORE_OFF_PAGE( - GC_toggleref_array_capacity * sizeof(GCToggleRef), - NORMAL); - if (NULL == new_array) - return FALSE; - if (EXPECT(GC_toggleref_array_size > 0, TRUE)) - BCOPY(GC_toggleref_arr, new_array, - GC_toggleref_array_size * sizeof(GCToggleRef)); - GC_INTERNAL_FREE(GC_toggleref_arr); - GC_toggleref_arr = new_array; - } - return TRUE; -} - -GC_API int GC_CALL GC_toggleref_add(void *obj, int is_strong_ref) { - int res = GC_SUCCESS; - - GC_ASSERT(NONNULL_ARG_NOT_NULL(obj)); - LOCK(); - if (GC_toggleref_callback != 0) { - if (!ensure_toggleref_capacity(1)) { - res = GC_NO_MEMORY; - } else { - GC_toggleref_arr[GC_toggleref_array_size].strong_ref = - is_strong_ref ? obj : (void *)GC_HIDE_POINTER(obj); - if (is_strong_ref) - GC_dirty(GC_toggleref_arr + GC_toggleref_array_size); - GC_toggleref_array_size++; - } - } - UNLOCK(); - return res; -} -#endif /* !GC_TOGGLE_REFS_NOT_NEEDED */ - -/* Finalizer callback support. */ -STATIC GC_await_finalize_proc GC_object_finalized_proc = 0; - -GC_API void GC_CALL GC_set_await_finalize_proc(GC_await_finalize_proc fn) { - LOCK(); - GC_object_finalized_proc = fn; - UNLOCK(); -} - -GC_API GC_await_finalize_proc GC_CALL GC_get_await_finalize_proc(void) { - GC_await_finalize_proc fn; - - LOCK(); - fn = GC_object_finalized_proc; - UNLOCK(); - return fn; -} - -#ifndef GC_LONG_REFS_NOT_NEEDED -GC_API int GC_CALL GC_register_long_link(void **link, const void *obj) { - if (((word)link & (ALIGNMENT - 1)) != 0 || !NONNULL_ARG_NOT_NULL(link)) - ABORT("Bad arg to GC_register_long_link"); - return GC_register_disappearing_link_inner(&GC_ll_hashtbl, link, obj, - "long dl"); -} - -GC_API int GC_CALL GC_unregister_long_link(void **link) { - struct disappearing_link *curr_dl; - - if (((word)link & (ALIGNMENT - 1)) != 0) return 0; /* Nothing to do. */ - - LOCK(); - curr_dl = GC_unregister_disappearing_link_inner(&GC_ll_hashtbl, link); - UNLOCK(); - if (NULL == curr_dl) return 0; - FREE_DL_ENTRY(curr_dl); - return 1; -} -#endif /* !GC_LONG_REFS_NOT_NEEDED */ - -#ifndef GC_MOVE_DISAPPEARING_LINK_NOT_NEEDED -STATIC int GC_move_disappearing_link_inner( - struct dl_hashtbl_s *dl_hashtbl, - void **link, void **new_link) { - struct disappearing_link *curr_dl, *new_dl; - struct disappearing_link *prev_dl = NULL; - size_t curr_index, new_index; - word curr_hidden_link, new_hidden_link; - -#ifdef GC_ASSERTIONS - GC_noop1((word)(*new_link)); -#endif - GC_ASSERT(I_HOLD_LOCK()); - if (EXPECT(NULL == dl_hashtbl->head, FALSE)) return GC_NOT_FOUND; - - /* Find current link. */ - curr_index = HASH2(link, dl_hashtbl->log_size); - curr_hidden_link = GC_HIDE_POINTER(link); - for (curr_dl = dl_hashtbl->head[curr_index]; curr_dl; - curr_dl = dl_next(curr_dl)) { - if (curr_dl->dl_hidden_link == curr_hidden_link) - break; - prev_dl = curr_dl; - } - if (EXPECT(NULL == curr_dl, FALSE)) { - return GC_NOT_FOUND; - } else if (link == new_link) { - return GC_SUCCESS; /* Nothing to do. */ - } - - /* link found; now check new_link not present. */ - new_index = HASH2(new_link, dl_hashtbl->log_size); - new_hidden_link = GC_HIDE_POINTER(new_link); - for (new_dl = dl_hashtbl->head[new_index]; new_dl; - new_dl = dl_next(new_dl)) { - if (new_dl->dl_hidden_link == new_hidden_link) { - /* Target already registered; bail. */ - return GC_DUPLICATE; - } - } - - /* Remove from old, add to new, update link. */ - if (NULL == prev_dl) { - dl_hashtbl->head[curr_index] = dl_next(curr_dl); - } else { - dl_set_next(prev_dl, dl_next(curr_dl)); - GC_dirty(prev_dl); - } - curr_dl->dl_hidden_link = new_hidden_link; - dl_set_next(curr_dl, dl_hashtbl->head[new_index]); - dl_hashtbl->head[new_index] = curr_dl; - GC_dirty(curr_dl); - GC_dirty(dl_hashtbl->head); /* entire object */ - return GC_SUCCESS; -} - -GC_API int GC_CALL GC_move_disappearing_link(void **link, void **new_link) { - int result; - - if (((word)new_link & (ALIGNMENT - 1)) != 0 || !NONNULL_ARG_NOT_NULL(new_link)) - ABORT("Bad new_link arg to GC_move_disappearing_link"); - if (((word)link & (ALIGNMENT - 1)) != 0) - return GC_NOT_FOUND; /* Nothing to do. */ - - LOCK(); - result = GC_move_disappearing_link_inner(&GC_dl_hashtbl, link, new_link); - UNLOCK(); - return result; -} - -#ifndef GC_LONG_REFS_NOT_NEEDED -GC_API int GC_CALL GC_move_long_link(void **link, void **new_link) { - int result; - - if (((word)new_link & (ALIGNMENT - 1)) != 0 || !NONNULL_ARG_NOT_NULL(new_link)) - ABORT("Bad new_link arg to GC_move_long_link"); - if (((word)link & (ALIGNMENT - 1)) != 0) - return GC_NOT_FOUND; /* Nothing to do. */ - - LOCK(); - result = GC_move_disappearing_link_inner(&GC_ll_hashtbl, link, new_link); - UNLOCK(); - return result; -} -#endif /* !GC_LONG_REFS_NOT_NEEDED */ -#endif /* !GC_MOVE_DISAPPEARING_LINK_NOT_NEEDED */ - -/* Possible finalization_marker procedures. Note that mark stack */ -/* overflow is handled by the caller, and is not a disaster. */ -#if defined(_MSC_VER) && defined(I386) -GC_ATTR_NOINLINE -/* Otherwise some optimizer bug is tickled in VC for x86 (v19, at least). */ -#endif -STATIC void GC_normal_finalize_mark_proc(ptr_t p) { - GC_mark_stack_top = GC_push_obj(p, HDR(p), GC_mark_stack_top, - GC_mark_stack + GC_mark_stack_size); -} - -/* This only pays very partial attention to the mark descriptor. */ -/* It does the right thing for normal and atomic objects, and treats */ -/* most others as normal. */ -STATIC void GC_ignore_self_finalize_mark_proc(ptr_t p) { - hdr *hhdr = HDR(p); - word descr = hhdr->hb_descr; - ptr_t current_p; - ptr_t scan_limit; - ptr_t target_limit = p + hhdr->hb_sz - 1; - - if ((descr & GC_DS_TAGS) == GC_DS_LENGTH) { - scan_limit = p + descr - sizeof(word); - } else { - scan_limit = target_limit + 1 - sizeof(word); - } - for (current_p = p; (word)current_p <= (word)scan_limit; - current_p += ALIGNMENT) { - word q; - - LOAD_WORD_OR_CONTINUE(q, current_p); - if (q < (word)p || q > (word)target_limit) { - GC_PUSH_ONE_HEAP(q, current_p, GC_mark_stack_top); - } - } -} - -STATIC void GC_null_finalize_mark_proc(ptr_t p) { - UNUSED_ARG(p); -} - -/* Possible finalization_marker procedures. Note that mark stack */ -/* overflow is handled by the caller, and is not a disaster. */ - -/* GC_unreachable_finalize_mark_proc is an alias for normal marking, */ -/* but it is explicitly tested for, and triggers different */ -/* behavior. Objects registered in this way are not finalized */ -/* if they are reachable by other finalizable objects, even if those */ -/* other objects specify no ordering. */ -STATIC void GC_unreachable_finalize_mark_proc(ptr_t p) { - /* A dummy comparison to ensure the compiler not to optimize two */ - /* identical functions into a single one (thus, to ensure a unique */ - /* address of each). Alternatively, GC_noop1(p) could be used. */ - if (EXPECT(NULL == p, FALSE)) return; - - GC_normal_finalize_mark_proc(p); -} - -static GC_bool need_unreachable_finalization = FALSE; -/* Avoid the work if this is not used. */ -/* TODO: turn need_unreachable_finalization into a counter */ - -/* Register a finalization function. See gc.h for details. */ -/* The last parameter is a procedure that determines */ -/* marking for finalization ordering. Any objects marked */ -/* by that procedure will be guaranteed to not have been */ -/* finalized when this finalizer is invoked. */ -STATIC void GC_register_finalizer_inner(void *obj, - GC_finalization_proc fn, void *cd, - GC_finalization_proc *ofn, void **ocd, - finalization_mark_proc mp) { - struct finalizable_object *curr_fo; - size_t index; - struct finalizable_object *new_fo = 0; - hdr *hhdr = NULL; /* initialized to prevent warning. */ - - GC_ASSERT(GC_is_initialized); - if (EXPECT(GC_find_leak, FALSE)) { - /* No-op. *ocd and *ofn remain unchanged. */ - return; - } - LOCK(); - GC_ASSERT(obj != NULL && GC_base_C(obj) == obj); - if (mp == GC_unreachable_finalize_mark_proc) - need_unreachable_finalization = TRUE; - if (EXPECT(NULL == GC_fnlz_roots.fo_head, FALSE) || EXPECT(GC_fo_entries > ((word)1 << GC_log_fo_table_size), FALSE)) { - GC_grow_table((struct hash_chain_entry ***)&GC_fnlz_roots.fo_head, - &GC_log_fo_table_size, &GC_fo_entries); - GC_COND_LOG_PRINTF("Grew fo table to %u entries\n", - 1U << GC_log_fo_table_size); - } - for (;;) { - struct finalizable_object *prev_fo = NULL; - GC_oom_func oom_fn; - - index = HASH2(obj, GC_log_fo_table_size); - curr_fo = GC_fnlz_roots.fo_head[index]; - while (curr_fo != 0) { - GC_ASSERT(GC_size(curr_fo) >= sizeof(struct finalizable_object)); - if (curr_fo->fo_hidden_base == GC_HIDE_POINTER(obj)) { - /* Interruption by a signal in the middle of this */ - /* should be safe. The client may see only *ocd */ - /* updated, but we'll declare that to be his problem. */ - if (ocd) *ocd = (void *)(curr_fo->fo_client_data); - if (ofn) *ofn = curr_fo->fo_fn; - /* Delete the structure for obj. */ - if (prev_fo == 0) { - GC_fnlz_roots.fo_head[index] = fo_next(curr_fo); - } else { - fo_set_next(prev_fo, fo_next(curr_fo)); - GC_dirty(prev_fo); - } - if (fn == 0) { - GC_fo_entries--; - /* May not happen if we get a signal. But a high */ - /* estimate will only make the table larger than */ - /* necessary. */ -#if !defined(THREADS) && !defined(DBG_HDRS_ALL) - GC_free((void *)curr_fo); -#endif - } else { - curr_fo->fo_fn = fn; - curr_fo->fo_client_data = (ptr_t)cd; - curr_fo->fo_mark_proc = mp; - GC_dirty(curr_fo); - /* Reinsert it. We deleted it first to maintain */ - /* consistency in the event of a signal. */ - if (prev_fo == 0) { - GC_fnlz_roots.fo_head[index] = curr_fo; - } else { - fo_set_next(prev_fo, curr_fo); - GC_dirty(prev_fo); - } - } - if (NULL == prev_fo) - GC_dirty(GC_fnlz_roots.fo_head + index); - UNLOCK(); -#ifndef DBG_HDRS_ALL - /* Free unused new_fo returned by GC_oom_fn() */ - GC_free((void *)new_fo); -#endif - return; - } - prev_fo = curr_fo; - curr_fo = fo_next(curr_fo); - } - if (EXPECT(new_fo != 0, FALSE)) { - /* new_fo is returned by GC_oom_fn(). */ - GC_ASSERT(fn != 0); -#ifdef LINT2 - if (NULL == hhdr) ABORT("Bad hhdr in GC_register_finalizer_inner"); -#endif - break; - } - if (fn == 0) { - if (ocd) *ocd = 0; - if (ofn) *ofn = 0; - UNLOCK(); - return; - } - GET_HDR(obj, hhdr); - if (EXPECT(0 == hhdr, FALSE)) { - /* We won't collect it, hence finalizer wouldn't be run. */ - if (ocd) *ocd = 0; - if (ofn) *ofn = 0; - UNLOCK(); - return; - } - new_fo = (struct finalizable_object *) - GC_INTERNAL_MALLOC(sizeof(struct finalizable_object), NORMAL); - if (EXPECT(new_fo != 0, TRUE)) - break; - oom_fn = GC_oom_fn; - UNLOCK(); - new_fo = (struct finalizable_object *)(*oom_fn)(sizeof(struct finalizable_object)); - if (0 == new_fo) { - /* No enough memory. *ocd and *ofn remain unchanged. */ - return; - } - /* It's not likely we'll make it here, but ... */ - LOCK(); - /* Recalculate index since the table may grow and */ - /* check again that our finalizer is not in the table. */ - } - GC_ASSERT(GC_size(new_fo) >= sizeof(struct finalizable_object)); - if (ocd) *ocd = 0; - if (ofn) *ofn = 0; - new_fo->fo_hidden_base = GC_HIDE_POINTER(obj); - new_fo->fo_fn = fn; - new_fo->fo_client_data = (ptr_t)cd; - new_fo->fo_object_size = hhdr->hb_sz; - new_fo->fo_mark_proc = mp; - fo_set_next(new_fo, GC_fnlz_roots.fo_head[index]); - GC_dirty(new_fo); - GC_fo_entries++; - GC_fnlz_roots.fo_head[index] = new_fo; - GC_dirty(GC_fnlz_roots.fo_head + index); - UNLOCK(); -} - -GC_API void GC_CALL GC_register_finalizer(void *obj, - GC_finalization_proc fn, void *cd, - GC_finalization_proc *ofn, void **ocd) { - GC_register_finalizer_inner(obj, fn, cd, ofn, - ocd, GC_normal_finalize_mark_proc); -} - -GC_API void GC_CALL GC_register_finalizer_ignore_self(void *obj, - GC_finalization_proc fn, void *cd, - GC_finalization_proc *ofn, void **ocd) { - GC_register_finalizer_inner(obj, fn, cd, ofn, - ocd, GC_ignore_self_finalize_mark_proc); -} - -GC_API void GC_CALL GC_register_finalizer_no_order(void *obj, - GC_finalization_proc fn, void *cd, - GC_finalization_proc *ofn, void **ocd) { - GC_register_finalizer_inner(obj, fn, cd, ofn, - ocd, GC_null_finalize_mark_proc); -} - -GC_API void GC_CALL GC_register_finalizer_unreachable(void *obj, - GC_finalization_proc fn, void *cd, - GC_finalization_proc *ofn, void **ocd) { - GC_ASSERT(GC_java_finalization); - GC_register_finalizer_inner(obj, fn, cd, ofn, - ocd, GC_unreachable_finalize_mark_proc); -} - -#ifndef NO_DEBUGGING -STATIC void GC_dump_finalization_links( - const struct dl_hashtbl_s *dl_hashtbl) { - size_t dl_size = (size_t)1 << dl_hashtbl->log_size; - size_t i; - - if (NULL == dl_hashtbl->head) return; /* empty table */ - - for (i = 0; i < dl_size; i++) { - struct disappearing_link *curr_dl; - - for (curr_dl = dl_hashtbl->head[i]; curr_dl != 0; - curr_dl = dl_next(curr_dl)) { - ptr_t real_ptr = (ptr_t)GC_REVEAL_POINTER(curr_dl->dl_hidden_obj); - ptr_t real_link = (ptr_t)GC_REVEAL_POINTER(curr_dl->dl_hidden_link); - - GC_printf("Object: %p, link value: %p, link addr: %p\n", - (void *)real_ptr, *(void **)real_link, (void *)real_link); - } - } -} - -GC_API void GC_CALL GC_dump_finalization(void) { - struct finalizable_object *curr_fo; - size_t i; - size_t fo_size = GC_fnlz_roots.fo_head == NULL ? 0 : (size_t)1 << GC_log_fo_table_size; - - GC_printf("\n***Disappearing (short) links:\n"); - GC_dump_finalization_links(&GC_dl_hashtbl); -#ifndef GC_LONG_REFS_NOT_NEEDED - GC_printf("\n***Disappearing long links:\n"); - GC_dump_finalization_links(&GC_ll_hashtbl); -#endif - GC_printf("\n***Finalizers:\n"); - for (i = 0; i < fo_size; i++) { - for (curr_fo = GC_fnlz_roots.fo_head[i]; - curr_fo != NULL; curr_fo = fo_next(curr_fo)) { - ptr_t real_ptr = (ptr_t)GC_REVEAL_POINTER(curr_fo->fo_hidden_base); - - GC_printf("Finalizable object: %p\n", (void *)real_ptr); - } - } -} -#endif /* !NO_DEBUGGING */ - -#ifndef SMALL_CONFIG -STATIC word GC_old_dl_entries = 0; /* for stats printing */ -#ifndef GC_LONG_REFS_NOT_NEEDED -STATIC word GC_old_ll_entries = 0; -#endif -#endif /* !SMALL_CONFIG */ - -#ifndef THREADS -/* Global variables to minimize the level of recursion when a client */ -/* finalizer allocates memory. */ -STATIC int GC_finalizer_nested = 0; -/* Only the lowest byte is used, the rest is */ -/* padding for proper global data alignment */ -/* required for some compilers (like Watcom). */ -STATIC unsigned GC_finalizer_skipped = 0; - -/* Checks and updates the level of finalizers recursion. */ -/* Returns NULL if GC_invoke_finalizers() should not be called by the */ -/* collector (to minimize the risk of a deep finalizers recursion), */ -/* otherwise returns a pointer to GC_finalizer_nested. */ -STATIC unsigned char *GC_check_finalizer_nested(void) { - unsigned nesting_level = *(unsigned char *)&GC_finalizer_nested; - if (nesting_level) { - /* We are inside another GC_invoke_finalizers(). */ - /* Skip some implicitly-called GC_invoke_finalizers() */ - /* depending on the nesting (recursion) level. */ - if (++GC_finalizer_skipped < (1U << nesting_level)) return NULL; - GC_finalizer_skipped = 0; - } - *(char *)&GC_finalizer_nested = (char)(nesting_level + 1); - return (unsigned char *)&GC_finalizer_nested; -} -#endif /* !THREADS */ - -GC_INLINE void GC_make_disappearing_links_disappear( - struct dl_hashtbl_s *dl_hashtbl, - GC_bool is_remove_dangling) { - size_t i; - size_t dl_size = (size_t)1 << dl_hashtbl->log_size; - GC_bool needs_barrier = FALSE; - - GC_ASSERT(I_HOLD_LOCK()); - if (NULL == dl_hashtbl->head) return; /* empty table */ - - for (i = 0; i < dl_size; i++) { - struct disappearing_link *curr_dl, *next_dl; - struct disappearing_link *prev_dl = NULL; - - for (curr_dl = dl_hashtbl->head[i]; curr_dl != NULL; curr_dl = next_dl) { - next_dl = dl_next(curr_dl); -#if defined(GC_ASSERTIONS) && !defined(THREAD_SANITIZER) - /* Check accessibility of the location pointed by link. */ - GC_noop1(*(word *)GC_REVEAL_POINTER(curr_dl->dl_hidden_link)); -#endif - if (is_remove_dangling) { - ptr_t real_link = (ptr_t)GC_base(GC_REVEAL_POINTER( - curr_dl->dl_hidden_link)); - - if (NULL == real_link || EXPECT(GC_is_marked(real_link), TRUE)) { - prev_dl = curr_dl; - continue; - } - } else { - if (EXPECT(GC_is_marked((ptr_t)GC_REVEAL_POINTER( - curr_dl->dl_hidden_obj)), - TRUE)) { - prev_dl = curr_dl; - continue; - } - *(ptr_t *)GC_REVEAL_POINTER(curr_dl->dl_hidden_link) = NULL; - } - - /* Delete curr_dl entry from dl_hashtbl. */ - if (NULL == prev_dl) { - dl_hashtbl->head[i] = next_dl; - needs_barrier = TRUE; - } else { - dl_set_next(prev_dl, next_dl); - GC_dirty(prev_dl); - } - GC_clear_mark_bit(curr_dl); - dl_hashtbl->entries--; - } - } - if (needs_barrier) - GC_dirty(dl_hashtbl->head); /* entire object */ -} - -/* Cause disappearing links to disappear and unreachable objects to be */ -/* enqueued for finalization. Called with the world running. */ -GC_INNER void GC_finalize(void) { - struct finalizable_object *curr_fo, *prev_fo, *next_fo; - ptr_t real_ptr; - size_t i; - size_t fo_size = GC_fnlz_roots.fo_head == NULL ? 0 : (size_t)1 << GC_log_fo_table_size; - GC_bool needs_barrier = FALSE; - - GC_ASSERT(I_HOLD_LOCK()); -#ifndef SMALL_CONFIG - /* Save current GC_[dl/ll]_entries value for stats printing */ - GC_old_dl_entries = GC_dl_hashtbl.entries; -#ifndef GC_LONG_REFS_NOT_NEEDED - GC_old_ll_entries = GC_ll_hashtbl.entries; -#endif -#endif - -#ifndef GC_TOGGLE_REFS_NOT_NEEDED - GC_mark_togglerefs(); -#endif - GC_make_disappearing_links_disappear(&GC_dl_hashtbl, FALSE); - - /* Mark all objects reachable via chains of 1 or more pointers */ - /* from finalizable objects. */ - GC_ASSERT(!GC_collection_in_progress()); - for (i = 0; i < fo_size; i++) { - for (curr_fo = GC_fnlz_roots.fo_head[i]; - curr_fo != NULL; curr_fo = fo_next(curr_fo)) { - GC_ASSERT(GC_size(curr_fo) >= sizeof(struct finalizable_object)); - real_ptr = (ptr_t)GC_REVEAL_POINTER(curr_fo->fo_hidden_base); - if (!GC_is_marked(real_ptr)) { - GC_MARKED_FOR_FINALIZATION(real_ptr); - GC_mark_fo(real_ptr, curr_fo->fo_mark_proc); - if (GC_is_marked(real_ptr)) { - WARN("Finalization cycle involving %p\n", real_ptr); - } - } - } - } - /* Enqueue for finalization all objects that are still */ - /* unreachable. */ - GC_bytes_finalized = 0; - for (i = 0; i < fo_size; i++) { - curr_fo = GC_fnlz_roots.fo_head[i]; - prev_fo = 0; - while (curr_fo != 0) { - real_ptr = (ptr_t)GC_REVEAL_POINTER(curr_fo->fo_hidden_base); - if (!GC_is_marked(real_ptr)) { - if (!GC_java_finalization) { - GC_set_mark_bit(real_ptr); - } - /* Delete from hash table */ - next_fo = fo_next(curr_fo); - if (NULL == prev_fo) { - GC_fnlz_roots.fo_head[i] = next_fo; - if (GC_object_finalized_proc) { - GC_dirty(GC_fnlz_roots.fo_head + i); - } else { - needs_barrier = TRUE; - } - } else { - fo_set_next(prev_fo, next_fo); - GC_dirty(prev_fo); - } - GC_fo_entries--; - if (GC_object_finalized_proc) - GC_object_finalized_proc(real_ptr); - - /* Add to list of objects awaiting finalization. */ - fo_set_next(curr_fo, GC_fnlz_roots.finalize_now); - GC_dirty(curr_fo); - SET_FINALIZE_NOW(curr_fo); - /* unhide object pointer so any future collections will */ - /* see it. */ - curr_fo->fo_hidden_base = - (word)GC_REVEAL_POINTER(curr_fo->fo_hidden_base); - GC_bytes_finalized += - curr_fo->fo_object_size + sizeof(struct finalizable_object); - GC_ASSERT(GC_is_marked(GC_base(curr_fo))); - curr_fo = next_fo; - } else { - prev_fo = curr_fo; - curr_fo = fo_next(curr_fo); - } - } - } - - if (GC_java_finalization) { - /* make sure we mark everything reachable from objects finalized - using the no_order mark_proc */ - for (curr_fo = GC_fnlz_roots.finalize_now; - curr_fo != NULL; curr_fo = fo_next(curr_fo)) { - real_ptr = (ptr_t)curr_fo->fo_hidden_base; - if (!GC_is_marked(real_ptr)) { - if (curr_fo->fo_mark_proc == GC_null_finalize_mark_proc) { - GC_mark_fo(real_ptr, GC_normal_finalize_mark_proc); - } - if (curr_fo->fo_mark_proc != GC_unreachable_finalize_mark_proc) { - GC_set_mark_bit(real_ptr); - } - } - } - - /* now revive finalize-when-unreachable objects reachable from - other finalizable objects */ - if (need_unreachable_finalization) { - curr_fo = GC_fnlz_roots.finalize_now; - GC_ASSERT(NULL == curr_fo || GC_fnlz_roots.fo_head != NULL); - prev_fo = NULL; - while (curr_fo != NULL) { - next_fo = fo_next(curr_fo); - if (curr_fo->fo_mark_proc == GC_unreachable_finalize_mark_proc) { - real_ptr = (ptr_t)curr_fo->fo_hidden_base; - if (!GC_is_marked(real_ptr)) { - GC_set_mark_bit(real_ptr); - } else { - if (NULL == prev_fo) { - SET_FINALIZE_NOW(next_fo); - } else { - fo_set_next(prev_fo, next_fo); - GC_dirty(prev_fo); - } - curr_fo->fo_hidden_base = - GC_HIDE_POINTER(curr_fo->fo_hidden_base); - GC_bytes_finalized -= - curr_fo->fo_object_size + sizeof(struct finalizable_object); - - i = HASH2(real_ptr, GC_log_fo_table_size); - fo_set_next(curr_fo, GC_fnlz_roots.fo_head[i]); - GC_dirty(curr_fo); - GC_fo_entries++; - GC_fnlz_roots.fo_head[i] = curr_fo; - curr_fo = prev_fo; - needs_barrier = TRUE; - } - } - prev_fo = curr_fo; - curr_fo = next_fo; - } - } - } - if (needs_barrier) - GC_dirty(GC_fnlz_roots.fo_head); /* entire object */ - - /* Remove dangling disappearing links. */ - GC_make_disappearing_links_disappear(&GC_dl_hashtbl, TRUE); - -#ifndef GC_TOGGLE_REFS_NOT_NEEDED - GC_clear_togglerefs(); -#endif -#ifndef GC_LONG_REFS_NOT_NEEDED - GC_make_disappearing_links_disappear(&GC_ll_hashtbl, FALSE); - GC_make_disappearing_links_disappear(&GC_ll_hashtbl, TRUE); -#endif - - if (GC_fail_count) { - /* Don't prevent running finalizers if there has been an allocation */ - /* failure recently. */ -#ifdef THREADS - GC_reset_finalizer_nested(); -#else - GC_finalizer_nested = 0; -#endif - } -} - -/* Count of finalizers to run, at most, during a single invocation */ -/* of GC_invoke_finalizers(); zero means no limit. Accessed with the */ -/* allocation lock held. */ -STATIC unsigned GC_interrupt_finalizers = 0; - -#ifndef JAVA_FINALIZATION_NOT_NEEDED - -/* Enqueue all remaining finalizers to be run. */ -/* A collection in progress, if any, is completed */ -/* when the first finalizer is enqueued. */ -STATIC void GC_enqueue_all_finalizers(void) { - size_t i; - size_t fo_size = GC_fnlz_roots.fo_head == NULL ? 0 : (size_t)1 << GC_log_fo_table_size; - - GC_ASSERT(I_HOLD_LOCK()); - GC_bytes_finalized = 0; - for (i = 0; i < fo_size; i++) { - struct finalizable_object *curr_fo = GC_fnlz_roots.fo_head[i]; - - GC_fnlz_roots.fo_head[i] = NULL; - while (curr_fo != NULL) { - struct finalizable_object *next_fo; - ptr_t real_ptr = (ptr_t)GC_REVEAL_POINTER(curr_fo->fo_hidden_base); - - GC_mark_fo(real_ptr, GC_normal_finalize_mark_proc); - GC_set_mark_bit(real_ptr); - GC_complete_ongoing_collection(); - next_fo = fo_next(curr_fo); - - /* Add to list of objects awaiting finalization. */ - fo_set_next(curr_fo, GC_fnlz_roots.finalize_now); - GC_dirty(curr_fo); - SET_FINALIZE_NOW(curr_fo); - - /* unhide object pointer so any future collections will */ - /* see it. */ - curr_fo->fo_hidden_base = - (word)GC_REVEAL_POINTER(curr_fo->fo_hidden_base); - GC_bytes_finalized += - curr_fo->fo_object_size + sizeof(struct finalizable_object); - curr_fo = next_fo; - } - } - GC_fo_entries = 0; /* all entries deleted from the hash table */ -} - -/* Invoke all remaining finalizers that haven't yet been run. - * This is needed for strict compliance with the Java standard, - * which can make the runtime guarantee that all finalizers are run. - * Unfortunately, the Java standard implies we have to keep running - * finalizers until there are no more left, a potential infinite loop. - * YUCK. - * Note that this is even more dangerous than the usual Java - * finalizers, in that objects reachable from static variables - * may have been finalized when these finalizers are run. - * Finalizers run at this point must be prepared to deal with a - * mostly broken world. - */ -GC_API void GC_CALL GC_finalize_all(void) { - LOCK(); - while (GC_fo_entries > 0) { - GC_enqueue_all_finalizers(); - GC_interrupt_finalizers = 0; /* reset */ - UNLOCK(); - GC_invoke_finalizers(); - /* Running the finalizers in this thread is arguably not a good */ - /* idea when we should be notifying another thread to run them. */ - /* But otherwise we don't have a great way to wait for them to */ - /* run. */ - LOCK(); - } - UNLOCK(); -} - -#endif /* !JAVA_FINALIZATION_NOT_NEEDED */ - -GC_API void GC_CALL GC_set_interrupt_finalizers(unsigned value) { - LOCK(); - GC_interrupt_finalizers = value; - UNLOCK(); -} - -GC_API unsigned GC_CALL GC_get_interrupt_finalizers(void) { - unsigned value; - - LOCK(); - value = GC_interrupt_finalizers; - UNLOCK(); - return value; -} - -/* Returns true if it is worth calling GC_invoke_finalizers. (Useful if */ -/* finalizers can only be called from some kind of "safe state" and */ -/* getting into that safe state is expensive.) */ -GC_API int GC_CALL GC_should_invoke_finalizers(void) { -#ifdef AO_HAVE_load - return AO_load((volatile AO_t *)&GC_fnlz_roots.finalize_now) != 0; -#else - return GC_fnlz_roots.finalize_now != NULL; -#endif /* !THREADS */ -} - -/* Invoke finalizers for all objects that are ready to be finalized. */ -GC_API int GC_CALL GC_invoke_finalizers(void) { - int count = 0; - word bytes_freed_before = 0; /* initialized to prevent warning. */ - - GC_ASSERT(I_DONT_HOLD_LOCK()); - while (GC_should_invoke_finalizers()) { - struct finalizable_object *curr_fo; - - LOCK(); - if (count == 0) { - bytes_freed_before = GC_bytes_freed; - /* Don't do this outside, since we need the lock. */ - } else if (EXPECT(GC_interrupt_finalizers != 0, FALSE) && (unsigned)count >= GC_interrupt_finalizers) { - UNLOCK(); - break; - } - curr_fo = GC_fnlz_roots.finalize_now; -#ifdef THREADS - if (EXPECT(NULL == curr_fo, FALSE)) { - UNLOCK(); - break; - } -#endif - SET_FINALIZE_NOW(fo_next(curr_fo)); - UNLOCK(); - fo_set_next(curr_fo, 0); - (*(curr_fo->fo_fn))((ptr_t)(curr_fo->fo_hidden_base), - curr_fo->fo_client_data); - curr_fo->fo_client_data = 0; - ++count; - /* Explicit freeing of curr_fo is probably a bad idea. */ - /* It throws off accounting if nearly all objects are */ - /* finalizable. Otherwise it should not matter. */ - } - /* bytes_freed_before is initialized whenever count != 0 */ - if (count != 0 -#if defined(THREADS) && !defined(THREAD_SANITIZER) - /* A quick check whether some memory was freed. */ - /* The race with GC_free() is safe to be ignored */ - /* because we only need to know if the current */ - /* thread has deallocated something. */ - && bytes_freed_before != GC_bytes_freed -#endif - ) { - LOCK(); - GC_finalizer_bytes_freed += (GC_bytes_freed - bytes_freed_before); - UNLOCK(); - } - return count; -} - -static word last_finalizer_notification = 0; - -GC_INNER void GC_notify_or_invoke_finalizers(void) { - GC_finalizer_notifier_proc notifier_fn = 0; -#if defined(KEEP_BACK_PTRS) || defined(MAKE_BACK_GRAPH) - static word last_back_trace_gc_no = 1; /* Skip first one. */ -#endif - -#if defined(THREADS) && !defined(KEEP_BACK_PTRS) && !defined(MAKE_BACK_GRAPH) - /* Quick check (while unlocked) for an empty finalization queue. */ - if (!GC_should_invoke_finalizers()) - return; -#endif - LOCK(); - - /* This is a convenient place to generate backtraces if appropriate, */ - /* since that code is not callable with the allocation lock. */ -#if defined(KEEP_BACK_PTRS) || defined(MAKE_BACK_GRAPH) - if (GC_gc_no > last_back_trace_gc_no) { -#ifdef KEEP_BACK_PTRS - long i; - /* Stops when GC_gc_no wraps; that's OK. */ - last_back_trace_gc_no = GC_WORD_MAX; /* disable others. */ - for (i = 0; i < GC_backtraces; ++i) { - /* FIXME: This tolerates concurrent heap mutation, */ - /* which may cause occasional mysterious results. */ - /* We need to release the GC lock, since GC_print_callers */ - /* acquires it. It probably shouldn't. */ - void *current = GC_generate_random_valid_address(); - - UNLOCK(); - GC_printf("\n****Chosen address %p in object\n", current); - GC_print_backtrace(current); - LOCK(); - } - last_back_trace_gc_no = GC_gc_no; -#endif -#ifdef MAKE_BACK_GRAPH - if (GC_print_back_height) { - GC_print_back_graph_stats(); - } -#endif - } -#endif - if (NULL == GC_fnlz_roots.finalize_now) { - UNLOCK(); - return; - } - - if (!GC_finalize_on_demand) { - unsigned char *pnested; - -#ifdef THREADS - if (EXPECT(GC_in_thread_creation, FALSE)) { - UNLOCK(); - return; - } -#endif - pnested = GC_check_finalizer_nested(); - UNLOCK(); - /* Skip GC_invoke_finalizers() if nested. */ - if (pnested != NULL) { - (void)GC_invoke_finalizers(); - *pnested = 0; /* Reset since no more finalizers or interrupted. */ -#ifndef THREADS - GC_ASSERT(NULL == GC_fnlz_roots.finalize_now || GC_interrupt_finalizers > 0); -#endif /* Otherwise GC can run concurrently and add more */ - } - return; - } - - /* These variables require synchronization to avoid data races. */ - if (last_finalizer_notification != GC_gc_no) { - notifier_fn = GC_finalizer_notifier; - last_finalizer_notification = GC_gc_no; - } - UNLOCK(); - if (notifier_fn != 0) - (*notifier_fn)(); /* Invoke the notifier */ -} - -#ifndef SMALL_CONFIG -#ifndef GC_LONG_REFS_NOT_NEEDED -#define IF_LONG_REFS_PRESENT_ELSE(x, y) (x) -#else -#define IF_LONG_REFS_PRESENT_ELSE(x, y) (y) -#endif - -GC_INNER void GC_print_finalization_stats(void) { - struct finalizable_object *fo; - unsigned long ready = 0; - - GC_log_printf( - "%lu finalization entries;" - " %lu/%lu short/long disappearing links alive\n", - (unsigned long)GC_fo_entries, - (unsigned long)GC_dl_hashtbl.entries, - (unsigned long)IF_LONG_REFS_PRESENT_ELSE( - GC_ll_hashtbl.entries, 0)); - - for (fo = GC_fnlz_roots.finalize_now; fo != NULL; fo = fo_next(fo)) - ++ready; - GC_log_printf( - "%lu finalization-ready objects;" - " %ld/%ld short/long links cleared\n", - ready, - (long)GC_old_dl_entries - (long)GC_dl_hashtbl.entries, - (long)IF_LONG_REFS_PRESENT_ELSE( - GC_old_ll_entries - GC_ll_hashtbl.entries, 0)); -} -#endif /* !SMALL_CONFIG */ - -#endif /* !GC_NO_FINALIZATION */ diff --git a/vendor/bdwgc/headers.c b/vendor/bdwgc/headers.c deleted file mode 100644 index 2db814b4..00000000 --- a/vendor/bdwgc/headers.c +++ /dev/null @@ -1,414 +0,0 @@ -/* - * Copyright 1988, 1989 Hans-J. Boehm, Alan J. Demers - * Copyright (c) 1991-1994 by Xerox Corporation. All rights reserved. - * Copyright (c) 1996 by Silicon Graphics. All rights reserved. - * Copyright (c) 2008-2022 Ivan Maidanski - * - * THIS MATERIAL IS PROVIDED AS IS, WITH ABSOLUTELY NO WARRANTY EXPRESSED - * OR IMPLIED. ANY USE IS AT YOUR OWN RISK. - * - * Permission is hereby granted to use or copy this program - * for any purpose, provided the above notices are retained on all copies. - * Permission to modify the code and to distribute modified code is granted, - * provided the above notices are retained, and a notice that the code was - * modified is included with the above copyright notice. - */ - -#ifdef _WIN32 -#define WIN32_LEAN_AND_MEAN -#include -#endif -#include "private/gc_priv.h" - -/* - * This implements: - * 1. allocation of heap block headers - * 2. A map from addresses to heap block addresses to heap block headers - * - * Access speed is crucial. We implement an index structure based on a 2 - * level tree. - */ - -/* Non-macro version of header location routine */ -GC_INNER hdr *GC_find_header(ptr_t h) { -#ifdef HASH_TL - hdr *result; - GET_HDR(h, result); - return result; -#else - return HDR_INNER(h); -#endif -} - -/* Handle a header cache miss. Returns a pointer to the */ -/* header corresponding to p, if p can possibly be a valid */ -/* object pointer, and 0 otherwise. */ -/* GUARANTEED to return 0 for a pointer past the first page */ -/* of an object unless both GC_all_interior_pointers is set */ -/* and p is in fact a valid object pointer. */ -/* Never returns a pointer to a free hblk. */ -GC_INNER hdr * -#ifdef PRINT_BLACK_LIST -GC_header_cache_miss(ptr_t p, hdr_cache_entry *hce, ptr_t source) -#else -GC_header_cache_miss(ptr_t p, hdr_cache_entry *hce) -#endif -{ - hdr *hhdr; - HC_MISS(); - GET_HDR(p, hhdr); - if (IS_FORWARDING_ADDR_OR_NIL(hhdr)) { - if (GC_all_interior_pointers) { - if (hhdr != 0) { - ptr_t current = p; - - current = (ptr_t)HBLKPTR(current); - do { - current = current - HBLKSIZE * (word)hhdr; - hhdr = HDR(current); - } while (IS_FORWARDING_ADDR_OR_NIL(hhdr)); - /* current points to near the start of the large object */ - if (hhdr->hb_flags & IGNORE_OFF_PAGE) - return 0; - if (HBLK_IS_FREE(hhdr) || p - current >= (ptrdiff_t)(hhdr->hb_sz)) { - GC_ADD_TO_BLACK_LIST_NORMAL(p, source); - /* Pointer past the end of the block */ - return 0; - } - } else { - GC_ADD_TO_BLACK_LIST_NORMAL(p, source); - /* And return zero: */ - } - GC_ASSERT(hhdr == 0 || !HBLK_IS_FREE(hhdr)); - return hhdr; - /* Pointers past the first page are probably too rare */ - /* to add them to the cache. We don't. */ - /* And correctness relies on the fact that we don't. */ - } else { - if (hhdr == 0) { - GC_ADD_TO_BLACK_LIST_NORMAL(p, source); - } - return 0; - } - } else { - if (HBLK_IS_FREE(hhdr)) { - GC_ADD_TO_BLACK_LIST_NORMAL(p, source); - return 0; - } else { - hce->block_addr = (word)(p) >> LOG_HBLKSIZE; - hce->hce_hdr = hhdr; - return hhdr; - } - } -} - -/* Routines to dynamically allocate collector data structures that will */ -/* never be freed. */ - -GC_INNER ptr_t GC_scratch_alloc(size_t bytes) { - ptr_t result = GC_scratch_free_ptr; - size_t bytes_to_get; - - GC_ASSERT(I_HOLD_LOCK()); - bytes = ROUNDUP_GRANULE_SIZE(bytes); - for (;;) { - GC_ASSERT((word)GC_scratch_end_ptr >= (word)result); - if (bytes <= (word)GC_scratch_end_ptr - (word)result) { - /* Unallocated space of scratch buffer has enough size. */ - GC_scratch_free_ptr = result + bytes; - return result; - } - - GC_ASSERT(GC_page_size != 0); - if (bytes >= MINHINCR * HBLKSIZE) { - bytes_to_get = ROUNDUP_PAGESIZE_IF_MMAP(bytes); - result = (ptr_t)GET_MEM(bytes_to_get); - if (result != NULL) { - GC_add_to_our_memory(result, bytes_to_get); - /* No update of scratch free area pointer; */ - /* get memory directly. */ -#ifdef USE_SCRATCH_LAST_END_PTR - /* Update end point of last obtained area (needed only */ - /* by GC_register_dynamic_libraries for some targets). */ - GC_scratch_last_end_ptr = result + bytes; -#endif - } - return result; - } - - bytes_to_get = ROUNDUP_PAGESIZE_IF_MMAP(MINHINCR * HBLKSIZE); - /* round up for safety */ - result = (ptr_t)GET_MEM(bytes_to_get); - if (EXPECT(NULL == result, FALSE)) { - WARN( - "Out of memory - trying to allocate requested amount" - " (%" WARN_PRIuPTR " bytes)...\n", - bytes); - bytes_to_get = ROUNDUP_PAGESIZE_IF_MMAP(bytes); - result = (ptr_t)GET_MEM(bytes_to_get); - if (result != NULL) { - GC_add_to_our_memory(result, bytes_to_get); -#ifdef USE_SCRATCH_LAST_END_PTR - GC_scratch_last_end_ptr = result + bytes; -#endif - } - return result; - } - - GC_add_to_our_memory(result, bytes_to_get); - /* TODO: some amount of unallocated space may remain unused forever */ - /* Update scratch area pointers and retry. */ - GC_scratch_free_ptr = result; - GC_scratch_end_ptr = GC_scratch_free_ptr + bytes_to_get; -#ifdef USE_SCRATCH_LAST_END_PTR - GC_scratch_last_end_ptr = GC_scratch_end_ptr; -#endif - } -} - -/* Return an uninitialized header */ -static hdr *alloc_hdr(void) { - hdr *result; - - GC_ASSERT(I_HOLD_LOCK()); - if (NULL == GC_hdr_free_list) { - result = (hdr *)GC_scratch_alloc(sizeof(hdr)); - } else { - result = GC_hdr_free_list; - GC_hdr_free_list = (hdr *)result->hb_next; - } - return result; -} - -GC_INLINE void free_hdr(hdr *hhdr) { - hhdr->hb_next = (struct hblk *)GC_hdr_free_list; - GC_hdr_free_list = hhdr; -} - -#ifdef COUNT_HDR_CACHE_HITS -/* Used for debugging/profiling (the symbols are externally visible). */ -word GC_hdr_cache_hits = 0; -word GC_hdr_cache_misses = 0; -#endif - -GC_INNER void GC_init_headers(void) { - unsigned i; - - GC_ASSERT(I_HOLD_LOCK()); - GC_ASSERT(NULL == GC_all_nils); - GC_all_nils = (bottom_index *)GC_scratch_alloc(sizeof(bottom_index)); - if (GC_all_nils == NULL) { - GC_err_printf("Insufficient memory for GC_all_nils\n"); - EXIT(); - } - BZERO(GC_all_nils, sizeof(bottom_index)); - for (i = 0; i < TOP_SZ; i++) { - GC_top_index[i] = GC_all_nils; - } -} - -/* Make sure that there is a bottom level index block for address addr. */ -/* Return FALSE on failure. */ -static GC_bool get_index(word addr) { - word hi = (word)(addr) >> (LOG_BOTTOM_SZ + LOG_HBLKSIZE); - bottom_index *r; - bottom_index *p; - bottom_index **prev; - bottom_index *pi; /* old_p */ - word i; - - GC_ASSERT(I_HOLD_LOCK()); -#ifdef HASH_TL - i = TL_HASH(hi); - - pi = GC_top_index[i]; - for (p = pi; p != GC_all_nils; p = p->hash_link) { - if (p->key == hi) return TRUE; - } -#else - if (GC_top_index[hi] != GC_all_nils) - return TRUE; - i = hi; -#endif - r = (bottom_index *)GC_scratch_alloc(sizeof(bottom_index)); - if (EXPECT(NULL == r, FALSE)) - return FALSE; - BZERO(r, sizeof(bottom_index)); - r->key = hi; -#ifdef HASH_TL - r->hash_link = pi; -#endif - - /* Add it to the list of bottom indices */ - prev = &GC_all_bottom_indices; /* pointer to p */ - pi = 0; /* bottom_index preceding p */ - while ((p = *prev) != 0 && p->key < hi) { - pi = p; - prev = &(p->asc_link); - } - r->desc_link = pi; - if (0 == p) { - GC_all_bottom_indices_end = r; - } else { - p->desc_link = r; - } - r->asc_link = p; - *prev = r; - - GC_top_index[i] = r; - return TRUE; -} - -/* Install a header for block h. */ -/* The header is uninitialized. */ -/* Returns the header or 0 on failure. */ -GC_INNER struct hblkhdr *GC_install_header(struct hblk *h) { - hdr *result; - - GC_ASSERT(I_HOLD_LOCK()); - if (EXPECT(!get_index((word)h), FALSE)) return NULL; - - result = alloc_hdr(); - if (EXPECT(result != NULL, TRUE)) { - SET_HDR(h, result); -#ifdef USE_MUNMAP - result->hb_last_reclaimed = (unsigned short)GC_gc_no; -#endif - } - return result; -} - -/* Set up forwarding counts for block h of size sz */ -GC_INNER GC_bool GC_install_counts(struct hblk *h, size_t sz /* bytes */) { - struct hblk *hbp; - - for (hbp = h; (word)hbp < (word)h + sz; hbp += BOTTOM_SZ) { - if (!get_index((word)hbp)) - return FALSE; - if ((word)hbp > GC_WORD_MAX - (word)BOTTOM_SZ * HBLKSIZE) - break; /* overflow of hbp+=BOTTOM_SZ is expected */ - } - if (!get_index((word)h + sz - 1)) - return FALSE; - for (hbp = h + 1; (word)hbp < (word)h + sz; hbp += 1) { - word i = HBLK_PTR_DIFF(hbp, h); - - SET_HDR(hbp, (hdr *)(i > MAX_JUMP ? MAX_JUMP : i)); - } - return TRUE; -} - -/* Remove the header for block h */ -GC_INNER void GC_remove_header(struct hblk *h) { - hdr **ha; - GET_HDR_ADDR(h, ha); - free_hdr(*ha); - *ha = 0; -} - -/* Remove forwarding counts for h */ -GC_INNER void GC_remove_counts(struct hblk *h, size_t sz /* bytes */) { - struct hblk *hbp; - - if (sz <= HBLKSIZE) return; - if (HDR(h + 1) == 0) { -#ifdef GC_ASSERTIONS - for (hbp = h + 2; (word)hbp < (word)h + sz; hbp++) - GC_ASSERT(HDR(hbp) == 0); -#endif - return; - } - - for (hbp = h + 1; (word)hbp < (word)h + sz; hbp += 1) { - SET_HDR(hbp, 0); - } -} - -GC_API void GC_CALL GC_apply_to_all_blocks(GC_walk_hblk_fn fn, - GC_word client_data) { - signed_word j; - bottom_index *index_p; - - for (index_p = GC_all_bottom_indices; index_p != 0; - index_p = index_p->asc_link) { - for (j = BOTTOM_SZ - 1; j >= 0;) { - if (!IS_FORWARDING_ADDR_OR_NIL(index_p->index[j])) { - if (!HBLK_IS_FREE(index_p->index[j])) { - (*fn)(((struct hblk *)(((index_p->key << LOG_BOTTOM_SZ) + (word)j) - << LOG_HBLKSIZE)), - client_data); - } - j--; - } else if (index_p->index[j] == 0) { - j--; - } else { - j -= (signed_word)(index_p->index[j]); - } - } - } -} - -GC_INNER struct hblk *GC_next_block(struct hblk *h, GC_bool allow_free) { - REGISTER bottom_index *bi; - REGISTER word j = ((word)h >> LOG_HBLKSIZE) & (BOTTOM_SZ - 1); - - GC_ASSERT(I_HOLD_LOCK()); - GET_BI(h, bi); - if (bi == GC_all_nils) { - REGISTER word hi = (word)h >> (LOG_BOTTOM_SZ + LOG_HBLKSIZE); - - bi = GC_all_bottom_indices; - while (bi != 0 && bi->key < hi) bi = bi->asc_link; - j = 0; - } - - while (bi != 0) { - while (j < BOTTOM_SZ) { - hdr *hhdr = bi->index[j]; - if (IS_FORWARDING_ADDR_OR_NIL(hhdr)) { - j++; - } else { - if (allow_free || !HBLK_IS_FREE(hhdr)) { - return (struct hblk *)(((bi->key << LOG_BOTTOM_SZ) + j) << LOG_HBLKSIZE); - } else { - j += divHBLKSZ(hhdr->hb_sz); - } - } - } - j = 0; - bi = bi->asc_link; - } - return NULL; -} - -GC_INNER struct hblk *GC_prev_block(struct hblk *h) { - bottom_index *bi; - signed_word j = ((word)h >> LOG_HBLKSIZE) & (BOTTOM_SZ - 1); - - GC_ASSERT(I_HOLD_LOCK()); - GET_BI(h, bi); - if (bi == GC_all_nils) { - word hi = (word)h >> (LOG_BOTTOM_SZ + LOG_HBLKSIZE); - - bi = GC_all_bottom_indices_end; - while (bi != NULL && bi->key > hi) - bi = bi->desc_link; - j = BOTTOM_SZ - 1; - } - for (; bi != NULL; bi = bi->desc_link) { - while (j >= 0) { - hdr *hhdr = bi->index[j]; - - if (NULL == hhdr) { - --j; - } else if (IS_FORWARDING_ADDR_OR_NIL(hhdr)) { - j -= (signed_word)hhdr; - } else { - return (struct hblk *)(((bi->key << LOG_BOTTOM_SZ) + j) - << LOG_HBLKSIZE); - } - } - j = BOTTOM_SZ - 1; - } - return NULL; -} diff --git a/vendor/bdwgc/mach_dep.c b/vendor/bdwgc/mach_dep.c deleted file mode 100644 index 807b497e..00000000 --- a/vendor/bdwgc/mach_dep.c +++ /dev/null @@ -1,448 +0,0 @@ -/* - * Copyright 1988, 1989 Hans-J. Boehm, Alan J. Demers - * Copyright (c) 1991-1994 by Xerox Corporation. All rights reserved. - * Copyright (c) 2008-2022 Ivan Maidanski - * - * THIS MATERIAL IS PROVIDED AS IS, WITH ABSOLUTELY NO WARRANTY EXPRESSED - * OR IMPLIED. ANY USE IS AT YOUR OWN RISK. - * - * Permission is hereby granted to use or copy this program - * for any purpose, provided the above notices are retained on all copies. - * Permission to modify the code and to distribute modified code is granted, - * provided the above notices are retained, and a notice that the code was - * modified is included with the above copyright notice. - */ - -#ifdef _WIN32 -#define WIN32_LEAN_AND_MEAN -#include -#endif -#include "private/gc_priv.h" - -#if !defined(PLATFORM_MACH_DEP) && !defined(SN_TARGET_PSP2) - -#ifdef AMIGA -#ifndef __GNUC__ -#include -#else -#include -#endif -#endif - -#ifdef E2K -#include -#include -#include -#include - -GC_INNER size_t GC_get_procedure_stack(ptr_t buf, size_t buf_sz) { - unsigned long long new_sz; - - GC_ASSERT(0 == buf_sz || buf != NULL); - for (;;) { - unsigned long long stack_ofs; - - new_sz = 0; - if (syscall(__NR_access_hw_stacks, E2K_GET_PROCEDURE_STACK_SIZE, - NULL, NULL, 0, &new_sz) == -1) { - if (errno != EAGAIN) - ABORT_ARG1("Cannot get size of procedure stack", - ": errno= %d", errno); - continue; - } - GC_ASSERT(new_sz > 0 && new_sz % sizeof(word) == 0); - if (new_sz > buf_sz) - break; - /* Immediately read the stack right after checking its size. */ - stack_ofs = 0; - if (syscall(__NR_access_hw_stacks, E2K_READ_PROCEDURE_STACK_EX, - &stack_ofs, buf, new_sz, NULL) != -1) - break; - if (errno != EAGAIN) - ABORT_ARG2("Cannot read procedure stack", - ": new_sz= %lu, errno= %d", (unsigned long)new_sz, errno); - } - return (size_t)new_sz; -} - -ptr_t GC_save_regs_in_stack(void) { - __asm__ __volatile__("flushr"); - return NULL; -} - -GC_INNER ptr_t GC_mmap_procedure_stack_buf(size_t aligned_sz) { - void *buf = mmap(NULL, aligned_sz, PROT_READ | PROT_WRITE, - MAP_PRIVATE | MAP_ANON, 0 /* fd */, 0 /* offset */); - if (MAP_FAILED == buf) - ABORT_ARG2("Could not map memory for procedure stack", - ": requested %lu bytes, errno= %d", - (unsigned long)aligned_sz, errno); - return (ptr_t)buf; -} - -GC_INNER void GC_unmap_procedure_stack_buf(ptr_t buf, size_t sz) { - if (munmap(buf, ROUNDUP_PAGESIZE(sz)) == -1) - ABORT_ARG1("munmap failed (for procedure stack space)", - ": errno= %d", errno); -} - -#ifdef THREADS -GC_INNER size_t GC_alloc_and_get_procedure_stack(ptr_t *pbuf) { - /* TODO: support saving from non-zero ofs in stack */ - ptr_t buf = NULL; - size_t new_sz, buf_sz; - - GC_ASSERT(I_HOLD_LOCK()); - for (buf_sz = 0;; buf_sz = new_sz) { - new_sz = GC_get_procedure_stack(buf, buf_sz); - if (new_sz <= buf_sz) break; - - if (EXPECT(buf != NULL, FALSE)) - GC_INTERNAL_FREE(buf); - buf = (ptr_t)GC_INTERNAL_MALLOC_IGNORE_OFF_PAGE(new_sz, PTRFREE); - if (NULL == buf) - ABORT("Could not allocate memory for procedure stack"); - } - *pbuf = buf; - return new_sz; -} -#endif /* THREADS */ -#endif /* E2K */ - -#if defined(MACOS) && defined(__MWERKS__) - -#if defined(POWERPC) - -#define NONVOLATILE_GPR_COUNT 19 -struct ppc_registers { - unsigned long gprs[NONVOLATILE_GPR_COUNT]; /* R13-R31 */ -}; -typedef struct ppc_registers ppc_registers; - -#if defined(CPPCHECK) -void getRegisters(ppc_registers *regs); -#else -asm static void getRegisters(register ppc_registers *regs) { - stmw r13, regs->gprs /* save R13-R31 */ - blr -} -#endif - -static void PushMacRegisters(void) { - ppc_registers regs; - int i; - getRegisters(®s); - for (i = 0; i < NONVOLATILE_GPR_COUNT; i++) - GC_push_one(regs.gprs[i]); -} - -#else /* M68K */ - -asm static void PushMacRegisters(void) { - sub.w #4, sp /* reserve space for one parameter */ - move.l a2, - (sp) - jsr GC_push_one - move.l a3, - (sp) - jsr GC_push_one - move.l a4, - (sp) - jsr GC_push_one -#if !__option(a6frames) - /* perhaps a6 should be pushed if stack frames are not being used */ - move.l a6, - (sp) - jsr GC_push_one -#endif - /* skip a5 (globals), a6 (frame pointer), and a7 (stack pointer) */ - move.l d2, - (sp) - jsr GC_push_one - move.l d3, - (sp) - jsr GC_push_one - move.l d4, - (sp) - jsr GC_push_one - move.l d5, - (sp) - jsr GC_push_one - move.l d6, - (sp) - jsr GC_push_one - move.l d7, - (sp) - jsr GC_push_one - add.w #4, - sp /* fix stack */ - rts -} - -#endif /* M68K */ - -#endif /* MACOS && __MWERKS__ */ - -#if defined(IA64) && !defined(THREADS) -/* Value returned from register flushing routine (ar.bsp). */ -GC_INNER ptr_t GC_save_regs_ret_val = NULL; -#endif - -/* Routine to mark from registers that are preserved by the C compiler. */ -/* This must be ported to every new architecture. It is not optional, */ -/* and should not be used on platforms that are either UNIX-like, or */ -/* require thread support. */ - -#undef HAVE_PUSH_REGS - -#if defined(USE_ASM_PUSH_REGS) -#define HAVE_PUSH_REGS -#else /* No asm implementation */ - -#ifdef STACK_NOT_SCANNED -void GC_push_regs(void) { - /* empty */ -} -#define HAVE_PUSH_REGS - -#elif defined(M68K) && defined(AMIGA) -/* This function is not static because it could also be */ -/* erroneously defined in .S file, so this error would be caught */ -/* by the linker. */ -void GC_push_regs(void) { - /* AMIGA - could be replaced by generic code */ - /* a0, a1, d0 and d1 are caller save */ - -#ifdef __GNUC__ - asm("subq.w &0x4,%sp"); /* allocate word on top of stack */ - - asm("mov.l %a2,(%sp)"); - asm("jsr _GC_push_one"); - asm("mov.l %a3,(%sp)"); - asm("jsr _GC_push_one"); - asm("mov.l %a4,(%sp)"); - asm("jsr _GC_push_one"); - asm("mov.l %a5,(%sp)"); - asm("jsr _GC_push_one"); - asm("mov.l %a6,(%sp)"); - asm("jsr _GC_push_one"); - /* Skip frame pointer and stack pointer */ - asm("mov.l %d2,(%sp)"); - asm("jsr _GC_push_one"); - asm("mov.l %d3,(%sp)"); - asm("jsr _GC_push_one"); - asm("mov.l %d4,(%sp)"); - asm("jsr _GC_push_one"); - asm("mov.l %d5,(%sp)"); - asm("jsr _GC_push_one"); - asm("mov.l %d6,(%sp)"); - asm("jsr _GC_push_one"); - asm("mov.l %d7,(%sp)"); - asm("jsr _GC_push_one"); - - asm("addq.w &0x4,%sp"); /* put stack back where it was */ -#else /* !__GNUC__ */ - GC_push_one(getreg(REG_A2)); - GC_push_one(getreg(REG_A3)); -#ifndef __SASC - /* Can probably be changed to #if 0 -Kjetil M. (a4=globals) */ - GC_push_one(getreg(REG_A4)); -#endif - GC_push_one(getreg(REG_A5)); - GC_push_one(getreg(REG_A6)); - /* Skip stack pointer */ - GC_push_one(getreg(REG_D2)); - GC_push_one(getreg(REG_D3)); - GC_push_one(getreg(REG_D4)); - GC_push_one(getreg(REG_D5)); - GC_push_one(getreg(REG_D6)); - GC_push_one(getreg(REG_D7)); -#endif /* !__GNUC__ */ -} -#define HAVE_PUSH_REGS - -#elif defined(MACOS) - -#if defined(M68K) && defined(THINK_C) && !defined(CPPCHECK) -#define PushMacReg(reg) \ - move.l reg, (sp) \ - jsr GC_push_one -void GC_push_regs(void) { - asm { - sub.w #4,sp ; reserve space for one parameter. - PushMacReg(a2); - PushMacReg(a3); - PushMacReg(a4); - ; skip a5 (globals), a6 (frame pointer), and a7 (stack pointer) - PushMacReg(d2); - PushMacReg(d3); - PushMacReg(d4); - PushMacReg(d5); - PushMacReg(d6); - PushMacReg(d7); - add.w #4,sp ; fix stack. - } -} -#define HAVE_PUSH_REGS -#undef PushMacReg -#elif defined(__MWERKS__) -void GC_push_regs(void) { - PushMacRegisters(); -} -#define HAVE_PUSH_REGS -#endif /* __MWERKS__ */ -#endif /* MACOS */ - -#endif /* !USE_ASM_PUSH_REGS */ - -#if defined(HAVE_PUSH_REGS) && defined(THREADS) -#error GC_push_regs cannot be used with threads -/* Would fail for GC_do_blocking. There are probably other safety */ -/* issues. */ -#undef HAVE_PUSH_REGS -#endif - -#if !defined(HAVE_PUSH_REGS) && defined(UNIX_LIKE) -#include -#ifndef NO_GETCONTEXT -#if defined(DARWIN) && (MAC_OS_X_VERSION_MAX_ALLOWED >= 1060 /*MAC_OS_X_VERSION_10_6*/) -#include -#else -#include -#endif /* !DARWIN */ -#ifdef GETCONTEXT_FPU_EXCMASK_BUG -#include -#endif -#endif -#endif /* !HAVE_PUSH_REGS */ - -/* Ensure that either registers are pushed, or callee-save registers */ -/* are somewhere on the stack, and then call fn(arg, ctxt). */ -/* ctxt is either a pointer to a ucontext_t we generated, or NULL. */ -/* Could be called with or w/o the GC lock held; could be called from */ -/* a signal handler as well. */ -GC_ATTR_NO_SANITIZE_ADDR -GC_INNER void GC_with_callee_saves_pushed(void (*fn)(ptr_t, void *), - volatile ptr_t arg) { - volatile int dummy; - volatile ptr_t context = 0; - -#if defined(HAVE_PUSH_REGS) - GC_push_regs(); -#elif defined(EMSCRIPTEN) - /* No-op, "registers" are pushed in GC_push_other_roots(). */ -#else -#if defined(UNIX_LIKE) && !defined(NO_GETCONTEXT) - /* Older versions of Darwin seem to lack getcontext(). */ - /* ARM and MIPS Linux often doesn't support a real */ - /* getcontext(). */ - static signed char getcontext_works = 0; /* (-1) - broken, 1 - works */ - ucontext_t ctxt; -#ifdef GETCONTEXT_FPU_EXCMASK_BUG - /* Workaround a bug (clearing the FPU exception mask) in */ - /* getcontext on Linux/x64. */ -#ifdef X86_64 - /* We manipulate FPU control word here just not to force the */ - /* client application to use -lm linker option. */ - unsigned short old_fcw; - -#if defined(CPPCHECK) - GC_noop1((word)&old_fcw); -#endif - __asm__ __volatile__("fstcw %0" - : "=m"(*&old_fcw)); -#else - int except_mask = fegetexcept(); -#endif -#endif - - if (getcontext_works >= 0) { - if (getcontext(&ctxt) < 0) { - WARN( - "getcontext failed:" - " using another register retrieval method...\n", - 0); - /* getcontext() is broken, do not try again. */ - /* E.g., to workaround a bug in Docker ubuntu_32bit. */ - } else { - context = (ptr_t)&ctxt; - } - if (EXPECT(0 == getcontext_works, FALSE)) - getcontext_works = context != NULL ? 1 : -1; - } -#ifdef GETCONTEXT_FPU_EXCMASK_BUG -#ifdef X86_64 - __asm__ __volatile__("fldcw %0" - : - : "m"(*&old_fcw)); - { - unsigned mxcsr; - /* And now correct the exception mask in SSE MXCSR. */ - __asm__ __volatile__("stmxcsr %0" - : "=m"(*&mxcsr)); - mxcsr = (mxcsr & ~(FE_ALL_EXCEPT << 7)) | - ((old_fcw & FE_ALL_EXCEPT) << 7); - __asm__ __volatile__("ldmxcsr %0" - : - : "m"(*&mxcsr)); - } -#else /* !X86_64 */ - if (feenableexcept(except_mask) < 0) - ABORT("feenableexcept failed"); -#endif -#endif /* GETCONTEXT_FPU_EXCMASK_BUG */ -#if defined(E2K) || defined(IA64) || defined(SPARC) - /* On a register window machine, we need to save register */ - /* contents on the stack for this to work. This may already be */ - /* subsumed by the getcontext() call. */ -#if defined(IA64) && !defined(THREADS) - GC_save_regs_ret_val = -#endif - GC_save_regs_in_stack(); -#endif - if (NULL == context) /* getcontext failed */ -#endif /* !NO_GETCONTEXT */ - { -#if defined(HAVE_BUILTIN_UNWIND_INIT) - /* This was suggested by Richard Henderson as the way to */ - /* force callee-save registers and register windows onto */ - /* the stack. */ - __builtin_unwind_init(); -#elif defined(NO_CRT) && defined(MSWIN32) - CONTEXT ctx; - RtlCaptureContext(&ctx); -#else - /* Generic code */ - /* The idea is due to Parag Patel at HP. */ - /* We're not sure whether he would like */ - /* to be acknowledged for it or not. */ - jmp_buf regs; - word *i = (word *)®s; - ptr_t lim = (ptr_t)(®s) + sizeof(regs); - - /* setjmp doesn't always clear all of the buffer. */ - /* That tends to preserve garbage. Clear it. */ - for (; (word)i < (word)lim; i++) { - *i = 0; - } -#if defined(MSWIN32) || defined(MSWINCE) || defined(UTS4) || defined(OS2) || defined(CX_UX) || defined(__CC_ARM) || defined(LINUX) || defined(EWS4800) || defined(RTEMS) - (void)setjmp(regs); -#else - (void)_setjmp(regs); - /* We don't want to mess with signals. According to */ - /* SUSV3, setjmp() may or may not save signal mask. */ - /* _setjmp won't, but is less portable. */ -#endif -#endif /* !HAVE_BUILTIN_UNWIND_INIT */ - } -#endif /* !HAVE_PUSH_REGS */ - /* TODO: context here is sometimes just zero. At the moment, the */ - /* callees don't really need it. */ - fn(arg, (/* no volatile */ void *)context); - /* Strongly discourage the compiler from treating the above */ - /* as a tail-call, since that would pop the register */ - /* contents before we get a chance to look at them. */ - GC_noop1(COVERT_DATAFLOW(&dummy)); -} - -#endif /* !PLATFORM_MACH_DEP && !SN_TARGET_PSP2 */ diff --git a/vendor/bdwgc/malloc.c b/vendor/bdwgc/malloc.c deleted file mode 100644 index 4a32a553..00000000 --- a/vendor/bdwgc/malloc.c +++ /dev/null @@ -1,634 +0,0 @@ -/* - * Copyright 1988, 1989 Hans-J. Boehm, Alan J. Demers - * Copyright (c) 1991-1994 by Xerox Corporation. All rights reserved. - * Copyright (c) 1999-2004 Hewlett-Packard Development Company, L.P. - * Copyright (c) 2008-2022 Ivan Maidanski - * - * THIS MATERIAL IS PROVIDED AS IS, WITH ABSOLUTELY NO WARRANTY EXPRESSED - * OR IMPLIED. ANY USE IS AT YOUR OWN RISK. - * - * Permission is hereby granted to use or copy this program - * for any purpose, provided the above notices are retained on all copies. - * Permission to modify the code and to distribute modified code is granted, - * provided the above notices are retained, and a notice that the code was - * modified is included with the above copyright notice. - */ - -#include - -#ifdef _WIN32 -#define WIN32_LEAN_AND_MEAN -#include -#endif -#include "private/gc_priv.h" - -/* Allocate reclaim list for the kind. Returns TRUE on success. */ -STATIC GC_bool GC_alloc_reclaim_list(struct obj_kind *kind) { - struct hblk **result; - - GC_ASSERT(I_HOLD_LOCK()); - result = (struct hblk **)GC_scratch_alloc( - (MAXOBJGRANULES + 1) * sizeof(struct hblk *)); - if (EXPECT(NULL == result, FALSE)) return FALSE; - - BZERO(result, (MAXOBJGRANULES + 1) * sizeof(struct hblk *)); - kind->ok_reclaim_list = result; - return TRUE; -} - -GC_INNER ptr_t GC_alloc_large(size_t lb, int k, unsigned flags, - size_t align_m1) { - struct hblk *h; - size_t n_blocks; /* includes alignment */ - ptr_t result = NULL; - GC_bool retry = FALSE; - - GC_ASSERT(I_HOLD_LOCK()); - lb = ROUNDUP_GRANULE_SIZE(lb); - n_blocks = OBJ_SZ_TO_BLOCKS_CHECKED(SIZET_SAT_ADD(lb, align_m1)); - if (!EXPECT(GC_is_initialized, TRUE)) { - UNLOCK(); /* just to unset GC_lock_holder */ - GC_init(); - LOCK(); - } - /* Do our share of marking work. */ - if (GC_incremental && !GC_dont_gc) { - ENTER_GC(); - GC_collect_a_little_inner((int)n_blocks); - EXIT_GC(); - } - - h = GC_allochblk(lb, k, flags, align_m1); -#ifdef USE_MUNMAP - if (NULL == h) { - GC_merge_unmapped(); - h = GC_allochblk(lb, k, flags, align_m1); - } -#endif - while (0 == h && GC_collect_or_expand(n_blocks, flags, retry)) { - h = GC_allochblk(lb, k, flags, align_m1); - retry = TRUE; - } - if (EXPECT(h != NULL, TRUE)) { - GC_bytes_allocd += lb; - if (lb > HBLKSIZE) { - GC_large_allocd_bytes += HBLKSIZE * OBJ_SZ_TO_BLOCKS(lb); - if (GC_large_allocd_bytes > GC_max_large_allocd_bytes) - GC_max_large_allocd_bytes = GC_large_allocd_bytes; - } - /* FIXME: Do we need some way to reset GC_max_large_allocd_bytes? */ - result = h->hb_body; - GC_ASSERT(((word)result & align_m1) == 0); - } - return result; -} - -/* Allocate a large block of size lb bytes. Clear if appropriate. */ -/* EXTRA_BYTES were already added to lb. Update GC_bytes_allocd. */ -STATIC ptr_t GC_alloc_large_and_clear(size_t lb, int k, unsigned flags) { - ptr_t result; - - GC_ASSERT(I_HOLD_LOCK()); - result = GC_alloc_large(lb, k, flags, 0 /* align_m1 */); - if (EXPECT(result != NULL, TRUE) && (GC_debugging_started || GC_obj_kinds[k].ok_init)) { - /* Clear the whole block, in case of GC_realloc call. */ - BZERO(result, HBLKSIZE * OBJ_SZ_TO_BLOCKS(lb)); - } - return result; -} - -/* Fill in additional entries in GC_size_map, including the i-th one. */ -/* Note that a filled in section of the array ending at n always */ -/* has the length of at least n/4. */ -STATIC void GC_extend_size_map(size_t i) { - size_t orig_granule_sz = ALLOC_REQUEST_GRANS(i); - size_t granule_sz; - size_t byte_sz = GRANULES_TO_BYTES(orig_granule_sz); - /* The size we try to preserve. */ - /* Close to i, unless this would */ - /* introduce too many distinct sizes. */ - size_t smaller_than_i = byte_sz - (byte_sz >> 3); - size_t low_limit; /* The lowest indexed entry we initialize. */ - size_t number_of_objs; - - GC_ASSERT(I_HOLD_LOCK()); - GC_ASSERT(0 == GC_size_map[i]); - if (0 == GC_size_map[smaller_than_i]) { - low_limit = byte_sz - (byte_sz >> 2); /* much smaller than i */ - granule_sz = orig_granule_sz; - while (GC_size_map[low_limit] != 0) - low_limit++; - } else { - low_limit = smaller_than_i + 1; - while (GC_size_map[low_limit] != 0) - low_limit++; - - granule_sz = ALLOC_REQUEST_GRANS(low_limit); - granule_sz += granule_sz >> 3; - if (granule_sz < orig_granule_sz) - granule_sz = orig_granule_sz; - } - - /* For these larger sizes, we use an even number of granules. */ - /* This makes it easier to, e.g., construct a 16-byte-aligned */ - /* allocator even if GRANULE_BYTES is 8. */ - granule_sz = (granule_sz + 1) & ~1; - if (granule_sz > MAXOBJGRANULES) - granule_sz = MAXOBJGRANULES; - - /* If we can fit the same number of larger objects in a block, do so. */ - number_of_objs = HBLK_GRANULES / granule_sz; - GC_ASSERT(number_of_objs != 0); - granule_sz = (HBLK_GRANULES / number_of_objs) & ~1; - - byte_sz = GRANULES_TO_BYTES(granule_sz) - EXTRA_BYTES; - /* We may need one extra byte; do not always */ - /* fill in GC_size_map[byte_sz]. */ - - for (; low_limit <= byte_sz; low_limit++) - GC_size_map[low_limit] = granule_sz; -} - -STATIC void *GC_generic_malloc_inner_small(size_t lb, int k) { - struct obj_kind *kind = GC_obj_kinds + k; - size_t lg = GC_size_map[lb]; - void **opp = &(kind->ok_freelist[lg]); - void *op = *opp; - - GC_ASSERT(I_HOLD_LOCK()); - if (EXPECT(NULL == op, FALSE)) { - if (lg == 0) { - if (!EXPECT(GC_is_initialized, TRUE)) { - UNLOCK(); /* just to unset GC_lock_holder */ - GC_init(); - LOCK(); - lg = GC_size_map[lb]; - } - if (0 == lg) { - GC_extend_size_map(lb); - lg = GC_size_map[lb]; - GC_ASSERT(lg != 0); - } - /* Retry */ - opp = &(kind->ok_freelist[lg]); - op = *opp; - } - if (NULL == op) { - if (NULL == kind->ok_reclaim_list && !GC_alloc_reclaim_list(kind)) - return NULL; - op = GC_allocobj(lg, k); - if (NULL == op) return NULL; - } - } - *opp = obj_link(op); - obj_link(op) = NULL; - GC_bytes_allocd += GRANULES_TO_BYTES((word)lg); - return op; -} - -GC_INNER void *GC_generic_malloc_inner(size_t lb, int k, unsigned flags) { - GC_ASSERT(I_HOLD_LOCK()); - GC_ASSERT(k < MAXOBJKINDS); - if (SMALL_OBJ(lb)) { - return GC_generic_malloc_inner_small(lb, k); - } - - return GC_alloc_large_and_clear(ADD_EXTRA_BYTES(lb), k, flags); -} - -#ifdef GC_COLLECT_AT_MALLOC -/* Parameter to force GC at every malloc of size greater or equal to */ -/* the given value. This might be handy during debugging. */ -#if defined(CPPCHECK) -size_t GC_dbg_collect_at_malloc_min_lb = 16 * 1024; /* e.g. */ -#else -size_t GC_dbg_collect_at_malloc_min_lb = (GC_COLLECT_AT_MALLOC); -#endif -#endif - -GC_INNER void *GC_generic_malloc_aligned(size_t lb, int k, unsigned flags, - size_t align_m1) { - void *result; - - GC_ASSERT(k < MAXOBJKINDS); - if (EXPECT(get_have_errors(), FALSE)) - GC_print_all_errors(); - GC_INVOKE_FINALIZERS(); - GC_DBG_COLLECT_AT_MALLOC(lb); - if (SMALL_OBJ(lb) && EXPECT(align_m1 < GRANULE_BYTES, TRUE)) { - LOCK(); - result = GC_generic_malloc_inner_small(lb, k); - UNLOCK(); - } else { - size_t lg; - size_t lb_rounded; - GC_bool init; - - if (EXPECT(0 == lb, FALSE)) lb = 1; - lg = ALLOC_REQUEST_GRANS(lb); - lb_rounded = GRANULES_TO_BYTES(lg); - init = GC_obj_kinds[k].ok_init; - if (EXPECT(align_m1 < GRANULE_BYTES, TRUE)) { - align_m1 = 0; - } else if (align_m1 < HBLKSIZE) { - align_m1 = HBLKSIZE - 1; - } - LOCK(); - result = GC_alloc_large(lb_rounded, k, flags, align_m1); - if (EXPECT(result != NULL, TRUE)) { - if (GC_debugging_started -#ifndef THREADS - || init -#endif - ) { - BZERO(result, HBLKSIZE * OBJ_SZ_TO_BLOCKS(lb_rounded)); - } else { -#ifdef THREADS - GC_ASSERT(GRANULES_TO_WORDS(lg) >= 2); - /* Clear any memory that might be used for GC descriptors */ - /* before we release the lock. */ - ((word *)result)[0] = 0; - ((word *)result)[1] = 0; - ((word *)result)[GRANULES_TO_WORDS(lg) - 1] = 0; - ((word *)result)[GRANULES_TO_WORDS(lg) - 2] = 0; -#endif - } - } - UNLOCK(); -#ifdef THREADS - if (init && !GC_debugging_started && result != NULL) { - /* Clear the rest (i.e. excluding the initial 2 words). */ - BZERO((word *)result + 2, - HBLKSIZE * OBJ_SZ_TO_BLOCKS(lb_rounded) - 2 * sizeof(word)); - } -#endif - } - if (EXPECT(NULL == result, FALSE)) - result = (*GC_get_oom_fn())(lb); /* might be misaligned */ - return result; -} - -GC_API GC_ATTR_MALLOC void *GC_CALL GC_generic_malloc(size_t lb, int k) { - return GC_generic_malloc_aligned(lb, k, 0 /* flags */, 0 /* align_m1 */); -} - -GC_API GC_ATTR_MALLOC void *GC_CALL GC_malloc_kind_global(size_t lb, int k) { - GC_ASSERT(k < MAXOBJKINDS); - if (SMALL_OBJ(lb)) { - void *op; - void **opp; - size_t lg; - - GC_DBG_COLLECT_AT_MALLOC(lb); - LOCK(); - lg = GC_size_map[lb]; - opp = &GC_obj_kinds[k].ok_freelist[lg]; - op = *opp; - if (EXPECT(op != NULL, TRUE)) { - if (k == PTRFREE) { - *opp = obj_link(op); - } else { - GC_ASSERT(0 == obj_link(op) || ((word)obj_link(op) <= (word)GC_greatest_plausible_heap_addr && (word)obj_link(op) >= (word)GC_least_plausible_heap_addr)); - *opp = obj_link(op); - obj_link(op) = 0; - } - GC_bytes_allocd += GRANULES_TO_BYTES((word)lg); - UNLOCK(); - return op; - } - UNLOCK(); - } - - /* We make the GC_clear_stack() call a tail one, hoping to get more */ - /* of the stack. */ - return GC_clear_stack(GC_generic_malloc_aligned(lb, k, 0 /* flags */, 0)); -} - -#if defined(THREADS) && !defined(THREAD_LOCAL_ALLOC) -GC_API GC_ATTR_MALLOC void *GC_CALL GC_malloc_kind(size_t lb, int k) { - return GC_malloc_kind_global(lb, k); -} -#endif - -/* Allocate lb bytes of atomic (pointer-free) data. */ -GC_API GC_ATTR_MALLOC void *GC_CALL GC_malloc_atomic(size_t lb) { - return GC_malloc_kind(lb, PTRFREE); -} - -/* Allocate lb bytes of composite (pointerful) data. */ -GC_API GC_ATTR_MALLOC void *GC_CALL GC_malloc(size_t lb) { - return GC_malloc_kind(lb, NORMAL); -} - -GC_API GC_ATTR_MALLOC void *GC_CALL GC_generic_malloc_uncollectable( - size_t lb, int k) { - void *op; - size_t lb_orig = lb; - - GC_ASSERT(k < MAXOBJKINDS); - if (EXTRA_BYTES != 0 && EXPECT(lb != 0, TRUE)) lb--; - /* We do not need the extra byte, since this will */ - /* not be collected anyway. */ - - if (SMALL_OBJ(lb)) { - void **opp; - size_t lg; - - if (EXPECT(get_have_errors(), FALSE)) - GC_print_all_errors(); - GC_INVOKE_FINALIZERS(); - GC_DBG_COLLECT_AT_MALLOC(lb_orig); - LOCK(); - lg = GC_size_map[lb]; - opp = &GC_obj_kinds[k].ok_freelist[lg]; - op = *opp; - if (EXPECT(op != NULL, TRUE)) { - *opp = obj_link(op); - obj_link(op) = 0; - GC_bytes_allocd += GRANULES_TO_BYTES((word)lg); - /* Mark bit was already set on free list. It will be */ - /* cleared only temporarily during a collection, as a */ - /* result of the normal free list mark bit clearing. */ - GC_non_gc_bytes += GRANULES_TO_BYTES((word)lg); - } else { - op = GC_generic_malloc_inner_small(lb, k); - if (NULL == op) { - GC_oom_func oom_fn = GC_oom_fn; - UNLOCK(); - return (*oom_fn)(lb_orig); - } - /* For small objects, the free lists are completely marked. */ - } - GC_ASSERT(GC_is_marked(op)); - UNLOCK(); - } else { - op = GC_generic_malloc_aligned(lb, k, 0 /* flags */, 0 /* align_m1 */); - if (op /* != NULL */) { /* CPPCHECK */ - hdr *hhdr = HDR(op); - - GC_ASSERT(HBLKDISPL(op) == 0); /* large block */ - /* We don't need the lock here, since we have an undisguised */ - /* pointer. We do need to hold the lock while we adjust */ - /* mark bits. */ - LOCK(); - set_mark_bit_from_hdr(hhdr, 0); /* Only object. */ -#ifndef THREADS - GC_ASSERT(hhdr->hb_n_marks == 0); - /* This is not guaranteed in the multi-threaded case */ - /* because the counter could be updated before locking. */ -#endif - hhdr->hb_n_marks = 1; - UNLOCK(); - } - } - return op; -} - -/* Allocate lb bytes of pointerful, traced, but not collectible data. */ -GC_API GC_ATTR_MALLOC void *GC_CALL GC_malloc_uncollectable(size_t lb) { - return GC_generic_malloc_uncollectable(lb, UNCOLLECTABLE); -} - -#ifdef GC_ATOMIC_UNCOLLECTABLE -/* Allocate lb bytes of pointer-free, untraced, uncollectible data */ -/* This is normally roughly equivalent to the system malloc. */ -/* But it may be useful if malloc is redefined. */ -GC_API GC_ATTR_MALLOC void *GC_CALL -GC_malloc_atomic_uncollectable(size_t lb) { - return GC_generic_malloc_uncollectable(lb, AUNCOLLECTABLE); -} -#endif /* GC_ATOMIC_UNCOLLECTABLE */ - -#if defined(REDIRECT_MALLOC) && !defined(REDIRECT_MALLOC_IN_HEADER) - -#ifndef MSWINCE -#include -#endif - -/* Avoid unnecessary nested procedure calls here, by #defining some */ -/* malloc replacements. Otherwise we end up saving a meaningless */ -/* return address in the object. It also speeds things up, but it is */ -/* admittedly quite ugly. */ -#define GC_debug_malloc_replacement(lb) GC_debug_malloc(lb, GC_DBG_EXTRAS) - -#if defined(CPPCHECK) -#define REDIRECT_MALLOC_F GC_malloc /* e.g. */ -#else -#define REDIRECT_MALLOC_F REDIRECT_MALLOC -#endif - -void *malloc(size_t lb) { - /* It might help to manually inline the GC_malloc call here. */ - /* But any decent compiler should reduce the extra procedure call */ - /* to at most a jump instruction in this case. */ -#if defined(I386) && defined(GC_SOLARIS_THREADS) - /* Thread initialization can call malloc before we are ready for. */ - /* It is not clear that this is enough to help matters. */ - /* The thread implementation may well call malloc at other */ - /* inopportune times. */ - if (!EXPECT(GC_is_initialized, TRUE)) return sbrk(lb); -#endif - return (void *)REDIRECT_MALLOC_F(lb); -} - -#if defined(GC_LINUX_THREADS) -#ifdef HAVE_LIBPTHREAD_SO -STATIC ptr_t GC_libpthread_start = NULL; -STATIC ptr_t GC_libpthread_end = NULL; -#endif -STATIC ptr_t GC_libld_start = NULL; -STATIC ptr_t GC_libld_end = NULL; - -STATIC void GC_init_lib_bounds(void) { - IF_CANCEL(int cancel_state;) - - DISABLE_CANCEL(cancel_state); - GC_init(); /* if not called yet */ -#ifdef HAVE_LIBPTHREAD_SO - if (!GC_text_mapping("libpthread-", - &GC_libpthread_start, &GC_libpthread_end)) { - WARN("Failed to find libpthread.so text mapping: Expect crash\n", 0); - /* This might still work with some versions of libpthread, */ - /* so we do not abort. */ - } -#endif - if (!GC_text_mapping("ld-", &GC_libld_start, &GC_libld_end)) { - WARN("Failed to find ld.so text mapping: Expect crash\n", 0); - } - RESTORE_CANCEL(cancel_state); -} -#endif /* GC_LINUX_THREADS */ - -void *calloc(size_t n, size_t lb) { - if (EXPECT((lb | n) > GC_SQRT_SIZE_MAX, FALSE) /* fast initial test */ - && lb && n > GC_SIZE_MAX / lb) - return (*GC_get_oom_fn())(GC_SIZE_MAX); /* n*lb overflow */ -#if defined(GC_LINUX_THREADS) - /* The linker may allocate some memory that is only pointed to by */ - /* mmapped thread stacks. Make sure it is not collectible. */ - { - static GC_bool lib_bounds_set = FALSE; - ptr_t caller = (ptr_t)__builtin_return_address(0); - - /* This test does not need to ensure memory visibility, since */ - /* the bounds will be set when/if we create another thread. */ - if (!EXPECT(lib_bounds_set, TRUE)) { - GC_init_lib_bounds(); - lib_bounds_set = TRUE; - } - if (((word)caller >= (word)GC_libld_start && (word)caller < (word)GC_libld_end) -#ifdef HAVE_LIBPTHREAD_SO - || ((word)caller >= (word)GC_libpthread_start && (word)caller < (word)GC_libpthread_end) - /* The two ranges are actually usually adjacent, */ - /* so there may be a way to speed this up. */ -#endif - ) { - return GC_generic_malloc_uncollectable(n * lb, UNCOLLECTABLE); - } - } -#endif - return (void *)REDIRECT_MALLOC_F(n * lb); -} - -#ifndef strdup -char *strdup(const char *s) { - size_t lb = strlen(s) + 1; - char *result = (char *)REDIRECT_MALLOC_F(lb); - - if (EXPECT(NULL == result, FALSE)) { - errno = ENOMEM; - return NULL; - } - BCOPY(s, result, lb); - return result; -} -#endif /* !defined(strdup) */ - /* If strdup is macro defined, we assume that it actually calls malloc, */ - /* and thus the right thing will happen even without overriding it. */ - /* This seems to be true on most Linux systems. */ - -#ifndef strndup -/* This is similar to strdup(). */ -char *strndup(const char *str, size_t size) { - char *copy; - size_t len = strlen(str); - if (EXPECT(len > size, FALSE)) - len = size; - copy = (char *)REDIRECT_MALLOC_F(len + 1); - if (EXPECT(NULL == copy, FALSE)) { - errno = ENOMEM; - return NULL; - } - if (EXPECT(len > 0, TRUE)) - BCOPY(str, copy, len); - copy[len] = '\0'; - return copy; -} -#endif /* !strndup */ - -#undef GC_debug_malloc_replacement - -#endif /* REDIRECT_MALLOC */ - -/* Explicitly deallocate the object. hhdr should correspond to p. */ -static void free_internal(void *p, hdr *hhdr) { - size_t sz = (size_t)(hhdr->hb_sz); /* in bytes */ - size_t ngranules = BYTES_TO_GRANULES(sz); /* size in granules */ - int k = hhdr->hb_obj_kind; - - GC_bytes_freed += sz; - if (IS_UNCOLLECTABLE(k)) GC_non_gc_bytes -= sz; - if (EXPECT(ngranules <= MAXOBJGRANULES, TRUE)) { - struct obj_kind *ok = &GC_obj_kinds[k]; - void **flh; - - /* It is unnecessary to clear the mark bit. If the object is */ - /* reallocated, it does not matter. Otherwise, the collector will */ - /* do it, since it is on a free list. */ - if (ok->ok_init && EXPECT(sz > sizeof(word), TRUE)) { - BZERO((word *)p + 1, sz - sizeof(word)); - } - - flh = &(ok->ok_freelist[ngranules]); - obj_link(p) = *flh; - *flh = (ptr_t)p; - } else { - if (sz > HBLKSIZE) { - GC_large_allocd_bytes -= HBLKSIZE * OBJ_SZ_TO_BLOCKS(sz); - } - GC_freehblk(HBLKPTR(p)); - } -} - -GC_API void GC_CALL GC_free(void *p) { - hdr *hhdr; - - if (p /* != NULL */) { - /* CPPCHECK */ - } else { - /* Required by ANSI. It's not my fault ... */ - return; - } - -#ifdef LOG_ALLOCS - GC_log_printf("GC_free(%p) after GC #%lu\n", - p, (unsigned long)GC_gc_no); -#endif - hhdr = HDR(p); -#if defined(REDIRECT_MALLOC) && \ - ((defined(NEED_CALLINFO) && defined(GC_HAVE_BUILTIN_BACKTRACE)) || defined(GC_SOLARIS_THREADS) || defined(GC_LINUX_THREADS) || defined(MSWIN32)) - /* This might be called indirectly by GC_print_callers to free */ - /* the result of backtrace_symbols. */ - /* For Solaris, we have to redirect malloc calls during */ - /* initialization. For the others, this seems to happen */ - /* implicitly. */ - /* Don't try to deallocate that memory. */ - if (EXPECT(NULL == hhdr, FALSE)) return; -#endif - GC_ASSERT(GC_base(p) == p); - LOCK(); - free_internal(p, hhdr); - UNLOCK(); -} - -#ifdef THREADS -GC_INNER void GC_free_inner(void *p) { - GC_ASSERT(I_HOLD_LOCK()); - free_internal(p, HDR(p)); -} -#endif /* THREADS */ - -#if defined(REDIRECT_MALLOC) && !defined(REDIRECT_FREE) -#define REDIRECT_FREE GC_free -#endif - -#if defined(REDIRECT_FREE) && !defined(REDIRECT_MALLOC_IN_HEADER) - -#if defined(CPPCHECK) -#define REDIRECT_FREE_F GC_free /* e.g. */ -#else -#define REDIRECT_FREE_F REDIRECT_FREE -#endif - -void free(void *p) { -#ifndef IGNORE_FREE -#if defined(GC_LINUX_THREADS) && !defined(USE_PROC_FOR_LIBRARIES) - /* Don't bother with initialization checks. If nothing */ - /* has been initialized, the check fails, and that's safe, */ - /* since we have not allocated uncollectible objects neither. */ - ptr_t caller = (ptr_t)__builtin_return_address(0); - /* This test does not need to ensure memory visibility, since */ - /* the bounds will be set when/if we create another thread. */ - if (((word)caller >= (word)GC_libld_start && (word)caller < (word)GC_libld_end) -#ifdef HAVE_LIBPTHREAD_SO - || ((word)caller >= (word)GC_libpthread_start && (word)caller < (word)GC_libpthread_end) -#endif - ) { - GC_free(p); - return; - } -#endif - REDIRECT_FREE_F(p); -#endif -} -#endif /* REDIRECT_FREE */ diff --git a/vendor/bdwgc/mallocx.c b/vendor/bdwgc/mallocx.c deleted file mode 100644 index fb4068d4..00000000 --- a/vendor/bdwgc/mallocx.c +++ /dev/null @@ -1,597 +0,0 @@ -/* - * Copyright 1988, 1989 Hans-J. Boehm, Alan J. Demers - * Copyright (c) 1991-1994 by Xerox Corporation. All rights reserved. - * Copyright (c) 1996 by Silicon Graphics. All rights reserved. - * Copyright (c) 2000 by Hewlett-Packard Company. All rights reserved. - * Copyright (c) 2009-2022 Ivan Maidanski - * - * THIS MATERIAL IS PROVIDED AS IS, WITH ABSOLUTELY NO WARRANTY EXPRESSED - * OR IMPLIED. ANY USE IS AT YOUR OWN RISK. - * - * Permission is hereby granted to use or copy this program - * for any purpose, provided the above notices are retained on all copies. - * Permission to modify the code and to distribute modified code is granted, - * provided the above notices are retained, and a notice that the code was - * modified is included with the above copyright notice. - */ - -#ifdef _WIN32 -#define WIN32_LEAN_AND_MEAN -#include -#endif -#include "private/gc_priv.h" - -/* - * These are extra allocation routines which are likely to be less - * frequently used than those in malloc.c. They are separate in the - * hope that the .o file will be excluded from statically linked - * executables. We should probably break this up further. - */ - -#include - -#ifndef MSWINCE -#include -#endif - -/* Some externally visible but unadvertised variables to allow access to */ -/* free lists from inlined allocators without including gc_priv.h */ -/* or introducing dependencies on internal data structure layouts. */ -#ifdef _WIN32 -#define WIN32_LEAN_AND_MEAN -#include -#endif -#include "private/gc_alloc_ptrs.h" -void **const GC_objfreelist_ptr = GC_objfreelist; -void **const GC_aobjfreelist_ptr = GC_aobjfreelist; -void **const GC_uobjfreelist_ptr = GC_uobjfreelist; -#ifdef GC_ATOMIC_UNCOLLECTABLE -void **const GC_auobjfreelist_ptr = GC_auobjfreelist; -#endif - -GC_API int GC_CALL GC_get_kind_and_size(const void *p, size_t *psize) { - hdr *hhdr = HDR(p); - - if (psize != NULL) { - *psize = (size_t)hhdr->hb_sz; - } - return hhdr->hb_obj_kind; -} - -GC_API GC_ATTR_MALLOC void *GC_CALL GC_generic_or_special_malloc(size_t lb, - int k) { - switch (k) { - case PTRFREE: - case NORMAL: - return GC_malloc_kind(lb, k); - case UNCOLLECTABLE: -#ifdef GC_ATOMIC_UNCOLLECTABLE - case AUNCOLLECTABLE: -#endif - return GC_generic_malloc_uncollectable(lb, k); - default: - return GC_generic_malloc_aligned(lb, k, 0 /* flags */, 0); - } -} - -/* Change the size of the block pointed to by p to contain at least */ -/* lb bytes. The object may be (and quite likely will be) moved. */ -/* The kind (e.g. atomic) is the same as that of the old. */ -/* Shrinking of large blocks is not implemented well. */ -GC_API void *GC_CALL GC_realloc(void *p, size_t lb) { - struct hblk *h; - hdr *hhdr; - void *result; -#if defined(_FORTIFY_SOURCE) && defined(__GNUC__) && !defined(__clang__) - volatile /* Use cleared_p instead of p as a workaround to avoid */ - /* passing alloc_size(lb) attribute associated with p */ - /* to memset (including memset call inside GC_free). */ -#endif - word cleared_p = (word)p; - size_t sz; /* Current size in bytes */ - size_t orig_sz; /* Original sz in bytes */ - int obj_kind; - - if (NULL == p) return GC_malloc(lb); /* Required by ANSI */ - if (0 == lb) /* and p != NULL */ { -#ifndef IGNORE_FREE - GC_free(p); -#endif - return NULL; - } - h = HBLKPTR(p); - hhdr = HDR(h); - sz = (size_t)hhdr->hb_sz; - obj_kind = hhdr->hb_obj_kind; - orig_sz = sz; - - if (sz > MAXOBJBYTES) { - /* Round it up to the next whole heap block */ - word descr = GC_obj_kinds[obj_kind].ok_descriptor; - - sz = (sz + HBLKSIZE - 1) & ~(HBLKSIZE - 1); - if (GC_obj_kinds[obj_kind].ok_relocate_descr) - descr += sz; - /* GC_realloc might be changing the block size while */ - /* GC_reclaim_block or GC_clear_hdr_marks is examining it. */ - /* The change to the size field is benign, in that GC_reclaim */ - /* (and GC_clear_hdr_marks) would work correctly with either */ - /* value, since we are not changing the number of objects in */ - /* the block. But seeing a half-updated value (though unlikely */ - /* to occur in practice) could be probably bad. */ - /* Using unordered atomic accesses on the size and hb_descr */ - /* fields would solve the issue. (The alternate solution might */ - /* be to initially overallocate large objects, so we do not */ - /* have to adjust the size in GC_realloc, if they still fit. */ - /* But that is probably more expensive, since we may end up */ - /* scanning a bunch of zeros during GC.) */ -#ifdef AO_HAVE_store - GC_STATIC_ASSERT(sizeof(hhdr->hb_sz) == sizeof(AO_t)); - AO_store((volatile AO_t *)&hhdr->hb_sz, (AO_t)sz); - AO_store((volatile AO_t *)&hhdr->hb_descr, (AO_t)descr); -#else - { - LOCK(); - hhdr->hb_sz = sz; - hhdr->hb_descr = descr; - UNLOCK(); - } -#endif - -#ifdef MARK_BIT_PER_OBJ - GC_ASSERT(hhdr->hb_inv_sz == LARGE_INV_SZ); -#endif -#ifdef MARK_BIT_PER_GRANULE - GC_ASSERT((hhdr->hb_flags & LARGE_BLOCK) != 0 && hhdr->hb_map[ANY_INDEX] == 1); -#endif - if (IS_UNCOLLECTABLE(obj_kind)) GC_non_gc_bytes += (sz - orig_sz); - /* Extra area is already cleared by GC_alloc_large_and_clear. */ - } - if (ADD_EXTRA_BYTES(lb) <= sz) { - if (lb >= (sz >> 1)) { - if (orig_sz > lb) { - /* Clear unneeded part of object to avoid bogus pointer */ - /* tracing. */ - BZERO((ptr_t)cleared_p + lb, orig_sz - lb); - } - return p; - } - /* shrink */ - sz = lb; - } - result = GC_generic_or_special_malloc((word)lb, obj_kind); - if (EXPECT(result != NULL, TRUE)) { - /* In case of shrink, it could also return original object. */ - /* But this gives the client warning of imminent disaster. */ - BCOPY(p, result, sz); -#ifndef IGNORE_FREE - GC_free((ptr_t)cleared_p); -#endif - } - return result; -} - -#if defined(REDIRECT_MALLOC) && !defined(REDIRECT_REALLOC) -#define REDIRECT_REALLOC GC_realloc -#endif - -#ifdef REDIRECT_REALLOC - -/* As with malloc, avoid two levels of extra calls here. */ -#define GC_debug_realloc_replacement(p, lb) \ - GC_debug_realloc(p, lb, GC_DBG_EXTRAS) - -#if !defined(REDIRECT_MALLOC_IN_HEADER) -void *realloc(void *p, size_t lb) { - return REDIRECT_REALLOC(p, lb); -} -#endif - -#undef GC_debug_realloc_replacement -#endif /* REDIRECT_REALLOC */ - -/* Allocate memory such that only pointers to near the */ -/* beginning of the object are considered. */ -/* We avoid holding allocation lock while we clear the memory. */ -GC_API GC_ATTR_MALLOC void *GC_CALL -GC_generic_malloc_ignore_off_page(size_t lb, int k) { - return GC_generic_malloc_aligned(lb, k, IGNORE_OFF_PAGE, 0 /* align_m1 */); -} - -GC_API GC_ATTR_MALLOC void *GC_CALL GC_malloc_ignore_off_page(size_t lb) { - return GC_generic_malloc_aligned(lb, NORMAL, IGNORE_OFF_PAGE, 0); -} - -GC_API GC_ATTR_MALLOC void *GC_CALL -GC_malloc_atomic_ignore_off_page(size_t lb) { - return GC_generic_malloc_aligned(lb, PTRFREE, IGNORE_OFF_PAGE, 0); -} - -/* Increment GC_bytes_allocd from code that doesn't have direct access */ -/* to GC_arrays. */ -void GC_CALL GC_incr_bytes_allocd(size_t n) { - GC_bytes_allocd += n; -} - -/* The same for GC_bytes_freed. */ -void GC_CALL GC_incr_bytes_freed(size_t n) { - GC_bytes_freed += n; -} - -GC_API size_t GC_CALL GC_get_expl_freed_bytes_since_gc(void) { - return (size_t)GC_bytes_freed; -} - -#ifdef PARALLEL_MARK -STATIC volatile AO_t GC_bytes_allocd_tmp = 0; -/* Number of bytes of memory allocated since */ -/* we released the GC lock. Instead of */ -/* reacquiring the GC lock just to add this in, */ -/* we add it in the next time we reacquire */ -/* the lock. (Atomically adding it doesn't */ -/* work, since we would have to atomically */ -/* update it in GC_malloc, which is too */ -/* expensive.) */ -#endif /* PARALLEL_MARK */ - -/* Return a list of 1 or more objects of the indicated size, linked */ -/* through the first word in the object. This has the advantage that */ -/* it acquires the allocation lock only once, and may greatly reduce */ -/* time wasted contending for the allocation lock. Typical usage would */ -/* be in a thread that requires many items of the same size. It would */ -/* keep its own free list in thread-local storage, and call */ -/* GC_malloc_many or friends to replenish it. (We do not round up */ -/* object sizes, since a call indicates the intention to consume many */ -/* objects of exactly this size.) */ -/* We assume that the size is a multiple of GRANULE_BYTES. */ -/* We return the free-list by assigning it to *result, since it is */ -/* not safe to return, e.g. a linked list of pointer-free objects, */ -/* since the collector would not retain the entire list if it were */ -/* invoked just as we were returning. */ -/* Note that the client should usually clear the link field. */ -GC_API void GC_CALL GC_generic_malloc_many(size_t lb, int k, void **result) { - void *op; - void *p; - void **opp; - size_t lw; /* Length in words. */ - size_t lg; /* Length in granules. */ - signed_word my_bytes_allocd = 0; - struct obj_kind *ok = &(GC_obj_kinds[k]); - struct hblk **rlh; - - GC_ASSERT(lb != 0 && (lb & (GRANULE_BYTES - 1)) == 0); - /* Currently a single object is always allocated if manual VDB. */ - /* TODO: GC_dirty should be called for each linked object (but */ - /* the last one) to support multiple objects allocation. */ - if (!SMALL_OBJ(lb) || GC_manual_vdb) { - op = GC_generic_malloc_aligned(lb, k, 0 /* flags */, 0 /* align_m1 */); - if (EXPECT(0 != op, TRUE)) - obj_link(op) = 0; - *result = op; -#ifndef GC_DISABLE_INCREMENTAL - if (GC_manual_vdb && GC_is_heap_ptr(result)) { - GC_dirty_inner(result); - REACHABLE_AFTER_DIRTY(op); - } -#endif - return; - } - GC_ASSERT(k < MAXOBJKINDS); - lw = BYTES_TO_WORDS(lb); - lg = BYTES_TO_GRANULES(lb); - if (EXPECT(get_have_errors(), FALSE)) - GC_print_all_errors(); - GC_INVOKE_FINALIZERS(); - GC_DBG_COLLECT_AT_MALLOC(lb); - if (!EXPECT(GC_is_initialized, TRUE)) GC_init(); - LOCK(); - /* Do our share of marking work */ - if (GC_incremental && !GC_dont_gc) { - ENTER_GC(); - GC_collect_a_little_inner(1); - EXIT_GC(); - } - /* First see if we can reclaim a page of objects waiting to be */ - /* reclaimed. */ - rlh = ok->ok_reclaim_list; - if (rlh != NULL) { - struct hblk *hbp; - hdr *hhdr; - - while ((hbp = rlh[lg]) != NULL) { - hhdr = HDR(hbp); - rlh[lg] = hhdr->hb_next; - GC_ASSERT(hhdr->hb_sz == lb); - hhdr->hb_last_reclaimed = (unsigned short)GC_gc_no; -#ifdef PARALLEL_MARK - if (GC_parallel) { - signed_word my_bytes_allocd_tmp = - (signed_word)AO_load(&GC_bytes_allocd_tmp); - GC_ASSERT(my_bytes_allocd_tmp >= 0); - /* We only decrement it while holding the GC lock. */ - /* Thus we can't accidentally adjust it down in more */ - /* than one thread simultaneously. */ - - if (my_bytes_allocd_tmp != 0) { - (void)AO_fetch_and_add(&GC_bytes_allocd_tmp, - (AO_t)(-my_bytes_allocd_tmp)); - GC_bytes_allocd += my_bytes_allocd_tmp; - } - GC_acquire_mark_lock(); - ++GC_fl_builder_count; - UNLOCK(); - GC_release_mark_lock(); - } -#endif - op = GC_reclaim_generic(hbp, hhdr, lb, - ok->ok_init, 0, &my_bytes_allocd); - if (op != 0) { -#ifdef PARALLEL_MARK - if (GC_parallel) { - *result = op; - (void)AO_fetch_and_add(&GC_bytes_allocd_tmp, - (AO_t)my_bytes_allocd); - GC_acquire_mark_lock(); - --GC_fl_builder_count; - if (GC_fl_builder_count == 0) GC_notify_all_builder(); -#ifdef THREAD_SANITIZER - GC_release_mark_lock(); - LOCK(); - GC_bytes_found += my_bytes_allocd; - UNLOCK(); -#else - GC_bytes_found += my_bytes_allocd; - /* The result may be inaccurate. */ - GC_release_mark_lock(); -#endif - (void)GC_clear_stack(0); - return; - } -#endif - /* We also reclaimed memory, so we need to adjust */ - /* that count. */ - GC_bytes_found += my_bytes_allocd; - GC_bytes_allocd += my_bytes_allocd; - goto out; - } -#ifdef PARALLEL_MARK - if (GC_parallel) { - GC_acquire_mark_lock(); - --GC_fl_builder_count; - if (GC_fl_builder_count == 0) GC_notify_all_builder(); - GC_release_mark_lock(); - LOCK(); - /* The GC lock is needed for reclaim list access. We */ - /* must decrement fl_builder_count before reacquiring */ - /* the lock. Hopefully this path is rare. */ - - rlh = ok->ok_reclaim_list; /* reload rlh after locking */ - if (NULL == rlh) break; - } -#endif - } - } - /* Next try to use prefix of global free list if there is one. */ - /* We don't refill it, but we need to use it up before allocating */ - /* a new block ourselves. */ - opp = &(GC_obj_kinds[k].ok_freelist[lg]); - if ((op = *opp) != 0) { - *opp = 0; - my_bytes_allocd = 0; - for (p = op; p != 0; p = obj_link(p)) { - my_bytes_allocd += lb; - if ((word)my_bytes_allocd >= HBLKSIZE) { - *opp = obj_link(p); - obj_link(p) = 0; - break; - } - } - GC_bytes_allocd += my_bytes_allocd; - goto out; - } - /* Next try to allocate a new block worth of objects of this size. */ - { - struct hblk *h = GC_allochblk(lb, k, 0 /* flags */, 0 /* align_m1 */); - - if (h /* != NULL */) { /* CPPCHECK */ - if (IS_UNCOLLECTABLE(k)) GC_set_hdr_marks(HDR(h)); - GC_bytes_allocd += HBLKSIZE - HBLKSIZE % lb; -#ifdef PARALLEL_MARK - if (GC_parallel) { - GC_acquire_mark_lock(); - ++GC_fl_builder_count; - UNLOCK(); - GC_release_mark_lock(); - - op = GC_build_fl(h, lw, - (ok->ok_init || GC_debugging_started), 0); - - *result = op; - GC_acquire_mark_lock(); - --GC_fl_builder_count; - if (GC_fl_builder_count == 0) GC_notify_all_builder(); - GC_release_mark_lock(); - (void)GC_clear_stack(0); - return; - } -#endif - op = GC_build_fl(h, lw, (ok->ok_init || GC_debugging_started), 0); - goto out; - } - } - - /* As a last attempt, try allocating a single object. Note that */ - /* this may trigger a collection or expand the heap. */ - op = GC_generic_malloc_inner(lb, k, 0 /* flags */); - if (op != NULL) obj_link(op) = NULL; - -out: - *result = op; - UNLOCK(); - (void)GC_clear_stack(0); -} - -/* Note that the "atomic" version of this would be unsafe, since the */ -/* links would not be seen by the collector. */ -GC_API GC_ATTR_MALLOC void *GC_CALL GC_malloc_many(size_t lb) { - void *result; - size_t lg = ALLOC_REQUEST_GRANS(lb); - - GC_generic_malloc_many(GRANULES_TO_BYTES(lg), NORMAL, &result); - return result; -} - -/* TODO: The debugging version of GC_memalign and friends is tricky */ -/* and currently missing. There are 2 major difficulties: */ -/* - GC_base() should always point to the beginning of the allocated */ -/* block (thus, for small objects allocation we should probably */ -/* iterate over the list of free objects to find the one with the */ -/* suitable alignment); */ -/* - store_debug_info() should return the pointer of the object with */ -/* the requested alignment (unlike the object header). */ - -GC_API GC_ATTR_MALLOC void *GC_CALL GC_memalign(size_t align, size_t lb) { - size_t offset; - ptr_t result; - size_t align_m1 = align - 1; - - /* Check the alignment argument. */ - if (align < sizeof(void *) || (align & align_m1) != 0) return NULL; - - if (align <= GRANULE_BYTES) return GC_malloc(lb); - - if (align >= HBLKSIZE / 2 || lb >= HBLKSIZE / 2) { - return GC_clear_stack(GC_generic_malloc_aligned(lb, NORMAL, - 0 /* flags */, align_m1)); - } - - /* We could also try to make sure that the real rounded-up object size */ - /* is a multiple of align. That would be correct up to HBLKSIZE. */ - /* TODO: Not space efficient for big align values. */ - result = (ptr_t)GC_malloc(SIZET_SAT_ADD(lb, align_m1)); - /* It is OK not to check result for NULL as in that case */ - /* GC_memalign returns NULL too since (0 + 0 % align) is 0. */ - offset = (size_t)(word)result & align_m1; - if (offset != 0) { - offset = align - offset; - if (!GC_all_interior_pointers) { - GC_STATIC_ASSERT(VALID_OFFSET_SZ <= HBLKSIZE); - GC_ASSERT(offset < VALID_OFFSET_SZ); - GC_register_displacement(offset); - } - result += offset; - } - GC_ASSERT(((word)result & align_m1) == 0); - return result; -} - -/* This one exists largely to redirect posix_memalign for leaks finding. */ -GC_API int GC_CALL GC_posix_memalign(void **memptr, size_t align, size_t lb) { - /* Check alignment properly. */ - size_t align_minus_one = align - 1; /* to workaround a cppcheck warning */ - if (align < sizeof(void *) || (align_minus_one & align) != 0) { -#ifdef MSWINCE - return ERROR_INVALID_PARAMETER; -#else - return EINVAL; -#endif - } - - if ((*memptr = GC_memalign(align, lb)) == NULL) { -#ifdef MSWINCE - return ERROR_NOT_ENOUGH_MEMORY; -#else - return ENOMEM; -#endif - } - return 0; -} - -#ifndef GC_NO_VALLOC -GC_API GC_ATTR_MALLOC void *GC_CALL GC_valloc(size_t lb) { - if (!EXPECT(GC_is_initialized, TRUE)) GC_init(); - GC_ASSERT(GC_real_page_size != 0); - return GC_memalign(GC_real_page_size, lb); -} - -GC_API GC_ATTR_MALLOC void *GC_CALL GC_pvalloc(size_t lb) { - if (!EXPECT(GC_is_initialized, TRUE)) GC_init(); - GC_ASSERT(GC_real_page_size != 0); - lb = SIZET_SAT_ADD(lb, GC_real_page_size - 1) & ~(GC_real_page_size - 1); - return GC_memalign(GC_real_page_size, lb); -} -#endif /* !GC_NO_VALLOC */ - -/* Provide a version of strdup() that uses the collector to allocate */ -/* the copy of the string. */ -GC_API GC_ATTR_MALLOC char *GC_CALL GC_strdup(const char *s) { - char *copy; - size_t lb; - if (s == NULL) return NULL; - lb = strlen(s) + 1; - copy = (char *)GC_malloc_atomic(lb); - if (EXPECT(NULL == copy, FALSE)) { -#ifndef MSWINCE - errno = ENOMEM; -#endif - return NULL; - } - BCOPY(s, copy, lb); - return copy; -} - -GC_API GC_ATTR_MALLOC char *GC_CALL GC_strndup(const char *str, size_t size) { - char *copy; - size_t len = strlen(str); /* str is expected to be non-NULL */ - if (EXPECT(len > size, FALSE)) - len = size; - copy = (char *)GC_malloc_atomic(len + 1); - if (EXPECT(NULL == copy, FALSE)) { -#ifndef MSWINCE - errno = ENOMEM; -#endif - return NULL; - } - if (EXPECT(len > 0, TRUE)) - BCOPY(str, copy, len); - copy[len] = '\0'; - return copy; -} - -#ifdef GC_REQUIRE_WCSDUP -#include /* for wcslen() */ - -GC_API GC_ATTR_MALLOC wchar_t *GC_CALL GC_wcsdup(const wchar_t *str) { - size_t lb = (wcslen(str) + 1) * sizeof(wchar_t); - wchar_t *copy = (wchar_t *)GC_malloc_atomic(lb); - - if (EXPECT(NULL == copy, FALSE)) { -#ifndef MSWINCE - errno = ENOMEM; -#endif - return NULL; - } - BCOPY(str, copy, lb); - return copy; -} -#endif /* GC_REQUIRE_WCSDUP */ - -#ifndef CPPCHECK -GC_API void *GC_CALL GC_malloc_stubborn(size_t lb) { - return GC_malloc(lb); -} - -GC_API void GC_CALL GC_change_stubborn(const void *p) { - UNUSED_ARG(p); -} -#endif /* !CPPCHECK */ - -GC_API void GC_CALL GC_end_stubborn_change(const void *p) { - GC_dirty(p); /* entire object */ -} - -GC_API void GC_CALL GC_ptr_store_and_dirty(void *p, const void *q) { - *(const void **)p = q; - GC_dirty(p); - REACHABLE_AFTER_DIRTY(q); -} diff --git a/vendor/bdwgc/mark.c b/vendor/bdwgc/mark.c deleted file mode 100644 index 6235d3da..00000000 --- a/vendor/bdwgc/mark.c +++ /dev/null @@ -1,1912 +0,0 @@ -/* - * Copyright 1988, 1989 Hans-J. Boehm, Alan J. Demers - * Copyright (c) 1991-1995 by Xerox Corporation. All rights reserved. - * Copyright (c) 2000 by Hewlett-Packard Company. All rights reserved. - * Copyright (c) 2008-2022 Ivan Maidanski - * - * THIS MATERIAL IS PROVIDED AS IS, WITH ABSOLUTELY NO WARRANTY EXPRESSED - * OR IMPLIED. ANY USE IS AT YOUR OWN RISK. - * - * Permission is hereby granted to use or copy this program - * for any purpose, provided the above notices are retained on all copies. - * Permission to modify the code and to distribute modified code is granted, - * provided the above notices are retained, and a notice that the code was - * modified is included with the above copyright notice. - * - */ - -#ifdef _WIN32 -#define WIN32_LEAN_AND_MEAN -#include -#endif -#include "private/gc_pmark.h" - -/* Make arguments appear live to compiler. Put here to minimize the */ -/* risk of inlining. Used to minimize junk left in registers. */ -GC_ATTR_NOINLINE -void GC_noop6(word arg1, word arg2, word arg3, word arg4, word arg5, word arg6) { - UNUSED_ARG(arg1); - UNUSED_ARG(arg2); - UNUSED_ARG(arg3); - UNUSED_ARG(arg4); - UNUSED_ARG(arg5); - UNUSED_ARG(arg6); - /* Avoid GC_noop6 calls to be optimized away. */ -#if defined(AO_HAVE_compiler_barrier) && !defined(BASE_ATOMIC_OPS_EMULATED) - AO_compiler_barrier(); /* to serve as a special side-effect */ -#else - GC_noop1(0); -#endif -} - -#if defined(AO_HAVE_store) && defined(THREAD_SANITIZER) -volatile AO_t GC_noop_sink; -#else -volatile word GC_noop_sink; -#endif - -/* Make the argument appear live to compiler. This is similar */ -/* to GC_noop6(), but with a single argument. Robust against */ -/* whole program analysis. */ -GC_API void GC_CALL GC_noop1(GC_word x) { -#if defined(AO_HAVE_store) && defined(THREAD_SANITIZER) - AO_store(&GC_noop_sink, (AO_t)x); -#else - GC_noop_sink = x; -#endif -} - -/* Initialize GC_obj_kinds properly and standard free lists properly. */ -/* This must be done statically since they may be accessed before */ -/* GC_init is called. */ -/* It's done here, since we need to deal with mark descriptors. */ -GC_INNER struct obj_kind GC_obj_kinds[MAXOBJKINDS] = { - /* PTRFREE */ {&GC_aobjfreelist[0], 0 /* filled in dynamically */, - /* 0 | */ GC_DS_LENGTH, FALSE, FALSE - /*, */ OK_DISCLAIM_INITZ}, - /* NORMAL */ {&GC_objfreelist[0], 0, - /* 0 | */ GC_DS_LENGTH, - /* adjusted in GC_init for EXTRA_BYTES */ - TRUE /* add length to descr */, TRUE - /*, */ OK_DISCLAIM_INITZ}, - /* UNCOLLECTABLE */ - {&GC_uobjfreelist[0], 0, - /* 0 | */ GC_DS_LENGTH, TRUE /* add length to descr */, TRUE - /*, */ OK_DISCLAIM_INITZ}, -#ifdef GC_ATOMIC_UNCOLLECTABLE - {&GC_auobjfreelist[0], 0, - /* 0 | */ GC_DS_LENGTH, FALSE /* add length to descr */, FALSE - /*, */ OK_DISCLAIM_INITZ}, -#endif -}; - -#ifndef INITIAL_MARK_STACK_SIZE -#define INITIAL_MARK_STACK_SIZE (1 * HBLKSIZE) -/* INITIAL_MARK_STACK_SIZE * sizeof(mse) should be a */ -/* multiple of HBLKSIZE. */ -/* The incremental collector actually likes a larger */ -/* size, since it wants to push all marked dirty */ -/* objects before marking anything new. Currently we */ -/* let it grow dynamically. */ -#endif - -#if !defined(GC_DISABLE_INCREMENTAL) -STATIC word GC_n_rescuing_pages = 0; -/* Number of dirty pages we marked from */ -/* excludes ptrfree pages, etc. */ -/* Used for logging only. */ -#endif - -#ifdef PARALLEL_MARK -GC_INNER GC_bool GC_parallel_mark_disabled = FALSE; -#endif - -/* Is a collection in progress? Note that this can return true in the */ -/* non-incremental case, if a collection has been abandoned and the */ -/* mark state is now MS_INVALID. */ -GC_INNER GC_bool GC_collection_in_progress(void) { - return GC_mark_state != MS_NONE; -} - -/* Clear all mark bits in the header. */ -GC_INNER void GC_clear_hdr_marks(hdr *hhdr) { - size_t last_bit; - -#ifdef AO_HAVE_load - /* Atomic access is used to avoid racing with GC_realloc. */ - last_bit = FINAL_MARK_BIT((size_t)AO_load((volatile AO_t *)&hhdr->hb_sz)); -#else - /* No race as GC_realloc holds the lock while updating hb_sz. */ - last_bit = FINAL_MARK_BIT((size_t)hhdr->hb_sz); -#endif - - BZERO(hhdr->hb_marks, sizeof(hhdr->hb_marks)); - set_mark_bit_from_hdr(hhdr, last_bit); - hhdr->hb_n_marks = 0; -} - -/* Set all mark bits in the header. Used for uncollectible blocks. */ -GC_INNER void GC_set_hdr_marks(hdr *hhdr) { - unsigned i; - size_t sz = (size_t)hhdr->hb_sz; - unsigned n_marks = (unsigned)FINAL_MARK_BIT(sz); - -#ifdef USE_MARK_BYTES - for (i = 0; i <= n_marks; i += (unsigned)MARK_BIT_OFFSET(sz)) { - hhdr->hb_marks[i] = 1; - } -#else - /* Note that all bits are set even in case of MARK_BIT_PER_GRANULE, */ - /* instead of setting every n-th bit where n is MARK_BIT_OFFSET(sz). */ - /* This is done for a performance reason. */ - for (i = 0; i < divWORDSZ(n_marks); ++i) { - hhdr->hb_marks[i] = GC_WORD_MAX; - } - /* Set the remaining bits near the end (plus one bit past the end). */ - hhdr->hb_marks[i] = ((((word)1 << modWORDSZ(n_marks)) - 1) << 1) | 1; -#endif -#ifdef MARK_BIT_PER_OBJ - hhdr->hb_n_marks = n_marks; -#else - hhdr->hb_n_marks = HBLK_OBJS(sz); -#endif -} - -/* Clear all mark bits associated with block h. */ -static void GC_CALLBACK clear_marks_for_block(struct hblk *h, GC_word dummy) { - hdr *hhdr = HDR(h); - - UNUSED_ARG(dummy); - if (IS_UNCOLLECTABLE(hhdr->hb_obj_kind)) return; - /* Mark bit for these is cleared only once the object is */ - /* explicitly deallocated. This either frees the block, or */ - /* the bit is cleared once the object is on the free list. */ - GC_clear_hdr_marks(hhdr); -} - -/* Slow but general routines for setting/clearing/asking about mark bits. */ -GC_API void GC_CALL GC_set_mark_bit(const void *p) { - struct hblk *h = HBLKPTR(p); - hdr *hhdr = HDR(h); - word bit_no = MARK_BIT_NO((ptr_t)p - (ptr_t)h, hhdr->hb_sz); - - if (!mark_bit_from_hdr(hhdr, bit_no)) { - set_mark_bit_from_hdr(hhdr, bit_no); - ++hhdr->hb_n_marks; - } -} - -GC_API void GC_CALL GC_clear_mark_bit(const void *p) { - struct hblk *h = HBLKPTR(p); - hdr *hhdr = HDR(h); - word bit_no = MARK_BIT_NO((ptr_t)p - (ptr_t)h, hhdr->hb_sz); - - if (mark_bit_from_hdr(hhdr, bit_no)) { - size_t n_marks = hhdr->hb_n_marks; - - GC_ASSERT(n_marks != 0); - clear_mark_bit_from_hdr(hhdr, bit_no); - n_marks--; -#ifdef PARALLEL_MARK - if (n_marks != 0 || !GC_parallel) - hhdr->hb_n_marks = n_marks; - /* Don't decrement to zero. The counts are approximate due to */ - /* concurrency issues, but we need to ensure that a count of */ - /* zero implies an empty block. */ -#else - hhdr->hb_n_marks = n_marks; -#endif - } -} - -GC_API int GC_CALL GC_is_marked(const void *p) { - struct hblk *h = HBLKPTR(p); - hdr *hhdr = HDR(h); - word bit_no = MARK_BIT_NO((ptr_t)p - (ptr_t)h, hhdr->hb_sz); - - return (int)mark_bit_from_hdr(hhdr, bit_no); /* 0 or 1 */ -} - -/* Clear mark bits in all allocated heap blocks. This invalidates the */ -/* marker invariant, and sets GC_mark_state to reflect this. (This */ -/* implicitly starts marking to reestablish the invariant.) */ -GC_INNER void GC_clear_marks(void) { - GC_ASSERT(GC_is_initialized); /* needed for GC_push_roots */ - GC_apply_to_all_blocks(clear_marks_for_block, (word)0); - GC_objects_are_marked = FALSE; - GC_mark_state = MS_INVALID; - GC_scan_ptr = NULL; -} - -/* Initiate a garbage collection. Initiates a full collection if the */ -/* mark state is invalid. */ -GC_INNER void GC_initiate_gc(void) { - GC_ASSERT(I_HOLD_LOCK()); - GC_ASSERT(GC_is_initialized); -#ifndef GC_DISABLE_INCREMENTAL - if (GC_incremental) { -#ifdef CHECKSUMS - GC_read_dirty(FALSE); - GC_check_dirty(); -#else - GC_read_dirty(GC_mark_state == MS_INVALID); -#endif - } - GC_n_rescuing_pages = 0; -#endif - if (GC_mark_state == MS_NONE) { - GC_mark_state = MS_PUSH_RESCUERS; - } else { - GC_ASSERT(GC_mark_state == MS_INVALID); - /* This is really a full collection, and mark bits are invalid. */ - } - GC_scan_ptr = NULL; -} - -#ifdef PARALLEL_MARK -STATIC void GC_do_parallel_mark(void); /* Initiate parallel marking. */ -#endif /* PARALLEL_MARK */ - -#ifdef GC_DISABLE_INCREMENTAL -#define GC_push_next_marked_dirty(h) GC_push_next_marked(h) -#else -STATIC struct hblk *GC_push_next_marked_dirty(struct hblk *h); -/* Invoke GC_push_marked on next dirty block above h. */ -/* Return a pointer just past the end of this block. */ -#endif /* !GC_DISABLE_INCREMENTAL */ -STATIC struct hblk *GC_push_next_marked(struct hblk *h); -/* Ditto, but also mark from clean pages. */ -STATIC struct hblk *GC_push_next_marked_uncollectable(struct hblk *h); -/* Ditto, but mark only from uncollectible pages. */ - -static void alloc_mark_stack(size_t); - -static void push_roots_and_advance(GC_bool push_all, ptr_t cold_gc_frame) { - if (GC_scan_ptr != NULL) return; /* not ready to push */ - - GC_push_roots(push_all, cold_gc_frame); - GC_objects_are_marked = TRUE; - if (GC_mark_state != MS_INVALID) - GC_mark_state = MS_ROOTS_PUSHED; -} - -STATIC GC_on_mark_stack_empty_proc GC_on_mark_stack_empty; - -GC_API void GC_CALL GC_set_on_mark_stack_empty(GC_on_mark_stack_empty_proc fn) { - LOCK(); - GC_on_mark_stack_empty = fn; - UNLOCK(); -} - -GC_API GC_on_mark_stack_empty_proc GC_CALL GC_get_on_mark_stack_empty(void) { - GC_on_mark_stack_empty_proc fn; - - LOCK(); - fn = GC_on_mark_stack_empty; - UNLOCK(); - return fn; -} - -/* Perform a small amount of marking. */ -/* We try to touch roughly a page of memory. */ -/* Return TRUE if we just finished a mark phase. */ -/* Cold_gc_frame is an address inside a GC frame that */ -/* remains valid until all marking is complete. */ -/* A zero value indicates that it's OK to miss some */ -/* register values. In the case of an incremental */ -/* collection, the world may be running. */ -#ifdef WRAP_MARK_SOME -/* For Win32, this is called after we establish a structured */ -/* exception (or signal) handler, in case Windows unmaps one */ -/* of our root segments. Note that this code should never */ -/* generate an incremental GC write fault. */ -STATIC GC_bool GC_mark_some_inner(ptr_t cold_gc_frame) -#else -GC_INNER GC_bool GC_mark_some(ptr_t cold_gc_frame) -#endif -{ - GC_ASSERT(I_HOLD_LOCK()); - switch (GC_mark_state) { - case MS_NONE: - return TRUE; - - case MS_PUSH_RESCUERS: - if ((word)GC_mark_stack_top >= (word)(GC_mark_stack_limit - INITIAL_MARK_STACK_SIZE / 2)) { - /* Go ahead and mark, even though that might cause us to */ - /* see more marked dirty objects later on. Avoid this */ - /* in the future. */ - GC_mark_stack_too_small = TRUE; - MARK_FROM_MARK_STACK(); - } else { - GC_scan_ptr = GC_push_next_marked_dirty(GC_scan_ptr); -#ifndef GC_DISABLE_INCREMENTAL - if (NULL == GC_scan_ptr) { - GC_COND_LOG_PRINTF("Marked from %lu dirty pages\n", - (unsigned long)GC_n_rescuing_pages); - } -#endif - push_roots_and_advance(FALSE, cold_gc_frame); - } - GC_ASSERT(GC_mark_state == MS_PUSH_RESCUERS || GC_mark_state == MS_ROOTS_PUSHED || GC_mark_state == MS_INVALID); - break; - - case MS_PUSH_UNCOLLECTABLE: - if ((word)GC_mark_stack_top >= (word)(GC_mark_stack + GC_mark_stack_size / 4)) { -#ifdef PARALLEL_MARK - /* Avoid this, since we don't parallelize the marker */ - /* here. */ - if (GC_parallel) GC_mark_stack_too_small = TRUE; -#endif - MARK_FROM_MARK_STACK(); - } else { - GC_scan_ptr = GC_push_next_marked_uncollectable(GC_scan_ptr); - push_roots_and_advance(TRUE, cold_gc_frame); - } - GC_ASSERT(GC_mark_state == MS_PUSH_UNCOLLECTABLE || GC_mark_state == MS_ROOTS_PUSHED || GC_mark_state == MS_INVALID); - break; - - case MS_ROOTS_PUSHED: -#ifdef PARALLEL_MARK - /* Eventually, incremental marking should run */ - /* asynchronously in multiple threads, without grabbing */ - /* the allocation lock. */ - /* For now, parallel marker is disabled if there is */ - /* a chance that marking could be interrupted by */ - /* a client-supplied time limit or custom stop function. */ - if (GC_parallel && !GC_parallel_mark_disabled) { - GC_do_parallel_mark(); - GC_ASSERT((word)GC_mark_stack_top < (word)GC_first_nonempty); - GC_mark_stack_top = GC_mark_stack - 1; - if (GC_mark_stack_too_small) { - alloc_mark_stack(2 * GC_mark_stack_size); - } - if (GC_mark_state == MS_ROOTS_PUSHED) { - GC_mark_state = MS_NONE; - return TRUE; - } - GC_ASSERT(GC_mark_state == MS_INVALID); - break; - } -#endif - if ((word)GC_mark_stack_top >= (word)GC_mark_stack) { - MARK_FROM_MARK_STACK(); - } else { - GC_on_mark_stack_empty_proc on_ms_empty; - - if (GC_mark_stack_too_small) { - GC_mark_state = MS_NONE; - alloc_mark_stack(2 * GC_mark_stack_size); - return TRUE; - } - on_ms_empty = GC_on_mark_stack_empty; - if (on_ms_empty != 0) { - GC_mark_stack_top = on_ms_empty(GC_mark_stack_top, - GC_mark_stack_limit); - /* If we pushed new items or overflowed the stack, */ - /* we need to continue processing. */ - if ((word)GC_mark_stack_top >= (word)GC_mark_stack || GC_mark_stack_too_small) - break; - } - - GC_mark_state = MS_NONE; - return TRUE; - } - GC_ASSERT(GC_mark_state == MS_ROOTS_PUSHED || GC_mark_state == MS_INVALID); - break; - - case MS_INVALID: - case MS_PARTIALLY_INVALID: - if (!GC_objects_are_marked) { - GC_mark_state = MS_PUSH_UNCOLLECTABLE; - break; - } - if ((word)GC_mark_stack_top >= (word)GC_mark_stack) { - MARK_FROM_MARK_STACK(); - GC_ASSERT(GC_mark_state == MS_PARTIALLY_INVALID || GC_mark_state == MS_INVALID); - break; - } - if (NULL == GC_scan_ptr && GC_mark_state == MS_INVALID) { - /* About to start a heap scan for marked objects. */ - /* Mark stack is empty. OK to reallocate. */ - if (GC_mark_stack_too_small) { - alloc_mark_stack(2 * GC_mark_stack_size); - } - GC_mark_state = MS_PARTIALLY_INVALID; - } - GC_scan_ptr = GC_push_next_marked(GC_scan_ptr); - if (GC_mark_state == MS_PARTIALLY_INVALID) - push_roots_and_advance(TRUE, cold_gc_frame); - GC_ASSERT(GC_mark_state == MS_ROOTS_PUSHED || GC_mark_state == MS_PARTIALLY_INVALID || GC_mark_state == MS_INVALID); - break; - - default: - ABORT("GC_mark_some: bad state"); - } - return FALSE; -} - -#ifdef WRAP_MARK_SOME -GC_INNER GC_bool GC_mark_some(ptr_t cold_gc_frame) { - GC_bool ret_val; - - if (GC_no_dls) { - ret_val = GC_mark_some_inner(cold_gc_frame); - } else { - /* Windows appears to asynchronously create and remove */ - /* writable memory mappings, for reasons we haven't yet */ - /* understood. Since we look for writable regions to */ - /* determine the root set, we may try to mark from an */ - /* address range that disappeared since we started the */ - /* collection. Thus we have to recover from faults here. */ - /* This code seems to be necessary for WinCE (at least in */ - /* the case we'd decide to add MEM_PRIVATE sections to */ - /* data roots in GC_register_dynamic_libraries()). */ - /* It's conceivable that this is the same issue as with */ - /* terminating threads that we see with Linux and */ - /* USE_PROC_FOR_LIBRARIES. */ -#ifndef NO_SEH_AVAILABLE - __try { - ret_val = GC_mark_some_inner(cold_gc_frame); - } __except (GetExceptionCode() == EXCEPTION_ACCESS_VIOLATION ? EXCEPTION_EXECUTE_HANDLER : EXCEPTION_CONTINUE_SEARCH) { - goto handle_ex; - } -#else -#if defined(USE_PROC_FOR_LIBRARIES) && !defined(DEFAULT_VDB) - if (GC_auto_incremental) { - static GC_bool is_warned = FALSE; - - if (!is_warned) { - is_warned = TRUE; - WARN("Incremental GC incompatible with /proc roots\n", 0); - } - /* I'm not sure if this could still work ... */ - } -#endif - /* If USE_PROC_FOR_LIBRARIES, we are handling the case in */ - /* which /proc is used for root finding, and we have threads. */ - /* We may find a stack for a thread that is in the process of */ - /* exiting, and disappears while we are marking it. */ - /* This seems extremely difficult to avoid otherwise. */ - GC_setup_temporary_fault_handler(); - if (SETJMP(GC_jmp_buf) != 0) goto handle_ex; - ret_val = GC_mark_some_inner(cold_gc_frame); - GC_reset_fault_handler(); -#endif - } - -#if defined(GC_WIN32_THREADS) && !defined(GC_PTHREADS) - /* With DllMain-based thread tracking, a thread may have */ - /* started while we were marking. This is logically equivalent */ - /* to the exception case; our results are invalid and we have */ - /* to start over. This cannot be prevented since we can't */ - /* block in DllMain. */ - if (GC_started_thread_while_stopped()) - goto handle_thr_start; -#endif - return ret_val; - -handle_ex: - /* Exception handler starts here for all cases. */ -#if defined(NO_SEH_AVAILABLE) - GC_reset_fault_handler(); -#endif - { - static word warned_gc_no; - - /* Report caught ACCESS_VIOLATION, once per collection. */ - if (warned_gc_no != GC_gc_no) { - GC_COND_LOG_PRINTF("Memory mapping disappeared at collection #%lu\n", - (unsigned long)GC_gc_no + 1); - warned_gc_no = GC_gc_no; - } - } -#if defined(GC_WIN32_THREADS) && !defined(GC_PTHREADS) -handle_thr_start: -#endif - /* We have bad roots on the mark stack - discard it. */ - /* Rescan from marked objects. Redetermine roots. */ -#ifdef REGISTER_LIBRARIES_EARLY - START_WORLD(); - GC_cond_register_dynamic_libraries(); - STOP_WORLD(); -#endif - GC_invalidate_mark_state(); - GC_scan_ptr = NULL; - return FALSE; -} -#endif /* WRAP_MARK_SOME */ - -GC_INNER void GC_invalidate_mark_state(void) { - GC_mark_state = MS_INVALID; - GC_mark_stack_top = GC_mark_stack - 1; -} - -GC_INNER mse *GC_signal_mark_stack_overflow(mse *msp) { - GC_mark_state = MS_INVALID; -#ifdef PARALLEL_MARK - /* We are using a local_mark_stack in parallel mode, so */ - /* do not signal the global mark stack to be resized. */ - /* That will be done if required in GC_return_mark_stack. */ - if (!GC_parallel) - GC_mark_stack_too_small = TRUE; -#else - GC_mark_stack_too_small = TRUE; -#endif - GC_COND_LOG_PRINTF("Mark stack overflow; current size: %lu entries\n", - (unsigned long)GC_mark_stack_size); - return msp - GC_MARK_STACK_DISCARDS; -} - -/* - * Mark objects pointed to by the regions described by - * mark stack entries between mark_stack and mark_stack_top, - * inclusive. Assumes the upper limit of a mark stack entry - * is never 0. A mark stack entry never has size 0. - * We try to traverse on the order of a hblk of memory before we return. - * Caller is responsible for calling this until the mark stack is empty. - * Note that this is the most performance critical routine in the - * collector. Hence it contains all sorts of ugly hacks to speed - * things up. In particular, we avoid procedure calls on the common - * path, we take advantage of peculiarities of the mark descriptor - * encoding, we optionally maintain a cache for the block address to - * header mapping, we prefetch when an object is "grayed", etc. - */ -GC_ATTR_NO_SANITIZE_ADDR GC_ATTR_NO_SANITIZE_MEMORY GC_ATTR_NO_SANITIZE_THREAD - GC_INNER mse * - GC_mark_from(mse *mark_stack_top, mse *mark_stack, - mse *mark_stack_limit) { - signed_word credit = HBLKSIZE; /* Remaining credit for marking work. */ - ptr_t current_p; /* Pointer to current candidate ptr. */ - word current; /* Candidate pointer. */ - ptr_t limit = 0; /* (Incl) limit of current candidate range. */ - word descr; - ptr_t greatest_ha = (ptr_t)GC_greatest_plausible_heap_addr; - ptr_t least_ha = (ptr_t)GC_least_plausible_heap_addr; - DECLARE_HDR_CACHE; - -#define SPLIT_RANGE_WORDS 128 /* Must be power of 2. */ - - GC_objects_are_marked = TRUE; - INIT_HDR_CACHE; -#ifdef OS2 /* Use untweaked version to circumvent compiler problem. */ - while ((word)mark_stack_top >= (word)mark_stack && credit >= 0) -#else - while (((((word)mark_stack_top - (word)mark_stack) | (word)credit) & SIGNB) == 0) -#endif - { - current_p = mark_stack_top->mse_start; - descr = mark_stack_top->mse_descr.w; - retry: - /* current_p and descr describe the current object. */ - /* (*mark_stack_top) is vacant. */ - /* The following is 0 only for small objects described by a simple */ - /* length descriptor. For many applications this is the common */ - /* case, so we try to detect it quickly. */ - if (descr & ((~(WORDS_TO_BYTES(SPLIT_RANGE_WORDS) - 1)) | GC_DS_TAGS)) { - word tag = descr & GC_DS_TAGS; - - GC_STATIC_ASSERT(GC_DS_TAGS == 0x3); - switch (tag) { - case GC_DS_LENGTH: - /* Large length. */ - /* Process part of the range to avoid pushing too much on the */ - /* stack. */ - GC_ASSERT(descr < (word)GC_greatest_plausible_heap_addr - (word)GC_least_plausible_heap_addr || (word)(current_p + descr) <= (word)GC_least_plausible_heap_addr || (word)current_p >= (word)GC_greatest_plausible_heap_addr); -#ifdef PARALLEL_MARK -#define SHARE_BYTES 2048 - if (descr > SHARE_BYTES && GC_parallel && (word)mark_stack_top < (word)(mark_stack_limit - 1)) { - word new_size = (descr / 2) & ~(word)(sizeof(word) - 1); - - mark_stack_top->mse_start = current_p; - mark_stack_top->mse_descr.w = new_size + sizeof(word); - /* Makes sure we handle */ - /* misaligned pointers. */ - mark_stack_top++; -#ifdef ENABLE_TRACE - if ((word)GC_trace_addr >= (word)current_p && (word)GC_trace_addr < (word)(current_p + descr)) { - GC_log_printf( - "GC #%lu: large section; start %p, len %lu," - " splitting (parallel) at %p\n", - (unsigned long)GC_gc_no, (void *)current_p, - (unsigned long)descr, - (void *)(current_p + new_size)); - } -#endif - current_p += new_size; - descr -= new_size; - goto retry; - } -#endif /* PARALLEL_MARK */ - mark_stack_top->mse_start = - limit = current_p + WORDS_TO_BYTES(SPLIT_RANGE_WORDS - 1); - mark_stack_top->mse_descr.w = - descr - WORDS_TO_BYTES(SPLIT_RANGE_WORDS - 1); -#ifdef ENABLE_TRACE - if ((word)GC_trace_addr >= (word)current_p && (word)GC_trace_addr < (word)(current_p + descr)) { - GC_log_printf( - "GC #%lu: large section; start %p, len %lu," - " splitting at %p\n", - (unsigned long)GC_gc_no, (void *)current_p, - (unsigned long)descr, (void *)limit); - } -#endif - /* Make sure that pointers overlapping the two ranges are */ - /* considered. */ - limit += sizeof(word) - ALIGNMENT; - break; - case GC_DS_BITMAP: - mark_stack_top--; -#ifdef ENABLE_TRACE - if ((word)GC_trace_addr >= (word)current_p && (word)GC_trace_addr < (word)(current_p + WORDS_TO_BYTES(WORDSZ - 2))) { - GC_log_printf("GC #%lu: tracing from %p bitmap descr %lu\n", - (unsigned long)GC_gc_no, (void *)current_p, - (unsigned long)descr); - } -#endif /* ENABLE_TRACE */ - descr &= ~GC_DS_TAGS; - credit -= WORDS_TO_BYTES(WORDSZ / 2); /* guess */ - for (; descr != 0; descr <<= 1, current_p += sizeof(word)) { - if ((descr & SIGNB) == 0) continue; - LOAD_WORD_OR_CONTINUE(current, current_p); - FIXUP_POINTER(current); - if (current >= (word)least_ha && current < (word)greatest_ha) { - PREFETCH((ptr_t)current); -#ifdef ENABLE_TRACE - if (GC_trace_addr == current_p) { - GC_log_printf("GC #%lu: considering(3) %p -> %p\n", - (unsigned long)GC_gc_no, (void *)current_p, - (void *)current); - } -#endif /* ENABLE_TRACE */ - PUSH_CONTENTS((ptr_t)current, mark_stack_top, - mark_stack_limit, current_p); - } - } - continue; - case GC_DS_PROC: - mark_stack_top--; -#ifdef ENABLE_TRACE - if ((word)GC_trace_addr >= (word)current_p && GC_base(current_p) != 0 && GC_base(current_p) == GC_base(GC_trace_addr)) { - GC_log_printf("GC #%lu: tracing from %p, proc descr %lu\n", - (unsigned long)GC_gc_no, (void *)current_p, - (unsigned long)descr); - } -#endif /* ENABLE_TRACE */ - credit -= GC_PROC_BYTES; - mark_stack_top = (*PROC(descr))((word *)current_p, mark_stack_top, - mark_stack_limit, ENV(descr)); - continue; - case GC_DS_PER_OBJECT: - if (!(descr & SIGNB)) { - /* Descriptor is in the object. */ - descr = *(word *)(current_p + descr - GC_DS_PER_OBJECT); - } else { - /* Descriptor is in type descriptor pointed to by first */ - /* word in object. */ - ptr_t type_descr = *(ptr_t *)current_p; - /* type_descr is either a valid pointer to the descriptor */ - /* structure, or this object was on a free list. */ - /* If it was anything but the last object on the free list, */ - /* we will misinterpret the next object on the free list as */ - /* the type descriptor, and get a 0 GC descriptor, which */ - /* is ideal. Unfortunately, we need to check for the last */ - /* object case explicitly. */ - if (EXPECT(0 == type_descr, FALSE)) { - mark_stack_top--; - continue; - } - descr = *(word *)(type_descr - ((signed_word)descr + (GC_INDIR_PER_OBJ_BIAS - GC_DS_PER_OBJECT))); - } - if (0 == descr) { - /* Can happen either because we generated a 0 descriptor */ - /* or we saw a pointer to a free object. */ - mark_stack_top--; - continue; - } - goto retry; - } - } else { - /* Small object with length descriptor. */ - mark_stack_top--; -#ifndef SMALL_CONFIG - if (descr < sizeof(word)) - continue; -#endif -#ifdef ENABLE_TRACE - if ((word)GC_trace_addr >= (word)current_p && (word)GC_trace_addr < (word)(current_p + descr)) { - GC_log_printf("GC #%lu: small object; start %p, len %lu\n", - (unsigned long)GC_gc_no, (void *)current_p, - (unsigned long)descr); - } -#endif - limit = current_p + (word)descr; - } - /* The simple case in which we're scanning a range. */ - GC_ASSERT(!((word)current_p & (ALIGNMENT - 1))); - credit -= limit - current_p; - limit -= sizeof(word); - { -#define PREF_DIST 4 - -#if !defined(SMALL_CONFIG) && !defined(USE_PTR_HWTAG) - word deferred; - - /* Try to prefetch the next pointer to be examined ASAP. */ - /* Empirically, this also seems to help slightly without */ - /* prefetches, at least on linux/x86. Presumably this loop */ - /* ends up with less register pressure, and gcc thus ends up */ - /* generating slightly better code. Overall gcc code quality */ - /* for this loop is still not great. */ - for (;;) { - PREFETCH(limit - PREF_DIST * CACHE_LINE_SIZE); - GC_ASSERT((word)limit >= (word)current_p); - deferred = *(word *)limit; - FIXUP_POINTER(deferred); - limit -= ALIGNMENT; - if (deferred >= (word)least_ha && deferred < (word)greatest_ha) { - PREFETCH((ptr_t)deferred); - break; - } - if ((word)current_p > (word)limit) goto next_object; - /* Unroll once, so we don't do too many of the prefetches */ - /* based on limit. */ - deferred = *(word *)limit; - FIXUP_POINTER(deferred); - limit -= ALIGNMENT; - if (deferred >= (word)least_ha && deferred < (word)greatest_ha) { - PREFETCH((ptr_t)deferred); - break; - } - if ((word)current_p > (word)limit) goto next_object; - } -#endif - - for (; (word)current_p <= (word)limit; current_p += ALIGNMENT) { - /* Empirically, unrolling this loop doesn't help a lot. */ - /* Since PUSH_CONTENTS expands to a lot of code, */ - /* we don't. */ - LOAD_WORD_OR_CONTINUE(current, current_p); - FIXUP_POINTER(current); - PREFETCH(current_p + PREF_DIST * CACHE_LINE_SIZE); - if (current >= (word)least_ha && current < (word)greatest_ha) { - /* Prefetch the contents of the object we just pushed. It's */ - /* likely we will need them soon. */ - PREFETCH((ptr_t)current); -#ifdef ENABLE_TRACE - if (GC_trace_addr == current_p) { - GC_log_printf("GC #%lu: considering(1) %p -> %p\n", - (unsigned long)GC_gc_no, (void *)current_p, - (void *)current); - } -#endif /* ENABLE_TRACE */ - PUSH_CONTENTS((ptr_t)current, mark_stack_top, - mark_stack_limit, current_p); - } - } - -#if !defined(SMALL_CONFIG) && !defined(USE_PTR_HWTAG) - /* We still need to mark the entry we previously prefetched. */ - /* We already know that it passes the preliminary pointer */ - /* validity test. */ -#ifdef ENABLE_TRACE - if (GC_trace_addr == current_p) { - GC_log_printf("GC #%lu: considering(2) %p -> %p\n", - (unsigned long)GC_gc_no, (void *)current_p, - (void *)deferred); - } -#endif /* ENABLE_TRACE */ - PUSH_CONTENTS((ptr_t)deferred, mark_stack_top, - mark_stack_limit, current_p); - next_object:; -#endif - } - } - return mark_stack_top; -} - -#ifdef PARALLEL_MARK - -STATIC GC_bool GC_help_wanted = FALSE; /* Protected by mark lock. */ -STATIC unsigned GC_helper_count = 0; /* Number of running helpers. */ - /* Protected by mark lock. */ -STATIC unsigned GC_active_count = 0; /* Number of active helpers. */ - /* Protected by mark lock. */ - /* May increase and decrease */ - /* within each mark cycle. But */ - /* once it returns to 0, it */ - /* stays zero for the cycle. */ - -GC_INNER word GC_mark_no = 0; - -#ifdef LINT2 -#define LOCAL_MARK_STACK_SIZE (HBLKSIZE / 8) -#else -#define LOCAL_MARK_STACK_SIZE HBLKSIZE -/* Under normal circumstances, this is big enough to guarantee */ -/* we don't overflow half of it in a single call to */ -/* GC_mark_from. */ -#endif - -/* Wait all markers to finish initialization (i.e. store */ -/* marker_[b]sp, marker_mach_threads, GC_marker_Id). */ -GC_INNER void GC_wait_for_markers_init(void) { - signed_word count; - - GC_ASSERT(I_HOLD_LOCK()); - if (GC_markers_m1 == 0) - return; - - /* Allocate the local mark stack for the thread that holds GC lock. */ -#ifndef CAN_HANDLE_FORK - GC_ASSERT(NULL == GC_main_local_mark_stack); -#else - if (NULL == GC_main_local_mark_stack) -#endif - { - size_t bytes_to_get = - ROUNDUP_PAGESIZE_IF_MMAP(LOCAL_MARK_STACK_SIZE * sizeof(mse)); - - GC_ASSERT(GC_page_size != 0); - GC_main_local_mark_stack = (mse *)GET_MEM(bytes_to_get); - if (NULL == GC_main_local_mark_stack) - ABORT("Insufficient memory for main local_mark_stack"); - GC_add_to_our_memory((ptr_t)GC_main_local_mark_stack, bytes_to_get); - } - - /* Reuse marker lock and builders count to synchronize */ - /* marker threads startup. */ - GC_acquire_mark_lock(); - GC_fl_builder_count += GC_markers_m1; - count = GC_fl_builder_count; - GC_release_mark_lock(); - if (count != 0) { - GC_ASSERT(count > 0); - GC_wait_for_reclaim(); - } -} - -/* Steal mark stack entries starting at mse low into mark stack local */ -/* until we either steal mse high, or we have max entries. */ -/* Return a pointer to the top of the local mark stack. */ -/* (*next) is replaced by a pointer to the next unscanned mark stack */ -/* entry. */ -STATIC mse *GC_steal_mark_stack(mse *low, mse *high, mse *local, - unsigned max, mse **next) { - mse *p; - mse *top = local - 1; - unsigned i = 0; - - GC_ASSERT((word)high >= (word)(low - 1) && (word)(high - low + 1) <= GC_mark_stack_size); - for (p = low; (word)p <= (word)high && i <= max; ++p) { - word descr = (word)AO_load(&p->mse_descr.ao); - if (descr != 0) { - /* Must be ordered after read of descr: */ - AO_store_release_write(&p->mse_descr.ao, 0); - /* More than one thread may get this entry, but that's only */ - /* a minor performance problem. */ - ++top; - top->mse_descr.w = descr; - top->mse_start = p->mse_start; - GC_ASSERT((descr & GC_DS_TAGS) != GC_DS_LENGTH || descr < (word)GC_greatest_plausible_heap_addr - (word)GC_least_plausible_heap_addr || (word)(p->mse_start + descr) <= (word)GC_least_plausible_heap_addr || (word)p->mse_start >= (word)GC_greatest_plausible_heap_addr); - /* If this is a big object, count it as */ - /* size/256 + 1 objects. */ - ++i; - if ((descr & GC_DS_TAGS) == GC_DS_LENGTH) i += (int)(descr >> 8); - } - } - *next = p; - return top; -} - -/* Copy back a local mark stack. */ -/* low and high are inclusive bounds. */ -STATIC void GC_return_mark_stack(mse *low, mse *high) { - mse *my_top; - mse *my_start; - size_t stack_size; - - if ((word)high < (word)low) return; - stack_size = high - low + 1; - GC_acquire_mark_lock(); - my_top = GC_mark_stack_top; /* Concurrent modification impossible. */ - my_start = my_top + 1; - if ((word)(my_start - GC_mark_stack + stack_size) > (word)GC_mark_stack_size) { - GC_COND_LOG_PRINTF("No room to copy back mark stack\n"); - GC_mark_state = MS_INVALID; - GC_mark_stack_too_small = TRUE; - /* We drop the local mark stack. We'll fix things later. */ - } else { - BCOPY(low, my_start, stack_size * sizeof(mse)); - GC_ASSERT((mse *)AO_load((volatile AO_t *)(&GC_mark_stack_top)) == my_top); - AO_store_release_write((volatile AO_t *)(&GC_mark_stack_top), - (AO_t)(my_top + stack_size)); - /* Ensures visibility of previously written stack contents. */ - } - GC_release_mark_lock(); - GC_notify_all_marker(); -} - -#ifndef N_LOCAL_ITERS -#define N_LOCAL_ITERS 1 -#endif - -/* This function is only called when the local */ -/* and the main mark stacks are both empty. */ -static GC_bool has_inactive_helpers(void) { - GC_bool res; - - GC_acquire_mark_lock(); - res = GC_active_count < GC_helper_count; - GC_release_mark_lock(); - return res; -} - -/* Mark from the local mark stack. */ -/* On return, the local mark stack is empty. */ -/* But this may be achieved by copying the */ -/* local mark stack back into the global one. */ -/* We do not hold the mark lock. */ -STATIC void GC_do_local_mark(mse *local_mark_stack, mse *local_top) { - unsigned n; - - for (;;) { - for (n = 0; n < N_LOCAL_ITERS; ++n) { - local_top = GC_mark_from(local_top, local_mark_stack, - local_mark_stack + LOCAL_MARK_STACK_SIZE); - if ((word)local_top < (word)local_mark_stack) return; - if ((word)(local_top - local_mark_stack) >= LOCAL_MARK_STACK_SIZE / 2) { - GC_return_mark_stack(local_mark_stack, local_top); - return; - } - } - if ((word)AO_load((volatile AO_t *)&GC_mark_stack_top) < (word)AO_load(&GC_first_nonempty) && (word)local_top > (word)(local_mark_stack + 1) && has_inactive_helpers()) { - /* Try to share the load, since the main stack is empty, */ - /* and helper threads are waiting for a refill. */ - /* The entries near the bottom of the stack are likely */ - /* to require more work. Thus we return those, even though */ - /* it's harder. */ - mse *new_bottom = local_mark_stack + (local_top - local_mark_stack) / 2; - GC_ASSERT((word)new_bottom > (word)local_mark_stack && (word)new_bottom < (word)local_top); - GC_return_mark_stack(local_mark_stack, new_bottom - 1); - memmove(local_mark_stack, new_bottom, - (local_top - new_bottom + 1) * sizeof(mse)); - local_top -= (new_bottom - local_mark_stack); - } - } -} - -#ifndef ENTRIES_TO_GET -#define ENTRIES_TO_GET 5 -#endif - -/* Mark using the local mark stack until the global mark stack is empty */ -/* and there are no active workers. Update GC_first_nonempty to reflect */ -/* progress. Caller holds the mark lock. */ -/* Caller has already incremented GC_helper_count. We decrement it, */ -/* and maintain GC_active_count. */ -STATIC void GC_mark_local(mse *local_mark_stack, int id) { - mse *my_first_nonempty; - - GC_active_count++; - my_first_nonempty = (mse *)AO_load(&GC_first_nonempty); - GC_ASSERT((word)GC_mark_stack <= (word)my_first_nonempty); - GC_ASSERT((word)my_first_nonempty <= (word)AO_load((volatile AO_t *)&GC_mark_stack_top) + sizeof(mse)); - GC_VERBOSE_LOG_PRINTF("Starting mark helper %d\n", id); - GC_release_mark_lock(); - for (;;) { - size_t n_on_stack; - unsigned n_to_get; - mse *my_top; - mse *local_top; - mse *global_first_nonempty = (mse *)AO_load(&GC_first_nonempty); - - GC_ASSERT((word)my_first_nonempty >= (word)GC_mark_stack && - (word)my_first_nonempty <= - (word)AO_load((volatile AO_t *)&GC_mark_stack_top) + sizeof(mse)); - GC_ASSERT((word)global_first_nonempty >= (word)GC_mark_stack); - if ((word)my_first_nonempty < (word)global_first_nonempty) { - my_first_nonempty = global_first_nonempty; - } else if ((word)global_first_nonempty < (word)my_first_nonempty) { - (void)AO_compare_and_swap(&GC_first_nonempty, - (AO_t)global_first_nonempty, - (AO_t)my_first_nonempty); - /* If this fails, we just go ahead, without updating */ - /* GC_first_nonempty. */ - } - /* Perhaps we should also update GC_first_nonempty, if it */ - /* is less. But that would require using atomic updates. */ - my_top = (mse *)AO_load_acquire((volatile AO_t *)(&GC_mark_stack_top)); - if ((word)my_top < (word)my_first_nonempty) { - GC_acquire_mark_lock(); - my_top = GC_mark_stack_top; - /* Asynchronous modification impossible here, */ - /* since we hold mark lock. */ - n_on_stack = my_top - my_first_nonempty + 1; - if (0 == n_on_stack) { - GC_active_count--; - GC_ASSERT(GC_active_count <= GC_helper_count); - /* Other markers may redeposit objects */ - /* on the stack. */ - if (0 == GC_active_count) GC_notify_all_marker(); - while (GC_active_count > 0 && (word)AO_load(&GC_first_nonempty) > (word)GC_mark_stack_top) { - /* We will be notified if either GC_active_count */ - /* reaches zero, or if more objects are pushed on */ - /* the global mark stack. */ - GC_wait_marker(); - } - if (GC_active_count == 0 && (word)AO_load(&GC_first_nonempty) > (word)GC_mark_stack_top) { - GC_bool need_to_notify = FALSE; - /* The above conditions can't be falsified while we */ - /* hold the mark lock, since neither */ - /* GC_active_count nor GC_mark_stack_top can */ - /* change. GC_first_nonempty can only be */ - /* incremented asynchronously. Thus we know that */ - /* both conditions actually held simultaneously. */ - GC_helper_count--; - if (0 == GC_helper_count) need_to_notify = TRUE; - GC_VERBOSE_LOG_PRINTF("Finished mark helper %d\n", id); - if (need_to_notify) GC_notify_all_marker(); - return; - } - /* Else there's something on the stack again, or */ - /* another helper may push something. */ - GC_active_count++; - GC_ASSERT(GC_active_count > 0); - GC_release_mark_lock(); - continue; - } else { - GC_release_mark_lock(); - } - } else { - n_on_stack = my_top - my_first_nonempty + 1; - } - n_to_get = ENTRIES_TO_GET; - if (n_on_stack < 2 * ENTRIES_TO_GET) n_to_get = 1; - local_top = GC_steal_mark_stack(my_first_nonempty, my_top, - local_mark_stack, n_to_get, - &my_first_nonempty); - GC_ASSERT((word)my_first_nonempty >= (word)GC_mark_stack && - (word)my_first_nonempty <= - (word)AO_load((volatile AO_t *)&GC_mark_stack_top) + sizeof(mse)); - GC_do_local_mark(local_mark_stack, local_top); - } -} - -/* Perform parallel mark. We hold the GC lock, not the mark lock. */ -/* Currently runs until the mark stack is empty. */ -STATIC void GC_do_parallel_mark(void) { - GC_ASSERT(I_HOLD_LOCK()); - GC_acquire_mark_lock(); - - /* This could be a GC_ASSERT, but it seems safer to keep it on */ - /* all the time, especially since it's cheap. */ - if (GC_help_wanted || GC_active_count != 0 || GC_helper_count != 0) - ABORT("Tried to start parallel mark in bad state"); - GC_VERBOSE_LOG_PRINTF("Starting marking for mark phase number %lu\n", - (unsigned long)GC_mark_no); - GC_first_nonempty = (AO_t)GC_mark_stack; - GC_active_count = 0; - GC_helper_count = 1; - GC_help_wanted = TRUE; - GC_notify_all_marker(); - /* Wake up potential helpers. */ - GC_mark_local(GC_main_local_mark_stack, 0); - GC_help_wanted = FALSE; - /* Done; clean up. */ - while (GC_helper_count > 0) { - GC_wait_marker(); - } - /* GC_helper_count cannot be incremented while not GC_help_wanted. */ - GC_VERBOSE_LOG_PRINTF("Finished marking for mark phase number %lu\n", - (unsigned long)GC_mark_no); - GC_mark_no++; - GC_release_mark_lock(); - GC_notify_all_marker(); -} - -/* Try to help out the marker, if it's running. We hold the mark lock */ -/* only, the initiating thread holds the allocation lock. */ -GC_INNER void GC_help_marker(word my_mark_no) { -#define my_id my_id_mse.mse_descr.w - mse my_id_mse; /* align local_mark_stack explicitly */ - mse local_mark_stack[LOCAL_MARK_STACK_SIZE]; - /* Note: local_mark_stack is quite big (up to 128 KiB). */ - - GC_ASSERT(I_DONT_HOLD_LOCK()); - GC_ASSERT(GC_parallel); - while (GC_mark_no < my_mark_no || (!GC_help_wanted && GC_mark_no == my_mark_no)) { - GC_wait_marker(); - } - my_id = GC_helper_count; - if (GC_mark_no != my_mark_no || my_id > (unsigned)GC_markers_m1) { - /* Second test is useful only if original threads can also */ - /* act as helpers. Under Linux they can't. */ - return; - } - GC_helper_count = (unsigned)my_id + 1; - GC_mark_local(local_mark_stack, (int)my_id); - /* GC_mark_local decrements GC_helper_count. */ -#undef my_id -} - -#endif /* PARALLEL_MARK */ - -/* Allocate or reallocate space for mark stack of size n entries. */ -/* May silently fail. */ -static void alloc_mark_stack(size_t n) { -#ifdef GWW_VDB - static GC_bool GC_incremental_at_stack_alloc = FALSE; - - GC_bool recycle_old; -#endif - mse *new_stack; - - GC_ASSERT(I_HOLD_LOCK()); - new_stack = (mse *)GC_scratch_alloc(n * sizeof(struct GC_ms_entry)); -#ifdef GWW_VDB - /* Don't recycle a stack segment obtained with the wrong flags. */ - /* Win32 GetWriteWatch requires the right kind of memory. */ - recycle_old = !GC_auto_incremental || GC_incremental_at_stack_alloc; - GC_incremental_at_stack_alloc = GC_auto_incremental; -#endif - - GC_mark_stack_too_small = FALSE; - if (GC_mark_stack != NULL) { - if (new_stack != 0) { -#ifdef GWW_VDB - if (recycle_old) -#endif - { - /* Recycle old space. */ - GC_scratch_recycle_inner(GC_mark_stack, - GC_mark_stack_size * sizeof(struct GC_ms_entry)); - } - GC_mark_stack = new_stack; - GC_mark_stack_size = n; - /* FIXME: Do we need some way to reset GC_mark_stack_size? */ - GC_mark_stack_limit = new_stack + n; - GC_COND_LOG_PRINTF("Grew mark stack to %lu frames\n", - (unsigned long)GC_mark_stack_size); - } else { - WARN("Failed to grow mark stack to %" WARN_PRIuPTR " frames\n", n); - } - } else if (NULL == new_stack) { - GC_err_printf("No space for mark stack\n"); - EXIT(); - } else { - GC_mark_stack = new_stack; - GC_mark_stack_size = n; - GC_mark_stack_limit = new_stack + n; - } - GC_mark_stack_top = GC_mark_stack - 1; -} - -GC_INNER void GC_mark_init(void) { - alloc_mark_stack(INITIAL_MARK_STACK_SIZE); -} - -/* - * Push all locations between b and t onto the mark stack. - * b is the first location to be checked. t is one past the last - * location to be checked. - * Should only be used if there is no possibility of mark stack - * overflow. - */ -GC_API void GC_CALL GC_push_all(void *bottom, void *top) { - word length; - - bottom = (void *)(((word)bottom + ALIGNMENT - 1) & ~(ALIGNMENT - 1)); - top = (void *)((word)top & ~(ALIGNMENT - 1)); - if ((word)bottom >= (word)top) return; - - GC_mark_stack_top++; - if ((word)GC_mark_stack_top >= (word)GC_mark_stack_limit) { - ABORT("Unexpected mark stack overflow"); - } - length = (word)top - (word)bottom; -#if GC_DS_TAGS > ALIGNMENT - 1 - length += GC_DS_TAGS; - length &= ~GC_DS_TAGS; -#endif - GC_mark_stack_top->mse_start = (ptr_t)bottom; - GC_mark_stack_top->mse_descr.w = length; -} - -#ifndef GC_DISABLE_INCREMENTAL - -/* Analogous to the above, but push only those pages h with */ -/* dirty_fn(h) != 0. We use GC_push_all to actually push the block. */ -/* Used both to selectively push dirty pages, or to push a block in */ -/* piecemeal fashion, to allow for more marking concurrency. */ -/* Will not overflow mark stack if GC_push_all pushes a small fixed */ -/* number of entries. (This is invoked only if GC_push_all pushes */ -/* a single entry, or if it marks each object before pushing it, thus */ -/* ensuring progress in the event of a stack overflow.) */ -STATIC void GC_push_selected(ptr_t bottom, ptr_t top, - GC_bool (*dirty_fn)(struct hblk *)) { - struct hblk *h; - - bottom = (ptr_t)(((word)bottom + ALIGNMENT - 1) & ~(ALIGNMENT - 1)); - top = (ptr_t)(((word)top) & ~(ALIGNMENT - 1)); - if ((word)bottom >= (word)top) return; - - h = HBLKPTR(bottom + HBLKSIZE); - if ((word)top <= (word)h) { - if ((*dirty_fn)(h - 1)) { - GC_push_all(bottom, top); - } - return; - } - if ((*dirty_fn)(h - 1)) { - if ((word)(GC_mark_stack_top - GC_mark_stack) > 3 * GC_mark_stack_size / 4) { - GC_push_all(bottom, top); - return; - } - GC_push_all(bottom, h); - } - - while ((word)(h + 1) <= (word)top) { - if ((*dirty_fn)(h)) { - if ((word)(GC_mark_stack_top - GC_mark_stack) > 3 * GC_mark_stack_size / 4) { - /* Danger of mark stack overflow. */ - GC_push_all(h, top); - return; - } else { - GC_push_all(h, h + 1); - } - } - h++; - } - - if ((ptr_t)h != top && (*dirty_fn)(h)) { - GC_push_all(h, top); - } -} - -GC_API void GC_CALL GC_push_conditional(void *bottom, void *top, int all) { - if (!all) { - GC_push_selected((ptr_t)bottom, (ptr_t)top, GC_page_was_dirty); - } else { -#ifdef PROC_VDB - if (GC_auto_incremental) { - /* Pages that were never dirtied cannot contain pointers. */ - GC_push_selected((ptr_t)bottom, (ptr_t)top, GC_page_was_ever_dirty); - } else -#endif - /* else */ { - GC_push_all(bottom, top); - } - } -} - -#ifndef NO_VDB_FOR_STATIC_ROOTS -#ifndef PROC_VDB -/* Same as GC_page_was_dirty but h is allowed to point to some */ -/* page in the registered static roots only. Not used if */ -/* manual VDB is on. */ -STATIC GC_bool GC_static_page_was_dirty(struct hblk *h) { - return get_pht_entry_from_index(GC_grungy_pages, PHT_HASH(h)); -} -#endif - -GC_INNER void GC_push_conditional_static(void *bottom, void *top, - GC_bool all) { -#ifdef PROC_VDB - /* Just redirect to the generic routine because PROC_VDB */ - /* implementation gets the dirty bits map for the whole */ - /* process memory. */ - GC_push_conditional(bottom, top, all); -#else - if (all || !GC_is_vdb_for_static_roots()) { - GC_push_all(bottom, top); - } else { - GC_push_selected((ptr_t)bottom, (ptr_t)top, - GC_static_page_was_dirty); - } -#endif -} -#endif /* !NO_VDB_FOR_STATIC_ROOTS */ - -#else -GC_API void GC_CALL GC_push_conditional(void *bottom, void *top, int all) { - UNUSED_ARG(all); - GC_push_all(bottom, top); -} -#endif /* GC_DISABLE_INCREMENTAL */ - -#if defined(AMIGA) || defined(MACOS) || defined(GC_DARWIN_THREADS) -void GC_push_one(word p) { - GC_PUSH_ONE_STACK(p, MARKED_FROM_REGISTER); -} -#endif - -#ifdef GC_WIN32_THREADS -GC_INNER void GC_push_many_regs(const word *regs, unsigned count) { - unsigned i; - for (i = 0; i < count; i++) - GC_PUSH_ONE_STACK(regs[i], MARKED_FROM_REGISTER); -} -#endif - -GC_API struct GC_ms_entry *GC_CALL GC_mark_and_push(void *obj, - mse *mark_stack_ptr, mse *mark_stack_limit, void **src) { - hdr *hhdr; - - PREFETCH(obj); - GET_HDR(obj, hhdr); - if ((EXPECT(IS_FORWARDING_ADDR_OR_NIL(hhdr), FALSE) && (!GC_all_interior_pointers || NULL == (hhdr = GC_find_header((ptr_t)GC_base(obj))))) || EXPECT(HBLK_IS_FREE(hhdr), FALSE)) { - GC_ADD_TO_BLACK_LIST_NORMAL(obj, (ptr_t)src); - return mark_stack_ptr; - } - return GC_push_contents_hdr((ptr_t)obj, mark_stack_ptr, mark_stack_limit, - (ptr_t)src, hhdr, TRUE); -} - -/* Mark and push (i.e. gray) a single object p onto the main */ -/* mark stack. Consider p to be valid if it is an interior */ -/* pointer. */ -/* The object p has passed a preliminary pointer validity */ -/* test, but we do not definitely know whether it is valid. */ -/* Mark bits are NOT atomically updated. Thus this must be the */ -/* only thread setting them. */ -GC_ATTR_NO_SANITIZE_ADDR -GC_INNER void -#if defined(PRINT_BLACK_LIST) || defined(KEEP_BACK_PTRS) -GC_mark_and_push_stack(ptr_t p, ptr_t source) -#else -GC_mark_and_push_stack(ptr_t p) -#define source ((ptr_t)0) -#endif -{ - hdr *hhdr; - ptr_t r = p; - - PREFETCH(p); - GET_HDR(p, hhdr); - if (EXPECT(IS_FORWARDING_ADDR_OR_NIL(hhdr), FALSE)) { - if (NULL == hhdr || (r = (ptr_t)GC_base(p)) == NULL || (hhdr = HDR(r)) == NULL) { - GC_ADD_TO_BLACK_LIST_STACK(p, source); - return; - } - } - if (EXPECT(HBLK_IS_FREE(hhdr), FALSE)) { - GC_ADD_TO_BLACK_LIST_NORMAL(p, source); - return; - } -#ifdef THREADS - /* Pointer is on the stack. We may have dirtied the object */ - /* it points to, but have not called GC_dirty yet. */ - GC_dirty(p); /* entire object */ -#endif - GC_mark_stack_top = GC_push_contents_hdr(r, GC_mark_stack_top, - GC_mark_stack_limit, - source, hhdr, FALSE); - /* We silently ignore pointers to near the end of a block, */ - /* which is very mildly suboptimal. */ - /* FIXME: We should probably add a header word to address */ - /* this. */ -} -#undef source - -#ifdef TRACE_BUF - -#ifndef TRACE_ENTRIES -#define TRACE_ENTRIES 1000 -#endif - -struct trace_entry { - char *kind; - word gc_no; - word bytes_allocd; - word arg1; - word arg2; -} GC_trace_buf[TRACE_ENTRIES] = {{NULL, 0, 0, 0, 0}}; - -void GC_add_trace_entry(char *kind, word arg1, word arg2) { - GC_trace_buf[GC_trace_buf_ptr].kind = kind; - GC_trace_buf[GC_trace_buf_ptr].gc_no = GC_gc_no; - GC_trace_buf[GC_trace_buf_ptr].bytes_allocd = GC_bytes_allocd; - GC_trace_buf[GC_trace_buf_ptr].arg1 = arg1 ^ 0x80000000; - GC_trace_buf[GC_trace_buf_ptr].arg2 = arg2 ^ 0x80000000; - GC_trace_buf_ptr++; - if (GC_trace_buf_ptr >= TRACE_ENTRIES) GC_trace_buf_ptr = 0; -} - -GC_API void GC_CALL GC_print_trace_inner(word gc_no) { - int i; - - for (i = GC_trace_buf_ptr - 1; i != GC_trace_buf_ptr; i--) { - struct trace_entry *p; - - if (i < 0) i = TRACE_ENTRIES - 1; - p = GC_trace_buf + i; - if (p->gc_no < gc_no || p->kind == 0) { - return; - } - GC_printf("Trace:%s (gc:%u, bytes:%lu) 0x%lX, 0x%lX\n", - p->kind, (unsigned)p->gc_no, - (unsigned long)p->bytes_allocd, - (long)p->arg1 ^ 0x80000000L, (long)p->arg2 ^ 0x80000000L); - } - GC_printf("Trace incomplete\n"); -} - -GC_API void GC_CALL GC_print_trace(word gc_no) { - LOCK(); - GC_print_trace_inner(gc_no); - UNLOCK(); -} - -#endif /* TRACE_BUF */ - -/* A version of GC_push_all that treats all interior pointers as valid */ -/* and scans the entire region immediately, in case the contents change.*/ -GC_ATTR_NO_SANITIZE_ADDR GC_ATTR_NO_SANITIZE_MEMORY GC_ATTR_NO_SANITIZE_THREAD - GC_API void GC_CALL - GC_push_all_eager(void *bottom, void *top) { - REGISTER ptr_t current_p; - REGISTER word *lim; - REGISTER ptr_t greatest_ha = (ptr_t)GC_greatest_plausible_heap_addr; - REGISTER ptr_t least_ha = (ptr_t)GC_least_plausible_heap_addr; -#define GC_greatest_plausible_heap_addr greatest_ha -#define GC_least_plausible_heap_addr least_ha - - if (top == 0) return; - - /* Check all pointers in range and push if they appear to be valid. */ - lim = (word *)(((word)top) & ~(ALIGNMENT - 1)) - 1; - for (current_p = (ptr_t)(((word)bottom + ALIGNMENT - 1) & ~(ALIGNMENT - 1)); - (word)current_p <= (word)lim; current_p += ALIGNMENT) { - REGISTER word q; - - LOAD_WORD_OR_CONTINUE(q, current_p); - GC_PUSH_ONE_STACK(q, current_p); - } -#undef GC_greatest_plausible_heap_addr -#undef GC_least_plausible_heap_addr -} - -GC_INNER void GC_push_all_stack(ptr_t bottom, ptr_t top) { -#ifndef NEED_FIXUP_POINTER - if (GC_all_interior_pointers -#if defined(THREADS) && defined(MPROTECT_VDB) - && !GC_auto_incremental -#endif - && (word)GC_mark_stack_top < (word)(GC_mark_stack_limit - INITIAL_MARK_STACK_SIZE / 8)) { - GC_push_all(bottom, top); - } else -#endif - /* else */ { - GC_push_all_eager(bottom, top); - } -} - -#if defined(WRAP_MARK_SOME) && defined(PARALLEL_MARK) -/* Similar to GC_push_conditional but scans the whole region immediately. */ -GC_ATTR_NO_SANITIZE_ADDR GC_ATTR_NO_SANITIZE_MEMORY - GC_ATTR_NO_SANITIZE_THREAD - GC_INNER void - GC_push_conditional_eager(void *bottom, void *top, - GC_bool all) { - REGISTER ptr_t current_p; - REGISTER word *lim; - REGISTER ptr_t greatest_ha = (ptr_t)GC_greatest_plausible_heap_addr; - REGISTER ptr_t least_ha = (ptr_t)GC_least_plausible_heap_addr; -#define GC_greatest_plausible_heap_addr greatest_ha -#define GC_least_plausible_heap_addr least_ha - - if (top == NULL) - return; - (void)all; /* TODO: If !all then scan only dirty pages. */ - - lim = (word *)(((word)top) & ~(ALIGNMENT - 1)) - 1; - for (current_p = (ptr_t)(((word)bottom + ALIGNMENT - 1) & ~(ALIGNMENT - 1)); - (word)current_p <= (word)lim; current_p += ALIGNMENT) { - REGISTER word q; - - LOAD_WORD_OR_CONTINUE(q, current_p); - GC_PUSH_ONE_HEAP(q, current_p, GC_mark_stack_top); - } -#undef GC_greatest_plausible_heap_addr -#undef GC_least_plausible_heap_addr -} -#endif /* WRAP_MARK_SOME && PARALLEL_MARK */ - -#if !defined(SMALL_CONFIG) && !defined(USE_MARK_BYTES) && \ - defined(MARK_BIT_PER_GRANULE) -#if GC_GRANULE_WORDS == 1 -#define USE_PUSH_MARKED_ACCELERATORS -#define PUSH_GRANULE(q) \ - do { \ - word qcontents = (q)[0]; \ - GC_PUSH_ONE_HEAP(qcontents, q, GC_mark_stack_top); \ - } while (0) -#elif GC_GRANULE_WORDS == 2 -#define USE_PUSH_MARKED_ACCELERATORS -#define PUSH_GRANULE(q) \ - do { \ - word qcontents = (q)[0]; \ - GC_PUSH_ONE_HEAP(qcontents, q, GC_mark_stack_top); \ - qcontents = (q)[1]; \ - GC_PUSH_ONE_HEAP(qcontents, (q) + 1, GC_mark_stack_top); \ - } while (0) -#elif GC_GRANULE_WORDS == 4 -#define USE_PUSH_MARKED_ACCELERATORS -#define PUSH_GRANULE(q) \ - do { \ - word qcontents = (q)[0]; \ - GC_PUSH_ONE_HEAP(qcontents, q, GC_mark_stack_top); \ - qcontents = (q)[1]; \ - GC_PUSH_ONE_HEAP(qcontents, (q) + 1, GC_mark_stack_top); \ - qcontents = (q)[2]; \ - GC_PUSH_ONE_HEAP(qcontents, (q) + 2, GC_mark_stack_top); \ - qcontents = (q)[3]; \ - GC_PUSH_ONE_HEAP(qcontents, (q) + 3, GC_mark_stack_top); \ - } while (0) -#endif -#endif /* !USE_MARK_BYTES && MARK_BIT_PER_GRANULE */ - -#ifdef USE_PUSH_MARKED_ACCELERATORS -/* Push all objects reachable from marked objects in the given block */ -/* containing objects of size 1 granule. */ -GC_ATTR_NO_SANITIZE_THREAD -STATIC void GC_push_marked1(struct hblk *h, hdr *hhdr) { - word *mark_word_addr = &(hhdr->hb_marks[0]); - word *p; - word *plim; - - /* Allow registers to be used for some frequently accessed */ - /* global variables. Otherwise aliasing issues are likely */ - /* to prevent that. */ - ptr_t greatest_ha = (ptr_t)GC_greatest_plausible_heap_addr; - ptr_t least_ha = (ptr_t)GC_least_plausible_heap_addr; - mse *mark_stack_top = GC_mark_stack_top; - mse *mark_stack_limit = GC_mark_stack_limit; - -#undef GC_mark_stack_top -#undef GC_mark_stack_limit -#define GC_mark_stack_top mark_stack_top -#define GC_mark_stack_limit mark_stack_limit -#define GC_greatest_plausible_heap_addr greatest_ha -#define GC_least_plausible_heap_addr least_ha - - p = (word *)(h->hb_body); - plim = (word *)(((word)h) + HBLKSIZE); - - /* Go through all words in block. */ - while ((word)p < (word)plim) { - word mark_word = *mark_word_addr++; - word *q = p; - - while (mark_word != 0) { - if (mark_word & 1) { - PUSH_GRANULE(q); - } - q += GC_GRANULE_WORDS; - mark_word >>= 1; - } - p += WORDSZ * GC_GRANULE_WORDS; - } - -#undef GC_greatest_plausible_heap_addr -#undef GC_least_plausible_heap_addr -#undef GC_mark_stack_top -#undef GC_mark_stack_limit -#define GC_mark_stack_limit GC_arrays._mark_stack_limit -#define GC_mark_stack_top GC_arrays._mark_stack_top - GC_mark_stack_top = mark_stack_top; -} - -#ifndef UNALIGNED_PTRS - -/* Push all objects reachable from marked objects in the given block */ -/* of size 2 (granules) objects. */ -GC_ATTR_NO_SANITIZE_THREAD -STATIC void GC_push_marked2(struct hblk *h, hdr *hhdr) { - word *mark_word_addr = &(hhdr->hb_marks[0]); - word *p; - word *plim; - - ptr_t greatest_ha = (ptr_t)GC_greatest_plausible_heap_addr; - ptr_t least_ha = (ptr_t)GC_least_plausible_heap_addr; - mse *mark_stack_top = GC_mark_stack_top; - mse *mark_stack_limit = GC_mark_stack_limit; - -#undef GC_mark_stack_top -#undef GC_mark_stack_limit -#define GC_mark_stack_top mark_stack_top -#define GC_mark_stack_limit mark_stack_limit -#define GC_greatest_plausible_heap_addr greatest_ha -#define GC_least_plausible_heap_addr least_ha - - p = (word *)(h->hb_body); - plim = (word *)(((word)h) + HBLKSIZE); - - /* Go through all words in block. */ - while ((word)p < (word)plim) { - word mark_word = *mark_word_addr++; - word *q = p; - - while (mark_word != 0) { - if (mark_word & 1) { - PUSH_GRANULE(q); - PUSH_GRANULE(q + GC_GRANULE_WORDS); - } - q += 2 * GC_GRANULE_WORDS; - mark_word >>= 2; - } - p += WORDSZ * GC_GRANULE_WORDS; - } - -#undef GC_greatest_plausible_heap_addr -#undef GC_least_plausible_heap_addr -#undef GC_mark_stack_top -#undef GC_mark_stack_limit -#define GC_mark_stack_limit GC_arrays._mark_stack_limit -#define GC_mark_stack_top GC_arrays._mark_stack_top - GC_mark_stack_top = mark_stack_top; -} - -#if GC_GRANULE_WORDS < 4 -/* Push all objects reachable from marked objects in the given block */ -/* of size 4 (granules) objects. */ -/* There is a risk of mark stack overflow here. But we handle that. */ -/* And only unmarked objects get pushed, so it's not very likely. */ -GC_ATTR_NO_SANITIZE_THREAD -STATIC void GC_push_marked4(struct hblk *h, hdr *hhdr) { - word *mark_word_addr = &(hhdr->hb_marks[0]); - word *p; - word *plim; - - ptr_t greatest_ha = (ptr_t)GC_greatest_plausible_heap_addr; - ptr_t least_ha = (ptr_t)GC_least_plausible_heap_addr; - mse *mark_stack_top = GC_mark_stack_top; - mse *mark_stack_limit = GC_mark_stack_limit; - -#undef GC_mark_stack_top -#undef GC_mark_stack_limit -#define GC_mark_stack_top mark_stack_top -#define GC_mark_stack_limit mark_stack_limit -#define GC_greatest_plausible_heap_addr greatest_ha -#define GC_least_plausible_heap_addr least_ha - - p = (word *)(h->hb_body); - plim = (word *)(((word)h) + HBLKSIZE); - - /* Go through all words in block. */ - while ((word)p < (word)plim) { - word mark_word = *mark_word_addr++; - word *q = p; - - while (mark_word != 0) { - if (mark_word & 1) { - PUSH_GRANULE(q); - PUSH_GRANULE(q + GC_GRANULE_WORDS); - PUSH_GRANULE(q + 2 * GC_GRANULE_WORDS); - PUSH_GRANULE(q + 3 * GC_GRANULE_WORDS); - } - q += 4 * GC_GRANULE_WORDS; - mark_word >>= 4; - } - p += WORDSZ * GC_GRANULE_WORDS; - } -#undef GC_greatest_plausible_heap_addr -#undef GC_least_plausible_heap_addr -#undef GC_mark_stack_top -#undef GC_mark_stack_limit -#define GC_mark_stack_limit GC_arrays._mark_stack_limit -#define GC_mark_stack_top GC_arrays._mark_stack_top - GC_mark_stack_top = mark_stack_top; -} - -#endif /* GC_GRANULE_WORDS < 4 */ - -#endif /* UNALIGNED_PTRS */ - -#endif /* USE_PUSH_MARKED_ACCELERATORS */ - -/* Push all objects reachable from marked objects in the given block. */ -STATIC void GC_push_marked(struct hblk *h, hdr *hhdr) { - word sz = hhdr->hb_sz; - word descr = hhdr->hb_descr; - ptr_t p; - word bit_no; - ptr_t lim; - mse *GC_mark_stack_top_reg; - mse *mark_stack_limit = GC_mark_stack_limit; - - /* Some quick shortcuts: */ - if ((/* 0 | */ GC_DS_LENGTH) == descr) return; - if (GC_block_empty(hhdr) /* nothing marked */) return; -#if !defined(GC_DISABLE_INCREMENTAL) - GC_n_rescuing_pages++; -#endif - GC_objects_are_marked = TRUE; - if (sz > MAXOBJBYTES) { - lim = h->hb_body; - } else { - lim = (ptr_t)((word)(h + 1)->hb_body - sz); - } - - switch (BYTES_TO_GRANULES(sz)) { -#if defined(USE_PUSH_MARKED_ACCELERATORS) - case 1: - GC_push_marked1(h, hhdr); - break; -#if !defined(UNALIGNED_PTRS) - case 2: - GC_push_marked2(h, hhdr); - break; -#if GC_GRANULE_WORDS < 4 - case 4: - GC_push_marked4(h, hhdr); - break; -#endif -#endif /* !UNALIGNED_PTRS */ -#else - case 1: /* to suppress "switch statement contains no case" warning */ -#endif - default: - GC_mark_stack_top_reg = GC_mark_stack_top; - for (p = h->hb_body, bit_no = 0; (word)p <= (word)lim; - p += sz, bit_no += MARK_BIT_OFFSET(sz)) { - if (mark_bit_from_hdr(hhdr, bit_no)) { - /* Mark from fields inside the object. */ - GC_mark_stack_top_reg = GC_push_obj(p, hhdr, GC_mark_stack_top_reg, - mark_stack_limit); - } - } - GC_mark_stack_top = GC_mark_stack_top_reg; - } -} - -#ifdef ENABLE_DISCLAIM -/* Unconditionally mark from all objects which have not been reclaimed. */ -/* This is useful in order to retain pointers which are reachable from */ -/* the disclaim notifiers. */ -/* To determine whether an object has been reclaimed, we require that */ -/* any live object has a non-zero as one of the two lowest bits of the */ -/* first word. On the other hand, a reclaimed object is a members of */ -/* free-lists, and thus contains a word-aligned next-pointer as the */ -/* first word. */ -GC_ATTR_NO_SANITIZE_THREAD -STATIC void GC_push_unconditionally(struct hblk *h, hdr *hhdr) { - word sz = hhdr->hb_sz; - word descr = hhdr->hb_descr; - ptr_t p; - ptr_t lim; - mse *GC_mark_stack_top_reg; - mse *mark_stack_limit = GC_mark_stack_limit; - - if ((/* 0 | */ GC_DS_LENGTH) == descr) - return; - -#if !defined(GC_DISABLE_INCREMENTAL) - GC_n_rescuing_pages++; -#endif - GC_objects_are_marked = TRUE; - if (sz > MAXOBJBYTES) - lim = h->hb_body; - else - lim = (ptr_t)((word)(h + 1)->hb_body - sz); - - GC_mark_stack_top_reg = GC_mark_stack_top; - for (p = h->hb_body; (word)p <= (word)lim; p += sz) - if ((*(word *)p & 0x3) != 0) - GC_mark_stack_top_reg = GC_push_obj(p, hhdr, GC_mark_stack_top_reg, - mark_stack_limit); - GC_mark_stack_top = GC_mark_stack_top_reg; -} -#endif /* ENABLE_DISCLAIM */ - -#ifndef GC_DISABLE_INCREMENTAL -/* Test whether any page in the given block is dirty. */ -STATIC GC_bool GC_block_was_dirty(struct hblk *h, hdr *hhdr) { - word sz; - ptr_t p; - -#ifdef AO_HAVE_load - /* Atomic access is used to avoid racing with GC_realloc. */ - sz = (word)AO_load((volatile AO_t *)&(hhdr->hb_sz)); -#else - sz = hhdr->hb_sz; -#endif - if (sz <= MAXOBJBYTES) { - return GC_page_was_dirty(h); - } - - for (p = (ptr_t)h; (word)p < (word)h + sz; p += HBLKSIZE) { - if (GC_page_was_dirty((struct hblk *)p)) return TRUE; - } - return FALSE; -} -#endif /* GC_DISABLE_INCREMENTAL */ - -/* Similar to GC_push_marked, but skip over unallocated blocks */ -/* and return address of next plausible block. */ -STATIC struct hblk *GC_push_next_marked(struct hblk *h) { - hdr *hhdr = HDR(h); - - if (EXPECT(IS_FORWARDING_ADDR_OR_NIL(hhdr) || HBLK_IS_FREE(hhdr), FALSE)) { - h = GC_next_block(h, FALSE); - if (NULL == h) return NULL; - hhdr = GC_find_header((ptr_t)h); - } else { -#ifdef LINT2 - if (NULL == h) ABORT("Bad HDR() definition"); -#endif - } - GC_push_marked(h, hhdr); - return h + OBJ_SZ_TO_BLOCKS(hhdr->hb_sz); -} - -#ifndef GC_DISABLE_INCREMENTAL -/* Identical to above, but mark only from dirty pages. */ -STATIC struct hblk *GC_push_next_marked_dirty(struct hblk *h) { - hdr *hhdr = HDR(h); - - if (!GC_incremental) ABORT("Dirty bits not set up"); - for (;;) { - if (EXPECT(IS_FORWARDING_ADDR_OR_NIL(hhdr) || HBLK_IS_FREE(hhdr), FALSE)) { - h = GC_next_block(h, FALSE); - if (NULL == h) return NULL; - hhdr = GC_find_header((ptr_t)h); - } else { -#ifdef LINT2 - if (NULL == h) ABORT("Bad HDR() definition"); -#endif - } - if (GC_block_was_dirty(h, hhdr)) - break; - h += OBJ_SZ_TO_BLOCKS(hhdr->hb_sz); - hhdr = HDR(h); - } -#ifdef ENABLE_DISCLAIM - if ((hhdr->hb_flags & MARK_UNCONDITIONALLY) != 0) { - GC_push_unconditionally(h, hhdr); - - /* Then we may ask, why not also add the MARK_UNCONDITIONALLY */ - /* case to GC_push_next_marked, which is also applied to */ - /* uncollectible blocks? But it seems to me that the function */ - /* does not need to scan uncollectible (and unconditionally */ - /* marked) blocks since those are already handled in the */ - /* MS_PUSH_UNCOLLECTABLE phase. */ - } else -#endif - /* else */ { - GC_push_marked(h, hhdr); - } - return h + OBJ_SZ_TO_BLOCKS(hhdr->hb_sz); -} -#endif /* !GC_DISABLE_INCREMENTAL */ - -/* Similar to above, but for uncollectible pages. Needed since we */ -/* do not clear marks for such pages, even for full collections. */ -STATIC struct hblk *GC_push_next_marked_uncollectable(struct hblk *h) { - hdr *hhdr = HDR(h); - - for (;;) { - if (EXPECT(IS_FORWARDING_ADDR_OR_NIL(hhdr) || HBLK_IS_FREE(hhdr), FALSE)) { - h = GC_next_block(h, FALSE); - if (NULL == h) return NULL; - hhdr = GC_find_header((ptr_t)h); - } else { -#ifdef LINT2 - if (NULL == h) ABORT("Bad HDR() definition"); -#endif - } - if (hhdr->hb_obj_kind == UNCOLLECTABLE) { - GC_push_marked(h, hhdr); - break; - } -#ifdef ENABLE_DISCLAIM - if ((hhdr->hb_flags & MARK_UNCONDITIONALLY) != 0) { - GC_push_unconditionally(h, hhdr); - break; - } -#endif - h += OBJ_SZ_TO_BLOCKS(hhdr->hb_sz); - hhdr = HDR(h); - } - return h + OBJ_SZ_TO_BLOCKS(hhdr->hb_sz); -} diff --git a/vendor/bdwgc/mark_rts.c b/vendor/bdwgc/mark_rts.c deleted file mode 100644 index 129ea401..00000000 --- a/vendor/bdwgc/mark_rts.c +++ /dev/null @@ -1,904 +0,0 @@ -/* - * Copyright 1988, 1989 Hans-J. Boehm, Alan J. Demers - * Copyright (c) 1991-1994 by Xerox Corporation. All rights reserved. - * Copyright (c) 2009-2022 Ivan Maidanski - * - * THIS MATERIAL IS PROVIDED AS IS, WITH ABSOLUTELY NO WARRANTY EXPRESSED - * OR IMPLIED. ANY USE IS AT YOUR OWN RISK. - * - * Permission is hereby granted to use or copy this program - * for any purpose, provided the above notices are retained on all copies. - * Permission to modify the code and to distribute modified code is granted, - * provided the above notices are retained, and a notice that the code was - * modified is included with the above copyright notice. - */ - -#ifdef _WIN32 -#define WIN32_LEAN_AND_MEAN -#include -#endif -#include "private/gc_priv.h" - -/* Data structure for list of root sets. */ -/* We keep a hash table, so that we can filter out duplicate additions. */ -/* Under Win32, we need to do a better job of filtering overlaps, so */ -/* we resort to sequential search, and pay the price. */ -/* This is really declared in gc_priv.h: -struct roots { - ptr_t r_start; - ptr_t r_end; -# if !defined(MSWIN32) && !defined(MSWINCE) && !defined(CYGWIN32) - struct roots * r_next; -# endif - GC_bool r_tmp; - -- Delete before registering new dynamic libraries -}; - -struct roots GC_static_roots[MAX_ROOT_SETS]; -*/ - -int GC_no_dls = 0; /* Register dynamic library data segments. */ - -#if !defined(NO_DEBUGGING) || defined(GC_ASSERTIONS) -/* Should return the same value as GC_root_size. */ -GC_INNER word GC_compute_root_size(void) { - int i; - word size = 0; - - for (i = 0; i < n_root_sets; i++) { - size += GC_static_roots[i].r_end - GC_static_roots[i].r_start; - } - return size; -} -#endif /* !NO_DEBUGGING || GC_ASSERTIONS */ - -#if !defined(NO_DEBUGGING) -/* For debugging: */ -void GC_print_static_roots(void) { - int i; - word size; - - for (i = 0; i < n_root_sets; i++) { - GC_printf("From %p to %p%s\n", - (void *)GC_static_roots[i].r_start, - (void *)GC_static_roots[i].r_end, - GC_static_roots[i].r_tmp ? " (temporary)" : ""); - } - GC_printf("GC_root_size= %lu\n", (unsigned long)GC_root_size); - - if ((size = GC_compute_root_size()) != GC_root_size) - GC_err_printf("GC_root_size incorrect!! Should be: %lu\n", - (unsigned long)size); -} -#endif /* !NO_DEBUGGING */ - -#ifndef THREADS -/* Primarily for debugging support: */ -/* Is the address p in one of the registered static root sections? */ -GC_INNER GC_bool GC_is_static_root(void *p) { - static int last_root_set = MAX_ROOT_SETS; - int i; - - if (last_root_set < n_root_sets && (word)p >= (word)GC_static_roots[last_root_set].r_start && (word)p < (word)GC_static_roots[last_root_set].r_end) - return TRUE; - for (i = 0; i < n_root_sets; i++) { - if ((word)p >= (word)GC_static_roots[i].r_start && (word)p < (word)GC_static_roots[i].r_end) { - last_root_set = i; - return TRUE; - } - } - return FALSE; -} -#endif /* !THREADS */ - -#if !defined(MSWIN32) && !defined(MSWINCE) && !defined(CYGWIN32) -/* -# define LOG_RT_SIZE 6 -# define RT_SIZE (1 << LOG_RT_SIZE) -- Power of 2, may be != MAX_ROOT_SETS - - struct roots * GC_root_index[RT_SIZE]; - -- Hash table header. Used only to check whether a range is - -- already present. - -- really defined in gc_priv.h -*/ - -GC_INLINE int rt_hash(ptr_t addr) { - word result = (word)addr; - -#if CPP_WORDSZ > 8 * LOG_RT_SIZE - result ^= result >> 8 * LOG_RT_SIZE; -#endif -#if CPP_WORDSZ > 4 * LOG_RT_SIZE - result ^= result >> 4 * LOG_RT_SIZE; -#endif - result ^= result >> 2 * LOG_RT_SIZE; - result ^= result >> LOG_RT_SIZE; - return result & (RT_SIZE - 1); -} - -/* Is a range starting at b already in the table? If so return a */ -/* pointer to it, else NULL. */ -GC_INNER void *GC_roots_present(ptr_t b) { - int h = rt_hash(b); - struct roots *p; - - for (p = GC_root_index[h]; p != NULL; p = p->r_next) { - if (p->r_start == (ptr_t)b) break; - } - return p; -} - -/* Add the given root structure to the index. */ -GC_INLINE void add_roots_to_index(struct roots *p) { - int h = rt_hash(p->r_start); - - p->r_next = GC_root_index[h]; - GC_root_index[h] = p; -} -#endif /* !MSWIN32 && !MSWINCE && !CYGWIN32 */ - -GC_INNER word GC_root_size = 0; - -GC_API void GC_CALL GC_add_roots(void *b, void *e) { - if (!EXPECT(GC_is_initialized, TRUE)) GC_init(); - LOCK(); - GC_add_roots_inner((ptr_t)b, (ptr_t)e, FALSE); - UNLOCK(); -} - -/* Add [b,e) to the root set. Adding the same interval a second time */ -/* is a moderately fast no-op, and hence benign. We do not handle */ -/* different but overlapping intervals efficiently. (We do handle */ -/* them correctly.) */ -/* Tmp specifies that the interval may be deleted before */ -/* re-registering dynamic libraries. */ -void GC_add_roots_inner(ptr_t b, ptr_t e, GC_bool tmp) { - GC_ASSERT((word)b <= (word)e); - b = (ptr_t)(((word)b + (sizeof(word) - 1)) & ~(word)(sizeof(word) - 1)); - /* round b up to word boundary */ - e = (ptr_t)((word)e & ~(word)(sizeof(word) - 1)); - /* round e down to word boundary */ - if ((word)b >= (word)e) return; /* nothing to do */ - -#if defined(MSWIN32) || defined(MSWINCE) || defined(CYGWIN32) - /* Spend the time to ensure that there are no overlapping */ - /* or adjacent intervals. */ - /* This could be done faster with e.g. a */ - /* balanced tree. But the execution time here is */ - /* virtually guaranteed to be dominated by the time it */ - /* takes to scan the roots. */ - { - int i; - struct roots *old = NULL; /* initialized to prevent warning. */ - - for (i = 0; i < n_root_sets; i++) { - old = GC_static_roots + i; - if ((word)b <= (word)old->r_end && (word)e >= (word)old->r_start) { - if ((word)b < (word)old->r_start) { - GC_root_size += old->r_start - b; - old->r_start = b; - } - if ((word)e > (word)old->r_end) { - GC_root_size += e - old->r_end; - old->r_end = e; - } - old->r_tmp &= tmp; - break; - } - } - if (i < n_root_sets) { - /* merge other overlapping intervals */ - struct roots *other; - - for (i++; i < n_root_sets; i++) { - other = GC_static_roots + i; - b = other->r_start; - e = other->r_end; - if ((word)b <= (word)old->r_end && (word)e >= (word)old->r_start) { - if ((word)b < (word)old->r_start) { - GC_root_size += old->r_start - b; - old->r_start = b; - } - if ((word)e > (word)old->r_end) { - GC_root_size += e - old->r_end; - old->r_end = e; - } - old->r_tmp &= other->r_tmp; - /* Delete this entry. */ - GC_root_size -= (other->r_end - other->r_start); - other->r_start = GC_static_roots[n_root_sets - 1].r_start; - other->r_end = GC_static_roots[n_root_sets - 1].r_end; - n_root_sets--; - } - } - return; - } - } -#else - { - struct roots *old = (struct roots *)GC_roots_present(b); - - if (old != 0) { - if ((word)e <= (word)old->r_end) { - old->r_tmp &= tmp; - return; /* already there */ - } - if (old->r_tmp == tmp || !tmp) { - /* Extend the existing root. */ - GC_root_size += e - old->r_end; - old->r_end = e; - old->r_tmp = tmp; - return; - } - b = old->r_end; - } - } -#endif - if (n_root_sets == MAX_ROOT_SETS) { - ABORT("Too many root sets"); - } - -#ifdef DEBUG_ADD_DEL_ROOTS - GC_log_printf("Adding data root section %d: %p .. %p%s\n", - n_root_sets, (void *)b, (void *)e, - tmp ? " (temporary)" : ""); -#endif - GC_static_roots[n_root_sets].r_start = (ptr_t)b; - GC_static_roots[n_root_sets].r_end = (ptr_t)e; - GC_static_roots[n_root_sets].r_tmp = tmp; -#if !defined(MSWIN32) && !defined(MSWINCE) && !defined(CYGWIN32) - GC_static_roots[n_root_sets].r_next = 0; - add_roots_to_index(GC_static_roots + n_root_sets); -#endif - GC_root_size += e - b; - n_root_sets++; -} - -GC_API void GC_CALL GC_clear_roots(void) { - if (!EXPECT(GC_is_initialized, TRUE)) GC_init(); - LOCK(); -#ifdef THREADS - GC_roots_were_cleared = TRUE; -#endif - n_root_sets = 0; - GC_root_size = 0; -#if !defined(MSWIN32) && !defined(MSWINCE) && !defined(CYGWIN32) - BZERO(GC_root_index, RT_SIZE * sizeof(void *)); -#endif -#ifdef DEBUG_ADD_DEL_ROOTS - GC_log_printf("Clear all data root sections\n"); -#endif - UNLOCK(); -} - -STATIC void GC_remove_root_at_pos(int i) { - GC_ASSERT(I_HOLD_LOCK()); -#ifdef DEBUG_ADD_DEL_ROOTS - GC_log_printf("Remove data root section at %d: %p .. %p%s\n", - i, (void *)GC_static_roots[i].r_start, - (void *)GC_static_roots[i].r_end, - GC_static_roots[i].r_tmp ? " (temporary)" : ""); -#endif - GC_root_size -= (GC_static_roots[i].r_end - GC_static_roots[i].r_start); - GC_static_roots[i].r_start = GC_static_roots[n_root_sets - 1].r_start; - GC_static_roots[i].r_end = GC_static_roots[n_root_sets - 1].r_end; - GC_static_roots[i].r_tmp = GC_static_roots[n_root_sets - 1].r_tmp; - n_root_sets--; -} - -#if !defined(MSWIN32) && !defined(MSWINCE) && !defined(CYGWIN32) -STATIC void GC_rebuild_root_index(void) { - int i; - BZERO(GC_root_index, RT_SIZE * sizeof(void *)); - for (i = 0; i < n_root_sets; i++) - add_roots_to_index(GC_static_roots + i); -} -#endif - -#if defined(DYNAMIC_LOADING) || defined(MSWIN32) || defined(MSWINCE) || defined(PCR) || defined(CYGWIN32) -STATIC void GC_remove_tmp_roots(void) { - int i; -#if !defined(MSWIN32) && !defined(MSWINCE) && !defined(CYGWIN32) - int old_n_roots = n_root_sets; -#endif - - GC_ASSERT(I_HOLD_LOCK()); - for (i = 0; i < n_root_sets;) { - if (GC_static_roots[i].r_tmp) { - GC_remove_root_at_pos(i); - } else { - i++; - } - } -#if !defined(MSWIN32) && !defined(MSWINCE) && !defined(CYGWIN32) - if (n_root_sets < old_n_roots) - GC_rebuild_root_index(); -#endif -} -#endif - -STATIC void GC_remove_roots_inner(ptr_t b, ptr_t e); - -GC_API void GC_CALL GC_remove_roots(void *b, void *e) { - /* Quick check whether has nothing to do */ - if ((((word)b + (sizeof(word) - 1)) & ~(word)(sizeof(word) - 1)) >= - ((word)e & ~(word)(sizeof(word) - 1))) - return; - - LOCK(); - GC_remove_roots_inner((ptr_t)b, (ptr_t)e); - UNLOCK(); -} - -STATIC void GC_remove_roots_inner(ptr_t b, ptr_t e) { - int i; -#if !defined(MSWIN32) && !defined(MSWINCE) && !defined(CYGWIN32) - int old_n_roots = n_root_sets; -#endif - - GC_ASSERT(I_HOLD_LOCK()); - for (i = 0; i < n_root_sets;) { - if ((word)GC_static_roots[i].r_start >= (word)b && (word)GC_static_roots[i].r_end <= (word)e) { - GC_remove_root_at_pos(i); - } else { - i++; - } - } -#if !defined(MSWIN32) && !defined(MSWINCE) && !defined(CYGWIN32) - if (n_root_sets < old_n_roots) - GC_rebuild_root_index(); -#endif -} - -#ifdef USE_PROC_FOR_LIBRARIES -/* Exchange the elements of the roots table. Requires rebuild of */ -/* the roots index table after the swap. */ -GC_INLINE void swap_static_roots(int i, int j) { - ptr_t r_start = GC_static_roots[i].r_start; - ptr_t r_end = GC_static_roots[i].r_end; - GC_bool r_tmp = GC_static_roots[i].r_tmp; - - GC_static_roots[i].r_start = GC_static_roots[j].r_start; - GC_static_roots[i].r_end = GC_static_roots[j].r_end; - GC_static_roots[i].r_tmp = GC_static_roots[j].r_tmp; - /* No need to swap r_next values. */ - GC_static_roots[j].r_start = r_start; - GC_static_roots[j].r_end = r_end; - GC_static_roots[j].r_tmp = r_tmp; -} - -/* Remove given range from every static root which intersects with */ -/* the range. It is assumed GC_remove_tmp_roots is called before */ -/* this function is called repeatedly by GC_register_map_entries. */ -GC_INNER void GC_remove_roots_subregion(ptr_t b, ptr_t e) { - int i; - GC_bool rebuild = FALSE; - - GC_ASSERT(I_HOLD_LOCK()); - GC_ASSERT((word)b % sizeof(word) == 0 && (word)e % sizeof(word) == 0); - for (i = 0; i < n_root_sets; i++) { - ptr_t r_start, r_end; - - if (GC_static_roots[i].r_tmp) { - /* The remaining roots are skipped as they are all temporary. */ -#ifdef GC_ASSERTIONS - int j; - for (j = i + 1; j < n_root_sets; j++) { - GC_ASSERT(GC_static_roots[j].r_tmp); - } -#endif - break; - } - r_start = GC_static_roots[i].r_start; - r_end = GC_static_roots[i].r_end; - if (!EXPECT((word)e <= (word)r_start || (word)r_end <= (word)b, TRUE)) { -#ifdef DEBUG_ADD_DEL_ROOTS - GC_log_printf("Removing %p .. %p from root section %d (%p .. %p)\n", - (void *)b, (void *)e, - i, (void *)r_start, (void *)r_end); -#endif - if ((word)r_start < (word)b) { - GC_root_size -= r_end - b; - GC_static_roots[i].r_end = b; - /* No need to rebuild as hash does not use r_end value. */ - if ((word)e < (word)r_end) { - int j; - - if (rebuild) { - GC_rebuild_root_index(); - rebuild = FALSE; - } - GC_add_roots_inner(e, r_end, FALSE); /* updates n_root_sets */ - for (j = i + 1; j < n_root_sets; j++) - if (GC_static_roots[j].r_tmp) - break; - if (j < n_root_sets - 1 && !GC_static_roots[n_root_sets - 1].r_tmp) { - /* Exchange the roots to have all temporary ones at the end. */ - swap_static_roots(j, n_root_sets - 1); - rebuild = TRUE; - } - } - } else { - if ((word)e < (word)r_end) { - GC_root_size -= e - r_start; - GC_static_roots[i].r_start = e; - } else { - GC_remove_root_at_pos(i); - if (i < n_root_sets - 1 && GC_static_roots[i].r_tmp && !GC_static_roots[i + 1].r_tmp) { - int j; - - for (j = i + 2; j < n_root_sets; j++) - if (GC_static_roots[j].r_tmp) - break; - /* Exchange the roots to have all temporary ones at the end. */ - swap_static_roots(i, j - 1); - } - i--; - } - rebuild = TRUE; - } - } - } - if (rebuild) - GC_rebuild_root_index(); -} -#endif /* USE_PROC_FOR_LIBRARIES */ - -#if !defined(NO_DEBUGGING) -/* For the debugging purpose only. */ -/* Workaround for the OS mapping and unmapping behind our back: */ -/* Is the address p in one of the temporary static root sections? */ -GC_API int GC_CALL GC_is_tmp_root(void *p) { - static int last_root_set = MAX_ROOT_SETS; - int i; - - if (last_root_set < n_root_sets && (word)p >= (word)GC_static_roots[last_root_set].r_start && (word)p < (word)GC_static_roots[last_root_set].r_end) - return GC_static_roots[last_root_set].r_tmp; - for (i = 0; i < n_root_sets; i++) { - if ((word)p >= (word)GC_static_roots[i].r_start && (word)p < (word)GC_static_roots[i].r_end) { - last_root_set = i; - return GC_static_roots[i].r_tmp; - } - } - return FALSE; -} -#endif /* !NO_DEBUGGING */ - -GC_INNER ptr_t GC_approx_sp(void) { - volatile word sp; -#if ((defined(E2K) && defined(__clang__)) || (defined(S390) && (__clang_major__ < 8))) && !defined(CPPCHECK) - /* Workaround some bugs in clang: */ - /* "undefined reference to llvm.frameaddress" error (clang-9/e2k); */ - /* a crash in SystemZTargetLowering of libLLVM-3.8 (S390). */ - sp = (word)&sp; -#elif defined(CPPCHECK) || (__GNUC__ >= 4 /* GC_GNUC_PREREQ(4, 0) */ \ - && !defined(STACK_NOT_SCANNED)) - /* TODO: Use GC_GNUC_PREREQ after fixing a bug in cppcheck. */ - sp = (word)__builtin_frame_address(0); -#else - sp = (word)&sp; -#endif - /* Also force stack to grow if necessary. Otherwise the */ - /* later accesses might cause the kernel to think we're */ - /* doing something wrong. */ - return (ptr_t)sp; -} - -/* - * Data structure for excluded static roots. - * Real declaration is in gc_priv.h. - -struct exclusion { - ptr_t e_start; - ptr_t e_end; -}; - -struct exclusion GC_excl_table[MAX_EXCLUSIONS]; - -- Array of exclusions, ascending - -- address order. -*/ - -GC_API void GC_CALL GC_clear_exclusion_table(void) { - GC_excl_table_entries = 0; -} - -/* Return the first exclusion range that includes an address >= start_addr */ -/* Assumes the exclusion table contains at least one entry (namely the */ -/* GC data structures). */ -STATIC struct exclusion *GC_next_exclusion(ptr_t start_addr) { - size_t low = 0; - size_t high; - - GC_ASSERT(GC_excl_table_entries > 0); - high = GC_excl_table_entries - 1; - while (high > low) { - size_t mid = (low + high) >> 1; - - /* low <= mid < high */ - if ((word)GC_excl_table[mid].e_end <= (word)start_addr) { - low = mid + 1; - } else { - high = mid; - } - } - if ((word)GC_excl_table[low].e_end <= (word)start_addr) return 0; - return GC_excl_table + low; -} - -/* The range boundaries should be properly aligned and valid. */ -GC_INNER void GC_exclude_static_roots_inner(void *start, void *finish) { - struct exclusion *next; - size_t next_index; - - GC_ASSERT(I_HOLD_LOCK()); - GC_ASSERT((word)start % sizeof(word) == 0); - GC_ASSERT((word)start < (word)finish); - - if (0 == GC_excl_table_entries) { - next = 0; - } else { - next = GC_next_exclusion((ptr_t)start); - } - if (0 != next) { - size_t i; - - if ((word)(next->e_start) < (word)finish) { - /* incomplete error check. */ - ABORT("Exclusion ranges overlap"); - } - if ((word)(next->e_start) == (word)finish) { - /* extend old range backwards */ - next->e_start = (ptr_t)start; - return; - } - next_index = next - GC_excl_table; - for (i = GC_excl_table_entries; i > next_index; --i) { - GC_excl_table[i] = GC_excl_table[i - 1]; - } - } else { - next_index = GC_excl_table_entries; - } - if (GC_excl_table_entries == MAX_EXCLUSIONS) ABORT("Too many exclusions"); - GC_excl_table[next_index].e_start = (ptr_t)start; - GC_excl_table[next_index].e_end = (ptr_t)finish; - ++GC_excl_table_entries; -} - -GC_API void GC_CALL GC_exclude_static_roots(void *b, void *e) { - if (b == e) return; /* nothing to exclude? */ - - /* Round boundaries (in direction reverse to that of GC_add_roots). */ - b = (void *)((word)b & ~(word)(sizeof(word) - 1)); - e = (void *)(((word)e + (sizeof(word) - 1)) & ~(word)(sizeof(word) - 1)); - if (NULL == e) - e = (void *)(~(word)(sizeof(word) - 1)); /* handle overflow */ - - LOCK(); - GC_exclude_static_roots_inner(b, e); - UNLOCK(); -} - -#if defined(WRAP_MARK_SOME) && defined(PARALLEL_MARK) -#define GC_PUSH_CONDITIONAL(b, t, all) \ - (GC_parallel \ - ? GC_push_conditional_eager(b, t, all) \ - : GC_push_conditional_static(b, t, all)) -#else -#define GC_PUSH_CONDITIONAL(b, t, all) GC_push_conditional_static(b, t, all) -#endif - -/* Invoke push_conditional on ranges that are not excluded. */ -STATIC void GC_push_conditional_with_exclusions(ptr_t bottom, ptr_t top, - GC_bool all) { - while ((word)bottom < (word)top) { - struct exclusion *next = GC_next_exclusion(bottom); - ptr_t excl_start; - - if (0 == next || (word)(excl_start = next->e_start) >= (word)top) { - GC_PUSH_CONDITIONAL(bottom, top, all); - break; - } - if ((word)excl_start > (word)bottom) - GC_PUSH_CONDITIONAL(bottom, excl_start, all); - bottom = next->e_end; - } -} - -#if defined(E2K) || defined(IA64) -/* Similar to GC_push_all_stack_sections() but for IA-64 registers store. */ -GC_INNER void GC_push_all_register_sections(ptr_t bs_lo, ptr_t bs_hi, - int eager, struct GC_traced_stack_sect_s *traced_stack_sect) { -#ifdef E2K - (void)traced_stack_sect; /* TODO: Not implemented yet */ -#else - while (traced_stack_sect != NULL) { - ptr_t frame_bs_lo = traced_stack_sect->backing_store_end; - GC_ASSERT((word)frame_bs_lo <= (word)bs_hi); - if (eager) { - GC_push_all_eager(frame_bs_lo, bs_hi); - } else { - GC_push_all_stack(frame_bs_lo, bs_hi); - } - bs_hi = traced_stack_sect->saved_backing_store_ptr; - traced_stack_sect = traced_stack_sect->prev; - } -#endif - GC_ASSERT((word)bs_lo <= (word)bs_hi); - if (eager) { - GC_push_all_eager(bs_lo, bs_hi); - } else { - GC_push_all_stack(bs_lo, bs_hi); - } -} -#endif /* E2K || IA64 */ - -#ifdef THREADS - -GC_INNER void GC_push_all_stack_sections(ptr_t lo, ptr_t hi, - struct GC_traced_stack_sect_s *traced_stack_sect) { - while (traced_stack_sect != NULL) { - GC_ASSERT((word)lo HOTTER_THAN(word) traced_stack_sect); -#ifdef STACK_GROWS_UP - GC_push_all_stack((ptr_t)traced_stack_sect, lo); -#else /* STACK_GROWS_DOWN */ - GC_push_all_stack(lo, (ptr_t)traced_stack_sect); -#endif - lo = traced_stack_sect->saved_stack_ptr; - GC_ASSERT(lo != NULL); - traced_stack_sect = traced_stack_sect->prev; - } - GC_ASSERT(!((word)hi HOTTER_THAN(word) lo)); -#ifdef STACK_GROWS_UP - /* We got them backwards! */ - GC_push_all_stack(hi, lo); -#else /* STACK_GROWS_DOWN */ - GC_push_all_stack(lo, hi); -#endif -} - -#else /* !THREADS */ - -/* Similar to GC_push_all_eager, but only the */ -/* part hotter than cold_gc_frame is scanned */ -/* immediately. Needed to ensure that callee- */ -/* save registers are not missed. */ -/* - * A version of GC_push_all that treats all interior pointers as valid - * and scans part of the area immediately, to make sure that saved - * register values are not lost. - * Cold_gc_frame delimits the stack section that must be scanned - * eagerly. A zero value indicates that no eager scanning is needed. - * We don't need to worry about the manual VDB case here, since this - * is only called in the single-threaded case. We assume that we - * cannot collect between an assignment and the corresponding - * GC_dirty() call. - */ -STATIC void GC_push_all_stack_partially_eager(ptr_t bottom, ptr_t top, - ptr_t cold_gc_frame) { -#ifndef NEED_FIXUP_POINTER - if (GC_all_interior_pointers) { - /* Push the hot end of the stack eagerly, so that register values */ - /* saved inside GC frames are marked before they disappear. */ - /* The rest of the marking can be deferred until later. */ - if (0 == cold_gc_frame) { - GC_push_all_stack(bottom, top); - return; - } - GC_ASSERT((word)bottom <= (word)cold_gc_frame && (word)cold_gc_frame <= (word)top); -#ifdef STACK_GROWS_DOWN - GC_push_all(cold_gc_frame - sizeof(ptr_t), top); - GC_push_all_eager(bottom, cold_gc_frame); -#else /* STACK_GROWS_UP */ - GC_push_all(bottom, cold_gc_frame + sizeof(ptr_t)); - GC_push_all_eager(cold_gc_frame, top); -#endif /* STACK_GROWS_UP */ - } else -#endif - /* else */ { - GC_push_all_eager(bottom, top); - } -#ifdef TRACE_BUF - GC_add_trace_entry("GC_push_all_stack", (word)bottom, (word)top); -#endif -} - -/* Similar to GC_push_all_stack_sections() but also uses cold_gc_frame. */ -STATIC void GC_push_all_stack_part_eager_sections(ptr_t lo, ptr_t hi, - ptr_t cold_gc_frame, struct GC_traced_stack_sect_s *traced_stack_sect) { - GC_ASSERT(traced_stack_sect == NULL || cold_gc_frame == NULL || - (word)cold_gc_frame HOTTER_THAN(word) traced_stack_sect); - - while (traced_stack_sect != NULL) { - GC_ASSERT((word)lo HOTTER_THAN(word) traced_stack_sect); -#ifdef STACK_GROWS_UP - GC_push_all_stack_partially_eager((ptr_t)traced_stack_sect, lo, - cold_gc_frame); -#else /* STACK_GROWS_DOWN */ - GC_push_all_stack_partially_eager(lo, (ptr_t)traced_stack_sect, - cold_gc_frame); -#endif - lo = traced_stack_sect->saved_stack_ptr; - GC_ASSERT(lo != NULL); - traced_stack_sect = traced_stack_sect->prev; - cold_gc_frame = NULL; /* Use at most once. */ - } - - GC_ASSERT(!((word)hi HOTTER_THAN(word) lo)); -#ifdef STACK_GROWS_UP - /* We got them backwards! */ - GC_push_all_stack_partially_eager(hi, lo, cold_gc_frame); -#else /* STACK_GROWS_DOWN */ - GC_push_all_stack_partially_eager(lo, hi, cold_gc_frame); -#endif -} - -#endif /* !THREADS */ - -/* Push enough of the current stack eagerly to ensure that callee-save */ -/* registers saved in GC frames are scanned. In the non-threads case, */ -/* schedule entire stack for scanning. The 2nd argument is a pointer */ -/* to the (possibly null) thread context, for (currently hypothetical) */ -/* more precise stack scanning. In the presence of threads, push */ -/* enough of the current stack to ensure that callee-save registers */ -/* saved in collector frames have been seen. */ -/* TODO: Merge it with per-thread stuff. */ -STATIC void GC_push_current_stack(ptr_t cold_gc_frame, void *context) { - UNUSED_ARG(context); -#if defined(THREADS) - /* cold_gc_frame is non-NULL. */ -#ifdef STACK_GROWS_DOWN - GC_push_all_eager(GC_approx_sp(), cold_gc_frame); - /* For IA64, the register stack backing store is handled */ - /* in the thread-specific code. */ -#else - GC_push_all_eager(cold_gc_frame, GC_approx_sp()); -#endif -#else - GC_push_all_stack_part_eager_sections(GC_approx_sp(), GC_stackbottom, - cold_gc_frame, GC_traced_stack_sect); -#ifdef IA64 - /* We also need to push the register stack backing store. */ - /* This should really be done in the same way as the */ - /* regular stack. For now we fudge it a bit. */ - /* Note that the backing store grows up, so we can't use */ - /* GC_push_all_stack_partially_eager. */ - { - ptr_t bsp = GC_save_regs_ret_val; - ptr_t cold_gc_bs_pointer = bsp - 2048; - if (GC_all_interior_pointers && (word)cold_gc_bs_pointer > (word)GC_register_stackbottom) { - /* Adjust cold_gc_bs_pointer if below our innermost */ - /* "traced stack section" in backing store. */ - if (GC_traced_stack_sect != NULL && (word)cold_gc_bs_pointer < (word)GC_traced_stack_sect->backing_store_end) - cold_gc_bs_pointer = - GC_traced_stack_sect->backing_store_end; - GC_push_all_register_sections(GC_register_stackbottom, - cold_gc_bs_pointer, FALSE, GC_traced_stack_sect); - GC_push_all_eager(cold_gc_bs_pointer, bsp); - } else { - GC_push_all_register_sections(GC_register_stackbottom, bsp, - TRUE /* eager */, GC_traced_stack_sect); - } - /* All values should be sufficiently aligned that we */ - /* don't have to worry about the boundary. */ - } -#elif defined(E2K) - /* We also need to push procedure stack store. */ - /* Procedure stack grows up. */ - { - ptr_t bs_lo; - size_t stack_size; - - GET_PROCEDURE_STACK_LOCAL(&bs_lo, &stack_size); - GC_push_all_register_sections(bs_lo, bs_lo + stack_size, - TRUE /* eager */, - GC_traced_stack_sect); - FREE_PROCEDURE_STACK_LOCAL(bs_lo, stack_size); - } -#endif -#endif /* !THREADS */ -} - -GC_INNER void (*GC_push_typed_structures)(void) = 0; - -GC_INNER void GC_cond_register_dynamic_libraries(void) { - GC_ASSERT(I_HOLD_LOCK()); -#if (defined(DYNAMIC_LOADING) && !defined(MSWIN_XBOX1)) || defined(CYGWIN32) || defined(MSWIN32) || defined(MSWINCE) || defined(PCR) - GC_remove_tmp_roots(); - if (!GC_no_dls) GC_register_dynamic_libraries(); -#else - GC_no_dls = TRUE; -#endif -} - -STATIC void GC_push_regs_and_stack(ptr_t cold_gc_frame) { -#ifdef THREADS - if (NULL == cold_gc_frame) - return; /* GC_push_all_stacks should push registers and stack */ -#endif - GC_with_callee_saves_pushed(GC_push_current_stack, cold_gc_frame); -} - -/* Call the mark routines (GC_push_one for a single pointer, */ -/* GC_push_conditional on groups of pointers) on every top level */ -/* accessible pointer. If all is false, arrange to push only possibly */ -/* altered values. Cold_gc_frame is an address inside a GC frame that */ -/* remains valid until all marking is complete; a NULL value indicates */ -/* that it is OK to miss some register values. */ -GC_INNER void GC_push_roots(GC_bool all, ptr_t cold_gc_frame) { - int i; - unsigned kind; - - GC_ASSERT(I_HOLD_LOCK()); - GC_ASSERT(GC_is_initialized); /* needed for GC_push_all_stacks */ - - /* Next push static data. This must happen early on, since it is */ - /* not robust against mark stack overflow. */ - /* Re-register dynamic libraries, in case one got added. */ - /* There is some argument for doing this as late as possible, */ - /* especially on Win32, where it can change asynchronously. */ - /* In those cases, we do it here. But on other platforms, it's */ - /* not safe with the world stopped, so we do it earlier. */ -#if !defined(REGISTER_LIBRARIES_EARLY) - GC_cond_register_dynamic_libraries(); -#endif - - /* Mark everything in static data areas. */ - for (i = 0; i < n_root_sets; i++) { - GC_push_conditional_with_exclusions( - GC_static_roots[i].r_start, - GC_static_roots[i].r_end, all); - } - - /* Mark all free list header blocks, if those were allocated from */ - /* the garbage collected heap. This makes sure they don't */ - /* disappear if we are not marking from static data. It also */ - /* saves us the trouble of scanning them, and possibly that of */ - /* marking the freelists. */ - for (kind = 0; kind < GC_n_kinds; kind++) { - void *base = GC_base(GC_obj_kinds[kind].ok_freelist); - if (base != NULL) { - GC_set_mark_bit(base); - } - } - - /* Mark from GC internal roots if those might otherwise have */ - /* been excluded. */ -#ifndef GC_NO_FINALIZATION - GC_push_finalizer_structures(); -#endif -#ifdef THREADS - if (GC_no_dls || GC_roots_were_cleared) - GC_push_thread_structures(); -#endif - if (GC_push_typed_structures) - GC_push_typed_structures(); - - /* Mark thread local free lists, even if their mark */ - /* descriptor excludes the link field. */ - /* If the world is not stopped, this is unsafe. It is */ - /* also unnecessary, since we will do this again with the */ - /* world stopped. */ -#if defined(THREAD_LOCAL_ALLOC) - if (GC_world_stopped) - GC_mark_thread_local_free_lists(); -#endif - - /* Now traverse stacks, and mark from register contents. */ - /* These must be done last, since they can legitimately */ - /* overflow the mark stack. This is usually done by saving */ - /* the current context on the stack, and then just tracing */ - /* from the stack. */ -#ifdef STACK_NOT_SCANNED - UNUSED_ARG(cold_gc_frame); -#else - GC_push_regs_and_stack(cold_gc_frame); -#endif - - if (GC_push_other_roots != 0) { - /* In the threads case, this also pushes thread stacks. */ - /* Note that without interior pointer recognition lots */ - /* of stuff may have been pushed already, and this */ - /* should be careful about mark stack overflows. */ - (*GC_push_other_roots)(); - } -} diff --git a/vendor/bdwgc/misc.c b/vendor/bdwgc/misc.c deleted file mode 100644 index dbb1bf91..00000000 --- a/vendor/bdwgc/misc.c +++ /dev/null @@ -1,2533 +0,0 @@ -/* - * Copyright 1988, 1989 Hans-J. Boehm, Alan J. Demers - * Copyright (c) 1991-1994 by Xerox Corporation. All rights reserved. - * Copyright (c) 1999-2001 by Hewlett-Packard Company. All rights reserved. - * Copyright (c) 2008-2022 Ivan Maidanski - * - * THIS MATERIAL IS PROVIDED AS IS, WITH ABSOLUTELY NO WARRANTY EXPRESSED - * OR IMPLIED. ANY USE IS AT YOUR OWN RISK. - * - * Permission is hereby granted to use or copy this program - * for any purpose, provided the above notices are retained on all copies. - * Permission to modify the code and to distribute modified code is granted, - * provided the above notices are retained, and a notice that the code was - * modified is included with the above copyright notice. - */ - -#include -#include - -#ifdef _WIN32 -#define WIN32_LEAN_AND_MEAN -#include -#endif -#include "private/gc_pmark.h" - -#ifndef MSWINCE -#include -#endif - -#ifdef GC_SOLARIS_THREADS -#include -#endif - -#if defined(UNIX_LIKE) || defined(CYGWIN32) || defined(SYMBIAN) || (defined(CONSOLE_LOG) && defined(MSWIN32)) -#include -#include -#include -#endif - -#if defined(CONSOLE_LOG) && defined(MSWIN32) && !defined(__GNUC__) -#include -#endif - -#ifdef NONSTOP -#include -#endif - -#ifdef THREADS -#ifdef PCR -#include "il/PCR_IL.h" -GC_INNER PCR_Th_ML GC_allocate_ml; -#elif defined(SN_TARGET_PSP2) -GC_INNER WapiMutex GC_allocate_ml_PSP2 = {0, NULL}; -#elif defined(GC_DEFN_ALLOCATE_ML) || defined(SN_TARGET_PS3) -#include -GC_INNER pthread_mutex_t GC_allocate_ml; -#endif -/* For other platforms with threads, the lock and possibly */ -/* GC_lock_holder variables are defined in the thread support code. */ -#endif /* THREADS */ - -#ifdef DYNAMIC_LOADING -/* We need to register the main data segment. Returns TRUE unless */ -/* this is done implicitly as part of dynamic library registration. */ -#define GC_REGISTER_MAIN_STATIC_DATA() GC_register_main_static_data() -#elif defined(GC_DONT_REGISTER_MAIN_STATIC_DATA) -#define GC_REGISTER_MAIN_STATIC_DATA() FALSE -#else -/* Don't unnecessarily call GC_register_main_static_data() in case */ -/* dyn_load.c isn't linked in. */ -#define GC_REGISTER_MAIN_STATIC_DATA() TRUE -#endif - -#ifdef NEED_CANCEL_DISABLE_COUNT -__thread unsigned char GC_cancel_disable_count = 0; -#endif - -GC_FAR struct _GC_arrays GC_arrays /* = { 0 } */; - -GC_INNER unsigned GC_n_mark_procs = GC_RESERVED_MARK_PROCS; - -GC_INNER unsigned GC_n_kinds = GC_N_KINDS_INITIAL_VALUE; - -GC_INNER GC_bool GC_debugging_started = FALSE; -/* defined here so we don't have to load dbg_mlc.o */ - -ptr_t GC_stackbottom = 0; - -#ifdef IA64 -GC_INNER ptr_t GC_register_stackbottom = NULL; -#endif - -int GC_dont_gc = FALSE; - -int GC_dont_precollect = FALSE; - -GC_bool GC_quiet = 0; /* used also in pcr_interface.c */ - -#if !defined(NO_CLOCK) || !defined(SMALL_CONFIG) -GC_INNER int GC_print_stats = 0; -#endif - -#ifdef GC_PRINT_BACK_HEIGHT -GC_INNER GC_bool GC_print_back_height = TRUE; -#else -GC_INNER GC_bool GC_print_back_height = FALSE; -#endif - -#ifndef NO_DEBUGGING -#ifdef GC_DUMP_REGULARLY -GC_INNER GC_bool GC_dump_regularly = TRUE; -/* Generate regular debugging dumps. */ -#else -GC_INNER GC_bool GC_dump_regularly = FALSE; -#endif -#ifndef NO_CLOCK -STATIC CLOCK_TYPE GC_init_time; -/* The time that the GC was initialized at. */ -#endif -#endif /* !NO_DEBUGGING */ - -#ifdef KEEP_BACK_PTRS -GC_INNER long GC_backtraces = 0; -/* Number of random backtraces to generate for each GC. */ -#endif - -#ifdef FIND_LEAK -int GC_find_leak = 1; -#else -int GC_find_leak = 0; -#endif - -#ifndef SHORT_DBG_HDRS -#ifdef GC_FINDLEAK_DELAY_FREE -GC_INNER GC_bool GC_findleak_delay_free = TRUE; -#else -GC_INNER GC_bool GC_findleak_delay_free = FALSE; -#endif -#endif /* !SHORT_DBG_HDRS */ - -#ifdef ALL_INTERIOR_POINTERS -int GC_all_interior_pointers = 1; -#else -int GC_all_interior_pointers = 0; -#endif - -#ifdef FINALIZE_ON_DEMAND -int GC_finalize_on_demand = 1; -#else -int GC_finalize_on_demand = 0; -#endif - -#ifdef JAVA_FINALIZATION -int GC_java_finalization = 1; -#else -int GC_java_finalization = 0; -#endif - -/* All accesses to it should be synchronized to avoid data races. */ -GC_finalizer_notifier_proc GC_finalizer_notifier = - (GC_finalizer_notifier_proc)0; - -#ifdef GC_FORCE_UNMAP_ON_GCOLLECT -/* Has no effect unless USE_MUNMAP. */ -/* Has no effect on implicitly-initiated garbage collections. */ -GC_INNER GC_bool GC_force_unmap_on_gcollect = TRUE; -#else -GC_INNER GC_bool GC_force_unmap_on_gcollect = FALSE; -#endif - -#ifndef GC_LARGE_ALLOC_WARN_INTERVAL -#define GC_LARGE_ALLOC_WARN_INTERVAL 5 -#endif -GC_INNER long GC_large_alloc_warn_interval = GC_LARGE_ALLOC_WARN_INTERVAL; -/* Interval between unsuppressed warnings. */ - -STATIC void *GC_CALLBACK GC_default_oom_fn(size_t bytes_requested) { - UNUSED_ARG(bytes_requested); - return NULL; -} - -/* All accesses to it should be synchronized to avoid data races. */ -GC_oom_func GC_oom_fn = GC_default_oom_fn; - -#ifdef CAN_HANDLE_FORK -#ifdef HANDLE_FORK -GC_INNER int GC_handle_fork = 1; -/* The value is examined by GC_thr_init. */ -#else -GC_INNER int GC_handle_fork = FALSE; -#endif - -#elif !defined(HAVE_NO_FORK) -GC_API void GC_CALL GC_atfork_prepare(void) { -#ifdef THREADS - ABORT("fork() handling unsupported"); -#endif -} - -GC_API void GC_CALL GC_atfork_parent(void) { - /* empty */ -} - -GC_API void GC_CALL GC_atfork_child(void) { - /* empty */ -} -#endif /* !CAN_HANDLE_FORK && !HAVE_NO_FORK */ - -/* Overrides the default automatic handle-fork mode. Has effect only */ -/* if called before GC_INIT. */ -GC_API void GC_CALL GC_set_handle_fork(int value) { -#ifdef CAN_HANDLE_FORK - if (!GC_is_initialized) - GC_handle_fork = value >= -1 ? value : 1; - /* Map all negative values except for -1 to a positive one. */ -#elif defined(THREADS) || (defined(DARWIN) && defined(MPROTECT_VDB)) - if (!GC_is_initialized && value) { -#ifndef SMALL_CONFIG - GC_init(); /* to initialize GC_manual_vdb and GC_stderr */ -#ifndef THREADS - if (GC_manual_vdb) - return; -#endif -#endif - ABORT("fork() handling unsupported"); - } -#else - /* No at-fork handler is needed in the single-threaded mode. */ - UNUSED_ARG(value); -#endif -} - -/* Set things up so that GC_size_map[i] >= granules(i), */ -/* but not too much bigger */ -/* and so that size_map contains relatively few distinct entries */ -/* This was originally stolen from Russ Atkinson's Cedar */ -/* quantization algorithm (but we precompute it). */ -STATIC void GC_init_size_map(void) { - size_t i; - - /* Map size 0 to something bigger. */ - /* This avoids problems at lower levels. */ - GC_size_map[0] = 1; - for (i = 1; i <= GRANULES_TO_BYTES(TINY_FREELISTS - 1) - EXTRA_BYTES; i++) { - GC_size_map[i] = ALLOC_REQUEST_GRANS(i); -#ifndef _MSC_VER - GC_ASSERT(GC_size_map[i] < TINY_FREELISTS); - /* Seems to tickle bug in VC++ 2008 for x64 */ -#endif - } - /* We leave the rest of the array to be filled in on demand. */ -} - -/* - * The following is a gross hack to deal with a problem that can occur - * on machines that are sloppy about stack frame sizes, notably SPARC. - * Bogus pointers may be written to the stack and not cleared for - * a LONG time, because they always fall into holes in stack frames - * that are not written. We partially address this by clearing - * sections of the stack whenever we get control. - */ - -#ifndef SMALL_CLEAR_SIZE -#define SMALL_CLEAR_SIZE 256 /* Clear this much every time. */ -#endif - -#if defined(ALWAYS_SMALL_CLEAR_STACK) || defined(STACK_NOT_SCANNED) -GC_API void *GC_CALL GC_clear_stack(void *arg) { -#ifndef STACK_NOT_SCANNED - word volatile dummy[SMALL_CLEAR_SIZE]; - BZERO((/* no volatile */ void *)dummy, sizeof(dummy)); -#endif - return arg; -} -#else - -#ifdef THREADS -#define BIG_CLEAR_SIZE 2048 /* Clear this much now and then. */ -#else -STATIC word GC_stack_last_cleared = 0; /* GC_no when we last did this */ -STATIC ptr_t GC_min_sp = NULL; -/* Coolest stack pointer value from which */ -/* we've already cleared the stack. */ -STATIC ptr_t GC_high_water = NULL; -/* "hottest" stack pointer value we have seen */ -/* recently. Degrades over time. */ -STATIC word GC_bytes_allocd_at_reset = 0; -#define DEGRADE_RATE 50 -#endif - -#if defined(ASM_CLEAR_CODE) -void *GC_clear_stack_inner(void *, ptr_t); -#else -/* Clear the stack up to about limit. Return arg. This function */ -/* is not static because it could also be erroneously defined in .S */ -/* file, so this error would be caught by the linker. */ -void *GC_clear_stack_inner(void *arg, -#if defined(__APPLE_CC__) && !GC_CLANG_PREREQ(6, 0) - volatile /* to workaround some bug */ -#endif - ptr_t limit) { -#define CLEAR_SIZE 213 /* granularity */ - volatile word dummy[CLEAR_SIZE]; - - BZERO((/* no volatile */ void *)dummy, sizeof(dummy)); - if ((word)GC_approx_sp() COOLER_THAN(word) limit) { - (void)GC_clear_stack_inner(arg, limit); - } - /* Make sure the recursive call is not a tail call, and the bzero */ - /* call is not recognized as dead code. */ -#if defined(CPPCHECK) - GC_noop1(dummy[0]); -#else - GC_noop1(COVERT_DATAFLOW(dummy)); -#endif - return arg; -} -#endif /* !ASM_CLEAR_CODE */ - -#ifdef THREADS -/* Used to occasionally clear a bigger chunk. */ -/* TODO: Should be more random than it is ... */ -static unsigned next_random_no(void) { -#ifdef AO_HAVE_fetch_and_add1 - static volatile AO_t random_no; - - return (unsigned)AO_fetch_and_add1(&random_no) % 13; -#else - static unsigned random_no = 0; - - return (random_no++) % 13; -#endif -} -#endif /* THREADS */ - -/* Clear some of the inaccessible part of the stack. Returns its */ -/* argument, so it can be used in a tail call position, hence clearing */ -/* another frame. */ -GC_API void *GC_CALL GC_clear_stack(void *arg) { - ptr_t sp = GC_approx_sp(); /* Hotter than actual sp */ -#ifdef THREADS - word volatile dummy[SMALL_CLEAR_SIZE]; -#endif - -#define SLOP 400 - /* Extra bytes we clear every time. This clears our own */ - /* activation record, and should cause more frequent */ - /* clearing near the cold end of the stack, a good thing. */ -#define GC_SLOP 4000 - /* We make GC_high_water this much hotter than we really saw */ - /* it, to cover for GC noise etc. above our current frame. */ -#define CLEAR_THRESHOLD 100000 - /* We restart the clearing process after this many bytes of */ - /* allocation. Otherwise very heavily recursive programs */ - /* with sparse stacks may result in heaps that grow almost */ - /* without bounds. As the heap gets larger, collection */ - /* frequency decreases, thus clearing frequency would decrease, */ - /* thus more junk remains accessible, thus the heap gets */ - /* larger ... */ -#ifdef THREADS - if (next_random_no() == 0) { - ptr_t limit = sp; - - MAKE_HOTTER(limit, BIG_CLEAR_SIZE * sizeof(word)); - limit = (ptr_t)((word)limit & ~0xf); - /* Make it sufficiently aligned for assembly */ - /* implementations of GC_clear_stack_inner. */ - return GC_clear_stack_inner(arg, limit); - } - BZERO((void *)dummy, SMALL_CLEAR_SIZE * sizeof(word)); -#else - if (GC_gc_no > GC_stack_last_cleared) { - /* Start things over, so we clear the entire stack again */ - if (GC_stack_last_cleared == 0) - GC_high_water = (ptr_t)GC_stackbottom; - GC_min_sp = GC_high_water; - GC_stack_last_cleared = GC_gc_no; - GC_bytes_allocd_at_reset = GC_bytes_allocd; - } - /* Adjust GC_high_water */ - MAKE_COOLER(GC_high_water, WORDS_TO_BYTES(DEGRADE_RATE) + GC_SLOP); - if ((word)sp HOTTER_THAN(word) GC_high_water) { - GC_high_water = sp; - } - MAKE_HOTTER(GC_high_water, GC_SLOP); - { - ptr_t limit = GC_min_sp; - - MAKE_HOTTER(limit, SLOP); - if ((word)sp COOLER_THAN(word) limit) { - limit = (ptr_t)((word)limit & ~0xf); - /* Make it sufficiently aligned for assembly */ - /* implementations of GC_clear_stack_inner. */ - GC_min_sp = sp; - return GC_clear_stack_inner(arg, limit); - } - } - if (GC_bytes_allocd - GC_bytes_allocd_at_reset > CLEAR_THRESHOLD) { - /* Restart clearing process, but limit how much clearing we do. */ - GC_min_sp = sp; - MAKE_HOTTER(GC_min_sp, CLEAR_THRESHOLD / 4); - if ((word)GC_min_sp HOTTER_THAN(word) GC_high_water) - GC_min_sp = GC_high_water; - GC_bytes_allocd_at_reset = GC_bytes_allocd; - } -#endif - return arg; -} - -#endif /* !ALWAYS_SMALL_CLEAR_STACK && !STACK_NOT_SCANNED */ - -/* Return a pointer to the base address of p, given a pointer to a */ -/* an address within an object. Return 0 o.w. */ -GC_API void *GC_CALL GC_base(void *p) { - ptr_t r; - struct hblk *h; - bottom_index *bi; - hdr *candidate_hdr; - - r = (ptr_t)p; - if (!EXPECT(GC_is_initialized, TRUE)) return NULL; - h = HBLKPTR(r); - GET_BI(r, bi); - candidate_hdr = HDR_FROM_BI(bi, r); - if (NULL == candidate_hdr) return NULL; - /* If it's a pointer to the middle of a large object, move it */ - /* to the beginning. */ - while (IS_FORWARDING_ADDR_OR_NIL(candidate_hdr)) { - h = FORWARDED_ADDR(h, candidate_hdr); - r = (ptr_t)h; - candidate_hdr = HDR(h); - } - if (HBLK_IS_FREE(candidate_hdr)) return NULL; - /* Make sure r points to the beginning of the object */ - r = (ptr_t)((word)r & ~(WORDS_TO_BYTES(1) - 1)); - { - word sz = candidate_hdr->hb_sz; - ptr_t limit; - - r -= HBLKDISPL(r) % sz; - limit = r + sz; - if (((word)limit > (word)(h + 1) && sz <= HBLKSIZE) || (word)p >= (word)limit) - return NULL; - } - return (void *)r; -} - -/* Return TRUE if and only if p points to somewhere in GC heap. */ -GC_API int GC_CALL GC_is_heap_ptr(const void *p) { - bottom_index *bi; - - GC_ASSERT(GC_is_initialized); - GET_BI(p, bi); - return HDR_FROM_BI(bi, p) != 0; -} - -/* Return the size of an object, given a pointer to its base. */ -/* (For small objects this also happens to work from interior pointers, */ -/* but that shouldn't be relied upon.) */ -GC_API size_t GC_CALL GC_size(const void *p) { - hdr *hhdr = HDR(p); - - return (size_t)hhdr->hb_sz; -} - -/* These getters remain unsynchronized for compatibility (since some */ -/* clients could call some of them from a GC callback holding the */ -/* allocator lock). */ -GC_API size_t GC_CALL GC_get_heap_size(void) { - /* ignore the memory space returned to OS (i.e. count only the */ - /* space owned by the garbage collector) */ - return (size_t)(GC_heapsize - GC_unmapped_bytes); -} - -GC_API size_t GC_CALL GC_get_obtained_from_os_bytes(void) { - return (size_t)GC_our_mem_bytes; -} - -GC_API size_t GC_CALL GC_get_free_bytes(void) { - /* ignore the memory space returned to OS */ - return (size_t)(GC_large_free_bytes - GC_unmapped_bytes); -} - -GC_API size_t GC_CALL GC_get_unmapped_bytes(void) { - return (size_t)GC_unmapped_bytes; -} - -GC_API size_t GC_CALL GC_get_bytes_since_gc(void) { - return (size_t)GC_bytes_allocd; -} - -GC_API size_t GC_CALL GC_get_total_bytes(void) { - return (size_t)(GC_bytes_allocd + GC_bytes_allocd_before_gc); -} - -#ifndef GC_GET_HEAP_USAGE_NOT_NEEDED - -GC_API size_t GC_CALL GC_get_size_map_at(int i) { - if ((unsigned)i > MAXOBJBYTES) - return GC_SIZE_MAX; - return GRANULES_TO_BYTES(GC_size_map[i]); -} - -/* Return the heap usage information. This is a thread-safe (atomic) */ -/* alternative for the five above getters. NULL pointer is allowed for */ -/* any argument. Returned (filled in) values are of word type. */ -GC_API void GC_CALL GC_get_heap_usage_safe(GC_word *pheap_size, - GC_word *pfree_bytes, GC_word *punmapped_bytes, - GC_word *pbytes_since_gc, GC_word *ptotal_bytes) { - LOCK(); - if (pheap_size != NULL) - *pheap_size = GC_heapsize - GC_unmapped_bytes; - if (pfree_bytes != NULL) - *pfree_bytes = GC_large_free_bytes - GC_unmapped_bytes; - if (punmapped_bytes != NULL) - *punmapped_bytes = GC_unmapped_bytes; - if (pbytes_since_gc != NULL) - *pbytes_since_gc = GC_bytes_allocd; - if (ptotal_bytes != NULL) - *ptotal_bytes = GC_bytes_allocd + GC_bytes_allocd_before_gc; - UNLOCK(); -} - -GC_INNER word GC_reclaimed_bytes_before_gc = 0; - -/* Fill in GC statistics provided the destination is of enough size. */ -static void fill_prof_stats(struct GC_prof_stats_s *pstats) { - pstats->heapsize_full = GC_heapsize; - pstats->free_bytes_full = GC_large_free_bytes; - pstats->unmapped_bytes = GC_unmapped_bytes; - pstats->bytes_allocd_since_gc = GC_bytes_allocd; - pstats->allocd_bytes_before_gc = GC_bytes_allocd_before_gc; - pstats->non_gc_bytes = GC_non_gc_bytes; - pstats->gc_no = GC_gc_no; /* could be -1 */ -#ifdef PARALLEL_MARK - pstats->markers_m1 = (word)((signed_word)GC_markers_m1); -#else - pstats->markers_m1 = 0; /* one marker */ -#endif - pstats->bytes_reclaimed_since_gc = GC_bytes_found > 0 ? (word)GC_bytes_found : 0; - pstats->reclaimed_bytes_before_gc = GC_reclaimed_bytes_before_gc; - pstats->expl_freed_bytes_since_gc = GC_bytes_freed; /* since gc-7.7 */ - pstats->obtained_from_os_bytes = GC_our_mem_bytes; /* since gc-8.2 */ -} - -#include /* for memset() */ - -GC_API size_t GC_CALL GC_get_prof_stats(struct GC_prof_stats_s *pstats, - size_t stats_sz) { - struct GC_prof_stats_s stats; - - LOCK(); - fill_prof_stats(stats_sz >= sizeof(stats) ? pstats : &stats); - UNLOCK(); - - if (stats_sz == sizeof(stats)) { - return sizeof(stats); - } else if (stats_sz > sizeof(stats)) { - /* Fill in the remaining part with -1. */ - memset((char *)pstats + sizeof(stats), 0xff, stats_sz - sizeof(stats)); - return sizeof(stats); - } else { - if (EXPECT(stats_sz > 0, TRUE)) - BCOPY(&stats, pstats, stats_sz); - return stats_sz; - } -} - -#ifdef THREADS -/* The _unsafe version assumes the caller holds the allocation lock. */ -GC_API size_t GC_CALL GC_get_prof_stats_unsafe( - struct GC_prof_stats_s *pstats, - size_t stats_sz) { - struct GC_prof_stats_s stats; - - if (stats_sz >= sizeof(stats)) { - fill_prof_stats(pstats); - if (stats_sz > sizeof(stats)) - memset((char *)pstats + sizeof(stats), 0xff, - stats_sz - sizeof(stats)); - return sizeof(stats); - } else { - if (EXPECT(stats_sz > 0, TRUE)) { - fill_prof_stats(&stats); - BCOPY(&stats, pstats, stats_sz); - } - return stats_sz; - } -} -#endif /* THREADS */ - -#endif /* !GC_GET_HEAP_USAGE_NOT_NEEDED */ - -#if defined(THREADS) && !defined(SIGNAL_BASED_STOP_WORLD) -/* GC does not use signals to suspend and restart threads. */ -GC_API void GC_CALL GC_set_suspend_signal(int sig) { - UNUSED_ARG(sig); -} - -GC_API void GC_CALL GC_set_thr_restart_signal(int sig) { - UNUSED_ARG(sig); -} - -GC_API int GC_CALL GC_get_suspend_signal(void) { - return -1; -} - -GC_API int GC_CALL GC_get_thr_restart_signal(void) { - return -1; -} -#endif /* THREADS && !SIGNAL_BASED_STOP_WORLD */ - -#if !defined(_MAX_PATH) && (defined(MSWIN32) || defined(MSWINCE) || defined(CYGWIN32)) -#define _MAX_PATH MAX_PATH -#endif - -#ifdef GC_READ_ENV_FILE -/* This works for Win32/WinCE for now. Really useful only for WinCE. */ -STATIC char *GC_envfile_content = NULL; -/* The content of the GC "env" file with CR and */ -/* LF replaced to '\0'. NULL if the file is */ -/* missing or empty. Otherwise, always ends */ -/* with '\0'. */ -STATIC unsigned GC_envfile_length = 0; -/* Length of GC_envfile_content (if non-NULL). */ - -#ifndef GC_ENVFILE_MAXLEN -#define GC_ENVFILE_MAXLEN 0x4000 -#endif - -#define GC_ENV_FILE_EXT ".gc.env" - -/* The routine initializes GC_envfile_content from the GC "env" file. */ -STATIC void GC_envfile_init(void) { -#if defined(MSWIN32) || defined(MSWINCE) || defined(CYGWIN32) - HANDLE hFile; - char *content; - unsigned ofs; - unsigned len; - DWORD nBytesRead; - TCHAR path[_MAX_PATH + 0x10]; /* buffer for path + ext */ - size_t bytes_to_get; - - GC_ASSERT(I_HOLD_LOCK()); - len = (unsigned)GetModuleFileName(NULL /* hModule */, path, - _MAX_PATH + 1); - /* If GetModuleFileName() has failed then len is 0. */ - if (len > 4 && path[len - 4] == (TCHAR)'.') { - len -= 4; /* strip executable file extension */ - } - BCOPY(TEXT(GC_ENV_FILE_EXT), &path[len], sizeof(TEXT(GC_ENV_FILE_EXT))); - hFile = CreateFile(path, GENERIC_READ, - FILE_SHARE_READ | FILE_SHARE_WRITE, - NULL /* lpSecurityAttributes */, OPEN_EXISTING, - FILE_ATTRIBUTE_NORMAL, NULL /* hTemplateFile */); - if (hFile == INVALID_HANDLE_VALUE) - return; /* the file is absent or the operation is failed */ - len = (unsigned)GetFileSize(hFile, NULL); - if (len <= 1 || len >= GC_ENVFILE_MAXLEN) { - CloseHandle(hFile); - return; /* invalid file length - ignoring the file content */ - } - /* At this execution point, GC_setpagesize() and GC_init_win32() */ - /* must already be called (for GET_MEM() to work correctly). */ - GC_ASSERT(GC_page_size != 0); - bytes_to_get = ROUNDUP_PAGESIZE_IF_MMAP((size_t)len + 1); - content = (char *)GET_MEM(bytes_to_get); - if (content == NULL) { - CloseHandle(hFile); - return; /* allocation failure */ - } - GC_add_to_our_memory(content, bytes_to_get); - ofs = 0; - nBytesRead = (DWORD)-1L; - /* Last ReadFile() call should clear nBytesRead on success. */ - while (ReadFile(hFile, content + ofs, len - ofs + 1, &nBytesRead, - NULL /* lpOverlapped */) && - nBytesRead != 0) { - if ((ofs += nBytesRead) > len) - break; - } - CloseHandle(hFile); - if (ofs != len || nBytesRead != 0) { - /* TODO: recycle content */ - return; /* read operation is failed - ignoring the file content */ - } - content[ofs] = '\0'; - while (ofs-- > 0) { - if (content[ofs] == '\r' || content[ofs] == '\n') - content[ofs] = '\0'; - } - GC_ASSERT(NULL == GC_envfile_content); - GC_envfile_length = len + 1; - GC_envfile_content = content; -#endif -} - -/* This routine scans GC_envfile_content for the specified */ -/* environment variable (and returns its value if found). */ -GC_INNER char *GC_envfile_getenv(const char *name) { - char *p; - char *end_of_content; - size_t namelen; - -#ifndef NO_GETENV - p = getenv(name); /* try the standard getenv() first */ - if (p != NULL) - return *p != '\0' ? p : NULL; -#endif - p = GC_envfile_content; - if (p == NULL) - return NULL; /* "env" file is absent (or empty) */ - namelen = strlen(name); - if (namelen == 0) /* a sanity check */ - return NULL; - for (end_of_content = p + GC_envfile_length; - p != end_of_content; p += strlen(p) + 1) { - if (strncmp(p, name, namelen) == 0 && *(p += namelen) == '=') { - p++; /* the match is found; skip '=' */ - return *p != '\0' ? p : NULL; - } - /* If not matching then skip to the next line. */ - } - return NULL; /* no match found */ -} -#endif /* GC_READ_ENV_FILE */ - -GC_INNER GC_bool GC_is_initialized = FALSE; - -GC_API int GC_CALL GC_is_init_called(void) { - return (int)GC_is_initialized; -} - -#if defined(GC_WIN32_THREADS) && ((defined(MSWIN32) && !defined(CONSOLE_LOG)) || defined(MSWINCE)) -GC_INNER CRITICAL_SECTION GC_write_cs; -#endif - -#ifndef DONT_USE_ATEXIT -#if !defined(PCR) && !defined(SMALL_CONFIG) -/* A dedicated variable to avoid a garbage collection on abort. */ -/* GC_find_leak cannot be used for this purpose as otherwise */ -/* TSan finds a data race (between GC_default_on_abort and, e.g., */ -/* GC_finish_collection). */ -static GC_bool skip_gc_atexit = FALSE; -#else -#define skip_gc_atexit FALSE -#endif - -STATIC void GC_exit_check(void) { - if (GC_find_leak && !skip_gc_atexit) { -#ifdef THREADS - /* GC_in_thread_creation should always be updated holding the */ - /* lock even if we are about to exit. */ - LOCK(); - GC_in_thread_creation = TRUE; /* OK to collect from unknown thread. */ - UNLOCK(); -#endif - GC_gcollect(); -#ifdef THREADS - LOCK(); - GC_in_thread_creation = FALSE; - UNLOCK(); -#endif - } -} -#endif - -#if defined(UNIX_LIKE) && !defined(NO_DEBUGGING) -static void looping_handler(int sig) { - GC_err_printf("Caught signal %d: looping in handler\n", sig); - for (;;) { - /* empty */ - } -} - -static GC_bool installed_looping_handler = FALSE; - -static void maybe_install_looping_handler(void) { - /* Install looping handler before the write fault handler, so we */ - /* handle write faults correctly. */ - if (!installed_looping_handler && 0 != GETENV("GC_LOOP_ON_ABORT")) { - GC_set_and_save_fault_handler(looping_handler); - installed_looping_handler = TRUE; - } -} - -#else /* !UNIX_LIKE */ -#define maybe_install_looping_handler() -#endif - -#define GC_DEFAULT_STDOUT_FD 1 -#define GC_DEFAULT_STDERR_FD 2 - -#if !defined(OS2) && !defined(MACOS) && !defined(GC_ANDROID_LOG) && !defined(NN_PLATFORM_CTR) && !defined(NINTENDO_SWITCH) && (!defined(MSWIN32) || defined(CONSOLE_LOG)) && !defined(MSWINCE) -STATIC int GC_stdout = GC_DEFAULT_STDOUT_FD; -STATIC int GC_stderr = GC_DEFAULT_STDERR_FD; -STATIC int GC_log = GC_DEFAULT_STDERR_FD; - -#ifndef MSWIN32 -GC_API void GC_CALL GC_set_log_fd(int fd) { - GC_log = fd; -} -#endif -#endif - -#ifdef MSGBOX_ON_ERROR -STATIC void GC_win32_MessageBoxA(const char *msg, const char *caption, - unsigned flags) { -#ifndef DONT_USE_USER32_DLL - /* Use static binding to "user32.dll". */ - (void)MessageBoxA(NULL, msg, caption, flags); -#else - /* This simplifies linking - resolve "MessageBoxA" at run-time. */ - HINSTANCE hU32 = LoadLibrary(TEXT("user32.dll")); - if (hU32) { - FARPROC pfn = GetProcAddress(hU32, "MessageBoxA"); - if (pfn) - (void)(*(int(WINAPI *)(HWND, LPCSTR, LPCSTR, UINT))(word)pfn)( - NULL /* hWnd */, msg, caption, flags); - (void)FreeLibrary(hU32); - } -#endif -} -#endif /* MSGBOX_ON_ERROR */ - -#if defined(THREADS) && defined(UNIX_LIKE) && !defined(NO_GETCONTEXT) -static void callee_saves_pushed_dummy_fn(ptr_t data, void *context) { - UNUSED_ARG(data); - UNUSED_ARG(context); -} -#endif - -#ifndef SMALL_CONFIG -#ifdef MANUAL_VDB -static GC_bool manual_vdb_allowed = TRUE; -#else -static GC_bool manual_vdb_allowed = FALSE; -#endif - -GC_API void GC_CALL GC_set_manual_vdb_allowed(int value) { - manual_vdb_allowed = (GC_bool)value; -} - -GC_API int GC_CALL GC_get_manual_vdb_allowed(void) { - return (int)manual_vdb_allowed; -} -#endif /* !SMALL_CONFIG */ - -STATIC word GC_parse_mem_size_arg(const char *str) { - word result; - char *endptr; - char ch; - - if ('\0' == *str) return GC_WORD_MAX; /* bad value */ - result = (word)STRTOULL(str, &endptr, 10); - ch = *endptr; - if (ch != '\0') { - if (*(endptr + 1) != '\0') return GC_WORD_MAX; - /* Allow k, M or G suffix. */ - switch (ch) { - case 'K': - case 'k': - result <<= 10; - break; - case 'M': - case 'm': - result <<= 20; - break; - case 'G': - case 'g': - result <<= 30; - break; - default: - result = GC_WORD_MAX; - } - } - return result; -} - -#define GC_LOG_STD_NAME "gc.log" - -GC_API void GC_CALL GC_init(void) { - /* LOCK(); -- no longer does anything this early. */ - word initial_heap_sz; - IF_CANCEL(int cancel_state;) - - if (EXPECT(GC_is_initialized, TRUE)) return; -#ifdef REDIRECT_MALLOC - { - static GC_bool init_started = FALSE; - if (init_started) - ABORT("Redirected malloc() called during GC init"); - init_started = TRUE; - } -#endif - -#if defined(GC_INITIAL_HEAP_SIZE) && !defined(CPPCHECK) - initial_heap_sz = GC_INITIAL_HEAP_SIZE; -#else - initial_heap_sz = MINHINCR * HBLKSIZE; -#endif - - DISABLE_CANCEL(cancel_state); - /* Note that although we are nominally called with the */ - /* allocation lock held, the allocation lock is now */ - /* only really acquired once a second thread is forked.*/ - /* And the initialization code needs to run before */ - /* then. Thus we really don't hold any locks, and can */ - /* in fact safely initialize them here. */ -#ifdef THREADS -#ifndef GC_ALWAYS_MULTITHREADED - GC_ASSERT(!GC_need_to_lock); -#endif -#ifdef SN_TARGET_PS3 - { - pthread_mutexattr_t mattr; - - if (0 != pthread_mutexattr_init(&mattr)) { - ABORT("pthread_mutexattr_init failed"); - } - if (0 != pthread_mutex_init(&GC_allocate_ml, &mattr)) { - ABORT("pthread_mutex_init failed"); - } - (void)pthread_mutexattr_destroy(&mattr); - } -#endif -#endif /* THREADS */ -#if defined(GC_WIN32_THREADS) && !defined(GC_PTHREADS) -#ifndef SPIN_COUNT -#define SPIN_COUNT 4000 -#endif -#ifdef MSWINRT_FLAVOR - InitializeCriticalSectionAndSpinCount(&GC_allocate_ml, SPIN_COUNT); -#else - { -#ifndef MSWINCE - FARPROC pfn = 0; - HMODULE hK32 = GetModuleHandle(TEXT("kernel32.dll")); - if (hK32) - pfn = GetProcAddress(hK32, - "InitializeCriticalSectionAndSpinCount"); - if (pfn) { - (*(BOOL(WINAPI *)(LPCRITICAL_SECTION, DWORD))(word)pfn)( - &GC_allocate_ml, SPIN_COUNT); - } else -#endif /* !MSWINCE */ - /* else */ InitializeCriticalSection(&GC_allocate_ml); - } -#endif -#endif /* GC_WIN32_THREADS && !GC_PTHREADS */ -#if defined(GC_WIN32_THREADS) && ((defined(MSWIN32) && !defined(CONSOLE_LOG)) || defined(MSWINCE)) - InitializeCriticalSection(&GC_write_cs); -#endif -#if defined(GC_ASSERTIONS) && defined(GC_ALWAYS_MULTITHREADED) - LOCK(); /* just to set GC_lock_holder */ -#endif - GC_setpagesize(); -#ifdef MSWIN32 - GC_init_win32(); -#endif -#ifdef GC_READ_ENV_FILE - GC_envfile_init(); -#endif -#if !defined(NO_CLOCK) || !defined(SMALL_CONFIG) -#ifdef GC_PRINT_VERBOSE_STATS - /* This is useful for debugging and profiling on platforms with */ - /* missing getenv() (like WinCE). */ - GC_print_stats = VERBOSE; -#else - if (0 != GETENV("GC_PRINT_VERBOSE_STATS")) { - GC_print_stats = VERBOSE; - } else if (0 != GETENV("GC_PRINT_STATS")) { - GC_print_stats = 1; - } -#endif -#endif -#if ((defined(UNIX_LIKE) && !defined(GC_ANDROID_LOG)) || (defined(CONSOLE_LOG) && defined(MSWIN32)) || defined(CYGWIN32) || defined(SYMBIAN)) && !defined(SMALL_CONFIG) - { - char *file_name = TRUSTED_STRING(GETENV("GC_LOG_FILE")); -#ifdef GC_LOG_TO_FILE_ALWAYS - if (NULL == file_name) - file_name = GC_LOG_STD_NAME; -#else - if (0 != file_name) -#endif - { -#if defined(_MSC_VER) - int log_d = _open(file_name, O_CREAT | O_WRONLY | O_APPEND); -#else - int log_d = open(file_name, O_CREAT | O_WRONLY | O_APPEND, 0644); -#endif - if (log_d < 0) { - GC_err_printf("Failed to open %s as log file\n", file_name); - } else { - char *str; - GC_log = log_d; - str = GETENV("GC_ONLY_LOG_TO_FILE"); -#ifdef GC_ONLY_LOG_TO_FILE - /* The similar environment variable set to "0" */ - /* overrides the effect of the macro defined. */ - if (str != NULL && *str == '0' && *(str + 1) == '\0') -#else - /* Otherwise setting the environment variable */ - /* to anything other than "0" will prevent from */ - /* redirecting stdout/err to the log file. */ - if (str == NULL || (*str == '0' && *(str + 1) == '\0')) -#endif - { - GC_stdout = log_d; - GC_stderr = log_d; - } - } - } - } -#endif -#if !defined(NO_DEBUGGING) && !defined(GC_DUMP_REGULARLY) - if (0 != GETENV("GC_DUMP_REGULARLY")) { - GC_dump_regularly = TRUE; - } -#endif -#ifdef KEEP_BACK_PTRS - { - char *backtraces_string = GETENV("GC_BACKTRACES"); - if (0 != backtraces_string) { - GC_backtraces = atol(backtraces_string); - if (backtraces_string[0] == '\0') GC_backtraces = 1; - } - } -#endif - if (0 != GETENV("GC_FIND_LEAK")) { - GC_find_leak = 1; - } -#ifndef SHORT_DBG_HDRS - if (0 != GETENV("GC_FINDLEAK_DELAY_FREE")) { - GC_findleak_delay_free = TRUE; - } -#endif - if (0 != GETENV("GC_ALL_INTERIOR_POINTERS")) { - GC_all_interior_pointers = 1; - } - if (0 != GETENV("GC_DONT_GC")) { -#if defined(LINT2) && !(defined(GC_ASSERTIONS) && defined(GC_ALWAYS_MULTITHREADED)) - GC_disable(); -#else - GC_dont_gc = 1; -#endif - } - if (0 != GETENV("GC_PRINT_BACK_HEIGHT")) { - GC_print_back_height = TRUE; - } - if (0 != GETENV("GC_NO_BLACKLIST_WARNING")) { - GC_large_alloc_warn_interval = LONG_MAX; - } - { - char *addr_string = GETENV("GC_TRACE"); - if (0 != addr_string) { -#ifndef ENABLE_TRACE - WARN("Tracing not enabled: Ignoring GC_TRACE value\n", 0); -#else - word addr = (word)STRTOULL(addr_string, NULL, 16); - if (addr < 0x1000) - WARN("Unlikely trace address: %p\n", (void *)addr); - GC_trace_addr = (ptr_t)addr; -#endif - } - } -#ifdef GC_COLLECT_AT_MALLOC - { - char *string = GETENV("GC_COLLECT_AT_MALLOC"); - if (0 != string) { - size_t min_lb = (size_t)STRTOULL(string, NULL, 10); - if (min_lb > 0) - GC_dbg_collect_at_malloc_min_lb = min_lb; - } - } -#endif -#if !defined(GC_DISABLE_INCREMENTAL) && !defined(NO_CLOCK) - { - char *time_limit_string = GETENV("GC_PAUSE_TIME_TARGET"); - if (0 != time_limit_string) { - long time_limit = atol(time_limit_string); - if (time_limit > 0) { - GC_time_limit = time_limit; - } - } - } -#endif -#ifndef SMALL_CONFIG - { - char *full_freq_string = GETENV("GC_FULL_FREQUENCY"); - if (full_freq_string != NULL) { - int full_freq = atoi(full_freq_string); - if (full_freq > 0) - GC_full_freq = full_freq; - } - } -#endif - { - char *interval_string = GETENV("GC_LARGE_ALLOC_WARN_INTERVAL"); - if (0 != interval_string) { - long interval = atol(interval_string); - if (interval <= 0) { - WARN( - "GC_LARGE_ALLOC_WARN_INTERVAL environment variable has" - " bad value - ignoring\n", - 0); - } else { - GC_large_alloc_warn_interval = interval; - } - } - } - { - char *space_divisor_string = GETENV("GC_FREE_SPACE_DIVISOR"); - if (space_divisor_string != NULL) { - int space_divisor = atoi(space_divisor_string); - if (space_divisor > 0) - GC_free_space_divisor = (unsigned)space_divisor; - } - } -#ifdef USE_MUNMAP - { - char *string = GETENV("GC_UNMAP_THRESHOLD"); - if (string != NULL) { - if (*string == '0' && *(string + 1) == '\0') { - /* "0" is used to disable unmapping. */ - GC_unmap_threshold = 0; - } else { - int unmap_threshold = atoi(string); - if (unmap_threshold > 0) - GC_unmap_threshold = unmap_threshold; - } - } - } - { - char *string = GETENV("GC_FORCE_UNMAP_ON_GCOLLECT"); - if (string != NULL) { - if (*string == '0' && *(string + 1) == '\0') { - /* "0" is used to turn off the mode. */ - GC_force_unmap_on_gcollect = FALSE; - } else { - GC_force_unmap_on_gcollect = TRUE; - } - } - } - { - char *string = GETENV("GC_USE_ENTIRE_HEAP"); - if (string != NULL) { - if (*string == '0' && *(string + 1) == '\0') { - /* "0" is used to turn off the mode. */ - GC_use_entire_heap = FALSE; - } else { - GC_use_entire_heap = TRUE; - } - } - } -#endif -#if !defined(NO_DEBUGGING) && !defined(NO_CLOCK) - GET_TIME(GC_init_time); -#endif - maybe_install_looping_handler(); -#if ALIGNMENT > GC_DS_TAGS - /* Adjust normal object descriptor for extra allocation. */ - if (EXTRA_BYTES != 0) - GC_obj_kinds[NORMAL].ok_descriptor = - ((~(word)ALIGNMENT) + 1) | GC_DS_LENGTH; -#endif - GC_exclude_static_roots_inner(beginGC_arrays, endGC_arrays); - GC_exclude_static_roots_inner(beginGC_obj_kinds, endGC_obj_kinds); -#ifdef SEPARATE_GLOBALS - GC_exclude_static_roots_inner(beginGC_objfreelist, endGC_objfreelist); - GC_exclude_static_roots_inner(beginGC_aobjfreelist, endGC_aobjfreelist); -#endif -#if defined(USE_PROC_FOR_LIBRARIES) && defined(GC_LINUX_THREADS) - WARN("USE_PROC_FOR_LIBRARIES + GC_LINUX_THREADS performs poorly\n", 0); - /* If thread stacks are cached, they tend to be scanned in */ - /* entirety as part of the root set. This will grow them to */ - /* maximum size, and is generally not desirable. */ -#endif -#if !defined(THREADS) || defined(GC_PTHREADS) || defined(NN_PLATFORM_CTR) || defined(NINTENDO_SWITCH) || defined(GC_WIN32_THREADS) || defined(GC_SOLARIS_THREADS) - if (GC_stackbottom == 0) { - GC_stackbottom = GC_get_main_stack_base(); -#if (defined(LINUX) || defined(HPUX)) && defined(IA64) - GC_register_stackbottom = GC_get_register_stack_base(); -#endif - } else { -#if (defined(LINUX) || defined(HPUX)) && defined(IA64) - if (GC_register_stackbottom == 0) { - WARN("GC_register_stackbottom should be set with GC_stackbottom\n", - 0); - /* The following may fail, since we may rely on */ - /* alignment properties that may not hold with a user set */ - /* GC_stackbottom. */ - GC_register_stackbottom = GC_get_register_stack_base(); - } -#endif - } -#endif -#if !defined(CPPCHECK) - GC_STATIC_ASSERT(sizeof(ptr_t) == sizeof(word)); - GC_STATIC_ASSERT(sizeof(signed_word) == sizeof(word)); -#if !defined(_AUX_SOURCE) || defined(__GNUC__) - GC_STATIC_ASSERT((word)(-1) > (word)0); - /* word should be unsigned */ -#endif - /* We no longer check for ((void*)(-1) > NULL) since all pointers */ - /* are explicitly cast to word in every less/greater comparison. */ - GC_STATIC_ASSERT((signed_word)(-1) < (signed_word)0); -#endif - GC_STATIC_ASSERT(sizeof(struct hblk) == HBLKSIZE); -#ifndef THREADS - GC_ASSERT(!((word)GC_stackbottom HOTTER_THAN(word) GC_approx_sp())); -#endif - GC_init_headers(); -#ifdef SEARCH_FOR_DATA_START - /* For MPROTECT_VDB, the temporary fault handler should be */ - /* installed first, before the write fault one in GC_dirty_init. */ - if (GC_REGISTER_MAIN_STATIC_DATA()) GC_init_linux_data_start(); -#endif -#ifndef GC_DISABLE_INCREMENTAL - if (GC_incremental || 0 != GETENV("GC_ENABLE_INCREMENTAL")) { -#if defined(BASE_ATOMIC_OPS_EMULATED) || defined(CHECKSUMS) || defined(REDIRECT_MALLOC) || defined(REDIRECT_MALLOC_IN_HEADER) || defined(SMALL_CONFIG) - /* TODO: Implement CHECKSUMS for manual VDB. */ -#else - if (manual_vdb_allowed) { - GC_manual_vdb = TRUE; - GC_incremental = TRUE; - } else -#endif - /* else */ { - /* For GWW_VDB on Win32, this needs to happen before any */ - /* heap memory is allocated. */ - GC_incremental = GC_dirty_init(); - GC_ASSERT(GC_bytes_allocd == 0); - } - } -#endif - - /* Add initial guess of root sets. Do this first, since sbrk(0) */ - /* might be used. */ - if (GC_REGISTER_MAIN_STATIC_DATA()) GC_register_data_segments(); - - GC_bl_init(); - GC_mark_init(); - { - char *sz_str = GETENV("GC_INITIAL_HEAP_SIZE"); - if (sz_str != NULL) { - word value = GC_parse_mem_size_arg(sz_str); - if (GC_WORD_MAX == value) { - WARN("Bad initial heap size %s - ignoring\n", sz_str); - } else { - initial_heap_sz = value; - } - } - } - { - char *sz_str = GETENV("GC_MAXIMUM_HEAP_SIZE"); - if (sz_str != NULL) { - word max_heap_sz = GC_parse_mem_size_arg(sz_str); - if (max_heap_sz < initial_heap_sz || GC_WORD_MAX == max_heap_sz) { - WARN("Bad maximum heap size %s - ignoring\n", sz_str); - } else { - if (0 == GC_max_retries) GC_max_retries = 2; - GC_set_max_heap_size(max_heap_sz); - } - } - } - if (initial_heap_sz != 0) { - if (!GC_expand_hp_inner(divHBLKSZ(initial_heap_sz))) { - GC_err_printf("Can't start up: not enough memory\n"); - EXIT(); - } else { - GC_requested_heapsize += initial_heap_sz; - } - } - if (GC_all_interior_pointers) - GC_initialize_offsets(); - GC_register_displacement_inner(0L); -#if defined(GC_LINUX_THREADS) && defined(REDIRECT_MALLOC) - if (!GC_all_interior_pointers) { - /* TLS ABI uses pointer-sized offsets for dtv. */ - GC_register_displacement_inner(sizeof(void *)); - } -#endif - GC_init_size_map(); -#ifdef PCR - if (PCR_IL_Lock(PCR_Bool_false, PCR_allSigsBlocked, PCR_waitForever) != PCR_ERes_okay) { - ABORT("Can't lock load state"); - } else if (PCR_IL_Unlock() != PCR_ERes_okay) { - ABORT("Can't unlock load state"); - } - PCR_IL_Unlock(); - GC_pcr_install(); -#endif - GC_is_initialized = TRUE; -#ifdef THREADS -#if defined(LINT2) && !(defined(GC_ASSERTIONS) && defined(GC_ALWAYS_MULTITHREADED)) - LOCK(); - GC_thr_init(); - UNLOCK(); -#else - GC_thr_init(); -#endif -#endif - COND_DUMP; - /* Get black list set up and/or incremental GC started */ - if (!GC_dont_precollect || GC_incremental) { - GC_gcollect_inner(); - } -#if defined(GC_ASSERTIONS) && defined(GC_ALWAYS_MULTITHREADED) - UNLOCK(); -#endif -#if defined(THREADS) && defined(UNIX_LIKE) && !defined(NO_GETCONTEXT) - /* Ensure getcontext_works is set to avoid potential data race. */ - if (GC_dont_gc || GC_dont_precollect) - GC_with_callee_saves_pushed(callee_saves_pushed_dummy_fn, NULL); -#endif -#ifndef DONT_USE_ATEXIT - if (GC_find_leak) { - /* This is to give us at least one chance to detect leaks. */ - /* This may report some very benign leaks, but ... */ - atexit(GC_exit_check); - } -#endif - - /* The rest of this again assumes we don't really hold */ - /* the allocation lock. */ - -#ifdef THREADS - /* Initialize thread-local allocation. */ - GC_init_parallel(); -#endif - -#if defined(DYNAMIC_LOADING) && defined(DARWIN) - /* This must be called WITHOUT the allocation lock held */ - /* and before any threads are created. */ - GC_init_dyld(); -#endif - RESTORE_CANCEL(cancel_state); -} - -GC_API void GC_CALL GC_enable_incremental(void) { -#if !defined(GC_DISABLE_INCREMENTAL) && !defined(KEEP_BACK_PTRS) - /* If we are keeping back pointers, the GC itself dirties all */ - /* pages on which objects have been marked, making */ - /* incremental GC pointless. */ - if (!GC_find_leak && 0 == GETENV("GC_DISABLE_INCREMENTAL")) { - LOCK(); - if (!GC_incremental) { - GC_setpagesize(); - /* TODO: Should we skip enabling incremental if win32s? */ - maybe_install_looping_handler(); /* Before write fault handler! */ - if (!GC_is_initialized) { - UNLOCK(); - GC_incremental = TRUE; /* indicate intention to turn it on */ - GC_init(); - LOCK(); - } else { -#if !defined(BASE_ATOMIC_OPS_EMULATED) && !defined(CHECKSUMS) && !defined(REDIRECT_MALLOC) && !defined(REDIRECT_MALLOC_IN_HEADER) && !defined(SMALL_CONFIG) - if (manual_vdb_allowed) { - GC_manual_vdb = TRUE; - GC_incremental = TRUE; - } else -#endif - /* else */ { - GC_incremental = GC_dirty_init(); - } - } - if (GC_incremental && !GC_dont_gc) { - /* Can't easily do it if GC_dont_gc. */ - IF_CANCEL(int cancel_state;) - - DISABLE_CANCEL(cancel_state); - if (GC_bytes_allocd > 0) { - /* There may be unmarked reachable objects. */ - GC_gcollect_inner(); - } - /* else we're OK in assuming everything's */ - /* clean since nothing can point to an */ - /* unmarked object. */ - GC_read_dirty(FALSE); - RESTORE_CANCEL(cancel_state); - } - } - UNLOCK(); - return; - } -#endif - GC_init(); -} - -GC_API void GC_CALL GC_start_mark_threads(void) { -#ifdef PARALLEL_MARK - IF_CANCEL(int cancel_state;) - - DISABLE_CANCEL(cancel_state); - LOCK(); - GC_start_mark_threads_inner(); - UNLOCK(); - RESTORE_CANCEL(cancel_state); -#else - /* No action since parallel markers are disabled (or no POSIX fork). */ - GC_ASSERT(I_DONT_HOLD_LOCK()); -#endif -} - -GC_API void GC_CALL GC_deinit(void) { - if (GC_is_initialized) { - /* Prevent duplicate resource close. */ - GC_is_initialized = FALSE; -#if defined(GC_WIN32_THREADS) && (defined(MSWIN32) || defined(MSWINCE)) -#if !defined(CONSOLE_LOG) || defined(MSWINCE) - DeleteCriticalSection(&GC_write_cs); -#endif -#ifndef GC_PTHREADS - DeleteCriticalSection(&GC_allocate_ml); -#endif -#endif - } -} - -#if (defined(MSWIN32) && !defined(CONSOLE_LOG)) || defined(MSWINCE) - -#if defined(_MSC_VER) && defined(_DEBUG) && !defined(MSWINCE) -#include -#endif - -STATIC HANDLE GC_log = 0; - -#ifdef THREADS -#if defined(PARALLEL_MARK) && !defined(GC_ALWAYS_MULTITHREADED) -#define IF_NEED_TO_LOCK(x) \ - if (GC_parallel || GC_need_to_lock) x -#else -#define IF_NEED_TO_LOCK(x) \ - if (GC_need_to_lock) x -#endif -#else -#define IF_NEED_TO_LOCK(x) -#endif /* !THREADS */ - -#ifdef MSWINRT_FLAVOR -#include - -/* This API is defined in roapi.h, but we cannot include it here */ -/* since it does not compile in C. */ -DECLSPEC_IMPORT HRESULT WINAPI RoGetActivationFactory( - HSTRING activatableClassId, - REFIID iid, void **factory); - -static GC_bool getWinRTLogPath(wchar_t *buf, size_t bufLen) { - static const GUID kIID_IApplicationDataStatics = { - 0x5612147B, 0xE843, 0x45E3, - 0x94, 0xD8, 0x06, 0x16, 0x9E, 0x3C, 0x8E, 0x17}; - static const GUID kIID_IStorageItem = { - 0x4207A996, 0xCA2F, 0x42F7, - 0xBD, 0xE8, 0x8B, 0x10, 0x45, 0x7A, 0x7F, 0x30}; - GC_bool result = FALSE; - HSTRING_HEADER appDataClassNameHeader; - HSTRING appDataClassName; - __x_ABI_CWindows_CStorage_CIApplicationDataStatics *appDataStatics = 0; - - GC_ASSERT(bufLen > 0); - if (SUCCEEDED(WindowsCreateStringReference( - RuntimeClass_Windows_Storage_ApplicationData, - (sizeof(RuntimeClass_Windows_Storage_ApplicationData) - 1) / sizeof(wchar_t), - &appDataClassNameHeader, &appDataClassName)) && - SUCCEEDED(RoGetActivationFactory(appDataClassName, - &kIID_IApplicationDataStatics, - &appDataStatics))) { - __x_ABI_CWindows_CStorage_CIApplicationData *appData = NULL; - __x_ABI_CWindows_CStorage_CIStorageFolder *tempFolder = NULL; - __x_ABI_CWindows_CStorage_CIStorageItem *tempFolderItem = NULL; - HSTRING tempPath = NULL; - - if (SUCCEEDED(appDataStatics->lpVtbl->get_Current(appDataStatics, - &appData)) && - SUCCEEDED(appData->lpVtbl->get_TemporaryFolder(appData, - &tempFolder)) && - SUCCEEDED(tempFolder->lpVtbl->QueryInterface(tempFolder, - &kIID_IStorageItem, - &tempFolderItem)) && - SUCCEEDED(tempFolderItem->lpVtbl->get_Path(tempFolderItem, - &tempPath))) { - UINT32 tempPathLen; - const wchar_t *tempPathBuf = - WindowsGetStringRawBuffer(tempPath, &tempPathLen); - - buf[0] = '\0'; - if (wcsncat_s(buf, bufLen, tempPathBuf, tempPathLen) == 0 && wcscat_s(buf, bufLen, L"\\") == 0 && wcscat_s(buf, bufLen, TEXT(GC_LOG_STD_NAME)) == 0) - result = TRUE; - WindowsDeleteString(tempPath); - } - - if (tempFolderItem != NULL) - tempFolderItem->lpVtbl->Release(tempFolderItem); - if (tempFolder != NULL) - tempFolder->lpVtbl->Release(tempFolder); - if (appData != NULL) - appData->lpVtbl->Release(appData); - appDataStatics->lpVtbl->Release(appDataStatics); - } - return result; -} -#endif /* MSWINRT_FLAVOR */ - -STATIC HANDLE GC_CreateLogFile(void) { - HANDLE hFile; -#ifdef MSWINRT_FLAVOR - TCHAR pathBuf[_MAX_PATH + 0x10]; /* buffer for path + ext */ - - hFile = INVALID_HANDLE_VALUE; - if (getWinRTLogPath(pathBuf, _MAX_PATH + 1)) { - CREATEFILE2_EXTENDED_PARAMETERS extParams; - - BZERO(&extParams, sizeof(extParams)); - extParams.dwSize = sizeof(extParams); - extParams.dwFileAttributes = FILE_ATTRIBUTE_NORMAL; - extParams.dwFileFlags = GC_print_stats == VERBOSE ? 0 - : FILE_FLAG_WRITE_THROUGH; - hFile = CreateFile2(pathBuf, GENERIC_WRITE, FILE_SHARE_READ, - CREATE_ALWAYS, &extParams); - } - -#else - TCHAR *logPath; -#if defined(NO_GETENV_WIN32) && defined(CPPCHECK) -#define appendToFile FALSE -#else - BOOL appendToFile = FALSE; -#endif -#if !defined(NO_GETENV_WIN32) || !defined(OLD_WIN32_LOG_FILE) - TCHAR pathBuf[_MAX_PATH + 0x10]; /* buffer for path + ext */ - - logPath = pathBuf; -#endif - - /* Use GetEnvironmentVariable instead of GETENV() for unicode support. */ -#ifndef NO_GETENV_WIN32 - if (GetEnvironmentVariable(TEXT("GC_LOG_FILE"), pathBuf, - _MAX_PATH + 1) - - 1U < - (DWORD)_MAX_PATH) { - appendToFile = TRUE; - } else -#endif - /* else */ { - /* Env var not found or its value too long. */ -#ifdef OLD_WIN32_LOG_FILE - logPath = TEXT(GC_LOG_STD_NAME); -#else - int len = (int)GetModuleFileName(NULL /* hModule */, pathBuf, - _MAX_PATH + 1); - /* If GetModuleFileName() has failed then len is 0. */ - if (len > 4 && pathBuf[len - 4] == (TCHAR)'.') { - len -= 4; /* strip executable file extension */ - } - BCOPY(TEXT(".") TEXT(GC_LOG_STD_NAME), &pathBuf[len], - sizeof(TEXT(".") TEXT(GC_LOG_STD_NAME))); -#endif - } - - hFile = CreateFile(logPath, GENERIC_WRITE, FILE_SHARE_READ, - NULL /* lpSecurityAttributes */, - appendToFile ? OPEN_ALWAYS : CREATE_ALWAYS, - GC_print_stats == VERBOSE ? FILE_ATTRIBUTE_NORMAL : - /* immediately flush writes unless very verbose */ - FILE_ATTRIBUTE_NORMAL | FILE_FLAG_WRITE_THROUGH, - NULL /* hTemplateFile */); - -#ifndef NO_GETENV_WIN32 - if (appendToFile && hFile != INVALID_HANDLE_VALUE) { - LONG posHigh = 0; - (void)SetFilePointer(hFile, 0, &posHigh, FILE_END); - /* Seek to file end (ignoring any error) */ - } -#endif -#undef appendToFile -#endif - return hFile; -} - -STATIC int GC_write(const char *buf, size_t len) { - BOOL res; - DWORD written; -#if defined(THREADS) && defined(GC_ASSERTIONS) - static GC_bool inside_write = FALSE; - /* to prevent infinite recursion at abort. */ - if (inside_write) - return -1; -#endif - - if (len == 0) - return 0; - IF_NEED_TO_LOCK(EnterCriticalSection(&GC_write_cs)); -#if defined(THREADS) && defined(GC_ASSERTIONS) - if (GC_write_disabled) { - inside_write = TRUE; - ABORT("Assertion failure: GC_write called with write_disabled"); - } -#endif - if (GC_log == 0) { - GC_log = GC_CreateLogFile(); - } - if (GC_log == INVALID_HANDLE_VALUE) { - IF_NEED_TO_LOCK(LeaveCriticalSection(&GC_write_cs)); -#ifdef NO_DEBUGGING - /* Ignore open log failure (e.g., it might be caused by */ - /* read-only folder of the client application). */ - return 0; -#else - return -1; -#endif - } - res = WriteFile(GC_log, buf, (DWORD)len, &written, NULL); -#if defined(_MSC_VER) && defined(_DEBUG) && !defined(NO_CRT) -#ifdef MSWINCE - /* There is no CrtDbgReport() in WinCE */ - { - WCHAR wbuf[1024]; - /* Always use Unicode variant of OutputDebugString() */ - wbuf[MultiByteToWideChar(CP_ACP, 0 /* dwFlags */, - buf, len, wbuf, - sizeof(wbuf) / sizeof(wbuf[0]) - 1)] = 0; - OutputDebugStringW(wbuf); - } -#else - _CrtDbgReport(_CRT_WARN, NULL, 0, NULL, "%.*s", len, buf); -#endif -#endif - IF_NEED_TO_LOCK(LeaveCriticalSection(&GC_write_cs)); - return res ? (int)written : -1; -} - -/* TODO: This is pretty ugly ... */ -#define WRITE(f, buf, len) GC_write(buf, len) - -#elif defined(OS2) || defined(MACOS) -STATIC FILE *GC_stdout = NULL; -STATIC FILE *GC_stderr = NULL; -STATIC FILE *GC_log = NULL; - -/* Initialize GC_log (and the friends) passed to GC_write(). */ -STATIC void GC_set_files(void) { - if (GC_stdout == NULL) { - GC_stdout = stdout; - } - if (GC_stderr == NULL) { - GC_stderr = stderr; - } - if (GC_log == NULL) { - GC_log = stderr; - } -} - -GC_INLINE int GC_write(FILE *f, const char *buf, size_t len) { - int res = fwrite(buf, 1, len, f); - fflush(f); - return res; -} - -#define WRITE(f, buf, len) (GC_set_files(), GC_write(f, buf, len)) - -#elif defined(GC_ANDROID_LOG) - -#include - -#ifndef GC_ANDROID_LOG_TAG -#define GC_ANDROID_LOG_TAG "BDWGC" -#endif - -#define GC_stdout ANDROID_LOG_DEBUG -#define GC_stderr ANDROID_LOG_ERROR -#define GC_log GC_stdout - -#define WRITE(level, buf, unused_len) \ - __android_log_write(level, GC_ANDROID_LOG_TAG, buf) - -#elif defined(NN_PLATFORM_CTR) -int n3ds_log_write(const char *text, int length); -#define WRITE(level, buf, len) n3ds_log_write(buf, len) - -#elif defined(NINTENDO_SWITCH) -int switch_log_write(const char *text, int length); -#define WRITE(level, buf, len) switch_log_write(buf, len) - -#else - -#if !defined(GC_NO_TYPES) && !defined(SN_TARGET_PSP2) -#if !defined(AMIGA) && !defined(MSWIN32) && !defined(MSWIN_XBOX1) && !defined(__CC_ARM) -#include -#endif -#if !defined(ECOS) && !defined(NOSYS) -#include -#endif -#endif /* !GC_NO_TYPES && !SN_TARGET_PSP2 */ - -STATIC int GC_write(int fd, const char *buf, size_t len) { -#if defined(ECOS) || defined(PLATFORM_WRITE) || defined(SN_TARGET_PSP2) || defined(NOSYS) -#ifdef ECOS - /* FIXME: This seems to be defined nowhere at present. */ - /* _Jv_diag_write(buf, len); */ -#else - /* No writing. */ -#endif - return len; -#else - int bytes_written = 0; - IF_CANCEL(int cancel_state;) - - DISABLE_CANCEL(cancel_state); - while ((unsigned)bytes_written < len) { -#ifdef GC_SOLARIS_THREADS - int result = syscall(SYS_write, fd, buf + bytes_written, - len - bytes_written); -#elif defined(_MSC_VER) - int result = _write(fd, buf + bytes_written, - (unsigned)(len - bytes_written)); -#else - int result = write(fd, buf + bytes_written, len - bytes_written); -#endif - - if (-1 == result) { - if (EAGAIN == errno) /* Resource temporarily unavailable */ - continue; - RESTORE_CANCEL(cancel_state); - return result; - } - bytes_written += result; - } - RESTORE_CANCEL(cancel_state); - return bytes_written; -#endif -} - -#define WRITE(f, buf, len) GC_write(f, buf, len) -#endif /* !MSWINCE && !OS2 && !MACOS && !GC_ANDROID_LOG */ - -#define BUFSZ 1024 - -#if defined(DJGPP) || defined(__STRICT_ANSI__) -/* vsnprintf is missing in DJGPP (v2.0.3) */ -#define GC_VSNPRINTF(buf, bufsz, format, args) vsprintf(buf, format, args) -#elif defined(_MSC_VER) -#ifdef MSWINCE -/* _vsnprintf is deprecated in WinCE */ -#define GC_VSNPRINTF StringCchVPrintfA -#else -#define GC_VSNPRINTF _vsnprintf -#endif -#else -#define GC_VSNPRINTF vsnprintf -#endif - -/* A version of printf that is unlikely to call malloc, and is thus safer */ -/* to call from the collector in case malloc has been bound to GC_malloc. */ -/* Floating point arguments and formats should be avoided, since FP */ -/* conversion is more likely to allocate memory. */ -/* Assumes that no more than BUFSZ-1 characters are written at once. */ -#define GC_PRINTF_FILLBUF(buf, format) \ - do { \ - va_list args; \ - va_start(args, format); \ - (buf)[sizeof(buf) - 1] = 0x15; /* guard */ \ - (void)GC_VSNPRINTF(buf, sizeof(buf) - 1, format, args); \ - va_end(args); \ - if ((buf)[sizeof(buf) - 1] != 0x15) \ - ABORT("GC_printf clobbered stack"); \ - } while (0) - -void GC_printf(const char *format, ...) { - if (!GC_quiet) { - char buf[BUFSZ + 1]; - - GC_PRINTF_FILLBUF(buf, format); -#ifdef NACL - (void)WRITE(GC_stdout, buf, strlen(buf)); - /* Ignore errors silently. */ -#else - if (WRITE(GC_stdout, buf, strlen(buf)) < 0 -#if defined(CYGWIN32) || (defined(CONSOLE_LOG) && defined(MSWIN32)) - && GC_stdout != GC_DEFAULT_STDOUT_FD -#endif - ) { - ABORT("write to stdout failed"); - } -#endif - } -} - -void GC_err_printf(const char *format, ...) { - char buf[BUFSZ + 1]; - - GC_PRINTF_FILLBUF(buf, format); - GC_err_puts(buf); -} - -void GC_log_printf(const char *format, ...) { - char buf[BUFSZ + 1]; - - GC_PRINTF_FILLBUF(buf, format); -#ifdef NACL - (void)WRITE(GC_log, buf, strlen(buf)); -#else - if (WRITE(GC_log, buf, strlen(buf)) < 0 -#if defined(CYGWIN32) || (defined(CONSOLE_LOG) && defined(MSWIN32)) - && GC_log != GC_DEFAULT_STDERR_FD -#endif - ) { - ABORT("write to GC log failed"); - } -#endif -} - -#ifndef GC_ANDROID_LOG - -#define GC_warn_printf GC_err_printf - -#else - -GC_INNER void GC_info_log_printf(const char *format, ...) { - char buf[BUFSZ + 1]; - - GC_PRINTF_FILLBUF(buf, format); - (void)WRITE(ANDROID_LOG_INFO, buf, 0 /* unused */); -} - -GC_INNER void GC_verbose_log_printf(const char *format, ...) { - char buf[BUFSZ + 1]; - - GC_PRINTF_FILLBUF(buf, format); - (void)WRITE(ANDROID_LOG_VERBOSE, buf, 0); /* ignore write errors */ -} - -STATIC void GC_warn_printf(const char *format, ...) { - char buf[BUFSZ + 1]; - - GC_PRINTF_FILLBUF(buf, format); - (void)WRITE(ANDROID_LOG_WARN, buf, 0); -} - -#endif /* GC_ANDROID_LOG */ - -void GC_err_puts(const char *s) { - (void)WRITE(GC_stderr, s, strlen(s)); /* ignore errors */ -} - -STATIC void GC_CALLBACK GC_default_warn_proc(char *msg, GC_word arg) { - /* TODO: Add assertion on arg comply with msg (format). */ - GC_warn_printf(msg, arg); -} - -GC_INNER GC_warn_proc GC_current_warn_proc = GC_default_warn_proc; - -/* This is recommended for production code (release). */ -GC_API void GC_CALLBACK GC_ignore_warn_proc(char *msg, GC_word arg) { - if (GC_print_stats) { - /* Don't ignore warnings if stats printing is on. */ - GC_default_warn_proc(msg, arg); - } -} - -GC_API void GC_CALL GC_set_warn_proc(GC_warn_proc p) { - GC_ASSERT(NONNULL_ARG_NOT_NULL(p)); -#ifdef GC_WIN32_THREADS -#ifdef CYGWIN32 - /* Need explicit GC_INIT call */ - GC_ASSERT(GC_is_initialized); -#else - if (!GC_is_initialized) GC_init(); -#endif -#endif - LOCK(); - GC_current_warn_proc = p; - UNLOCK(); -} - -GC_API GC_warn_proc GC_CALL GC_get_warn_proc(void) { - GC_warn_proc result; - - LOCK(); - result = GC_current_warn_proc; - UNLOCK(); - return result; -} - -#if !defined(PCR) && !defined(SMALL_CONFIG) -/* Print (or display) a message before abnormal exit (including */ -/* abort). Invoked from ABORT(msg) macro (there msg is non-NULL) */ -/* and from EXIT() macro (msg is NULL in that case). */ -STATIC void GC_CALLBACK GC_default_on_abort(const char *msg) { -#ifndef DONT_USE_ATEXIT - skip_gc_atexit = TRUE; /* disable at-exit GC_gcollect() */ -#endif - - if (msg != NULL) { -#ifdef MSGBOX_ON_ERROR - GC_win32_MessageBoxA(msg, "Fatal error in GC", MB_ICONERROR | MB_OK); - /* Also duplicate msg to GC log file. */ -#endif - -#ifndef GC_ANDROID_LOG - /* Avoid calling GC_err_printf() here, as GC_on_abort() could be */ - /* called from it. Note 1: this is not an atomic output. */ - /* Note 2: possible write errors are ignored. */ -#if defined(GC_WIN32_THREADS) && defined(GC_ASSERTIONS) && ((defined(MSWIN32) && !defined(CONSOLE_LOG)) || defined(MSWINCE)) - if (!GC_write_disabled) -#endif - { - if (WRITE(GC_stderr, msg, strlen(msg)) >= 0) - (void)WRITE(GC_stderr, "\n", 1); - } -#else - __android_log_assert("*" /* cond */, GC_ANDROID_LOG_TAG, "%s\n", msg); -#endif - } - -#if !defined(NO_DEBUGGING) && !defined(GC_ANDROID_LOG) - if (GETENV("GC_LOOP_ON_ABORT") != NULL) { - /* In many cases it's easier to debug a running process. */ - /* It's arguably nicer to sleep, but that makes it harder */ - /* to look at the thread if the debugger doesn't know much */ - /* about threads. */ - for (;;) { - /* Empty */ - } - } -#endif -} - -GC_abort_func GC_on_abort = GC_default_on_abort; - -GC_API void GC_CALL GC_set_abort_func(GC_abort_func fn) { - GC_ASSERT(NONNULL_ARG_NOT_NULL(fn)); - LOCK(); - GC_on_abort = fn; - UNLOCK(); -} - -GC_API GC_abort_func GC_CALL GC_get_abort_func(void) { - GC_abort_func fn; - - LOCK(); - fn = GC_on_abort; - UNLOCK(); - return fn; -} -#endif /* !SMALL_CONFIG */ - -GC_API void GC_CALL GC_enable(void) { - LOCK(); - GC_ASSERT(GC_dont_gc != 0); /* ensure no counter underflow */ - GC_dont_gc--; - if (!GC_dont_gc && GC_heapsize > GC_heapsize_on_gc_disable) - WARN("Heap grown by %" WARN_PRIuPTR " KiB while GC was disabled\n", - (GC_heapsize - GC_heapsize_on_gc_disable) >> 10); - UNLOCK(); -} - -GC_API void GC_CALL GC_disable(void) { - LOCK(); - if (!GC_dont_gc) - GC_heapsize_on_gc_disable = GC_heapsize; - GC_dont_gc++; - UNLOCK(); -} - -GC_API int GC_CALL GC_is_disabled(void) { - return GC_dont_gc != 0; -} - -/* Helper procedures for new kind creation. */ -GC_API void **GC_CALL GC_new_free_list_inner(void) { - void *result; - - GC_ASSERT(I_HOLD_LOCK()); - result = GC_INTERNAL_MALLOC((MAXOBJGRANULES + 1) * sizeof(ptr_t), PTRFREE); - if (NULL == result) ABORT("Failed to allocate freelist for new kind"); - BZERO(result, (MAXOBJGRANULES + 1) * sizeof(ptr_t)); - return (void **)result; -} - -GC_API void **GC_CALL GC_new_free_list(void) { - void **result; - - LOCK(); - result = GC_new_free_list_inner(); - UNLOCK(); - return result; -} - -GC_API unsigned GC_CALL GC_new_kind_inner(void **fl, GC_word descr, - int adjust, int clear) { - unsigned result = GC_n_kinds; - - GC_ASSERT(NONNULL_ARG_NOT_NULL(fl)); - GC_ASSERT(adjust == FALSE || adjust == TRUE); - /* If an object is not needed to be cleared (when moved to the */ - /* free list) then its descriptor should be zero to denote */ - /* a pointer-free object (and, as a consequence, the size of the */ - /* object should not be added to the descriptor template). */ - GC_ASSERT(clear == TRUE || (descr == 0 && adjust == FALSE && clear == FALSE)); - if (result < MAXOBJKINDS) { - GC_ASSERT(result > 0); - GC_n_kinds++; - GC_obj_kinds[result].ok_freelist = fl; - GC_obj_kinds[result].ok_reclaim_list = 0; - GC_obj_kinds[result].ok_descriptor = descr; - GC_obj_kinds[result].ok_relocate_descr = adjust; - GC_obj_kinds[result].ok_init = (GC_bool)clear; -#ifdef ENABLE_DISCLAIM - GC_obj_kinds[result].ok_mark_unconditionally = FALSE; - GC_obj_kinds[result].ok_disclaim_proc = 0; -#endif - } else { - ABORT("Too many kinds"); - } - return result; -} - -GC_API unsigned GC_CALL GC_new_kind(void **fl, GC_word descr, int adjust, - int clear) { - unsigned result; - - LOCK(); - result = GC_new_kind_inner(fl, descr, adjust, clear); - UNLOCK(); - return result; -} - -GC_API unsigned GC_CALL GC_new_proc_inner(GC_mark_proc proc) { - unsigned result = GC_n_mark_procs; - - if (result < MAX_MARK_PROCS) { - GC_n_mark_procs++; - GC_mark_procs[result] = proc; - } else { - ABORT("Too many mark procedures"); - } - return result; -} - -GC_API unsigned GC_CALL GC_new_proc(GC_mark_proc proc) { - unsigned result; - - LOCK(); - result = GC_new_proc_inner(proc); - UNLOCK(); - return result; -} - -GC_API void *GC_CALL GC_call_with_alloc_lock(GC_fn_type fn, void *client_data) { - void *result; - -#ifdef THREADS - LOCK(); -#endif - result = fn(client_data); -#ifdef THREADS - UNLOCK(); -#endif - return result; -} - -GC_API void *GC_CALL GC_call_with_stack_base(GC_stack_base_func fn, void *arg) { - struct GC_stack_base base; - void *result; - - base.mem_base = (void *)&base; -#ifdef IA64 - base.reg_base = (void *)GC_save_regs_in_stack(); - /* TODO: Unnecessarily flushes register stack, */ - /* but that probably doesn't hurt. */ -#elif defined(E2K) - base.reg_base = NULL; /* not used by GC currently */ -#endif - result = (*fn)(&base, arg); - /* Strongly discourage the compiler from treating the above */ - /* as a tail call. */ - GC_noop1(COVERT_DATAFLOW(&base)); - return result; -} - -#ifndef THREADS - -GC_INNER ptr_t GC_blocked_sp = NULL; -/* NULL value means we are not inside GC_do_blocking() call. */ -#ifdef IA64 -STATIC ptr_t GC_blocked_register_sp = NULL; -#endif - -GC_INNER struct GC_traced_stack_sect_s *GC_traced_stack_sect = NULL; - -/* This is nearly the same as in pthread_support.c. */ -GC_API void *GC_CALL GC_call_with_gc_active(GC_fn_type fn, - void *client_data) { - struct GC_traced_stack_sect_s stacksect; - GC_ASSERT(GC_is_initialized); - - /* Adjust our stack bottom pointer (this could happen if */ - /* GC_get_main_stack_base() is unimplemented or broken for */ - /* the platform). */ - if ((word)GC_stackbottom HOTTER_THAN(word)(&stacksect)) - GC_stackbottom = (ptr_t)COVERT_DATAFLOW(&stacksect); - - if (GC_blocked_sp == NULL) { - /* We are not inside GC_do_blocking() - do nothing more. */ - client_data = fn(client_data); - /* Prevent treating the above as a tail call. */ - GC_noop1(COVERT_DATAFLOW(&stacksect)); - return client_data; /* result */ - } - - /* Setup new "stack section". */ - stacksect.saved_stack_ptr = GC_blocked_sp; -#ifdef IA64 - /* This is the same as in GC_call_with_stack_base(). */ - stacksect.backing_store_end = GC_save_regs_in_stack(); - /* Unnecessarily flushes register stack, */ - /* but that probably doesn't hurt. */ - stacksect.saved_backing_store_ptr = GC_blocked_register_sp; -#endif - stacksect.prev = GC_traced_stack_sect; - GC_blocked_sp = NULL; - GC_traced_stack_sect = &stacksect; - - client_data = fn(client_data); - GC_ASSERT(GC_blocked_sp == NULL); - GC_ASSERT(GC_traced_stack_sect == &stacksect); - -#if defined(CPPCHECK) - GC_noop1((word)GC_traced_stack_sect - (word)GC_blocked_sp); -#endif - /* Restore original "stack section". */ - GC_traced_stack_sect = stacksect.prev; -#ifdef IA64 - GC_blocked_register_sp = stacksect.saved_backing_store_ptr; -#endif - GC_blocked_sp = stacksect.saved_stack_ptr; - - return client_data; /* result */ -} - -/* This is nearly the same as in pthread_support.c. */ -STATIC void GC_do_blocking_inner(ptr_t data, void *context) { - struct blocking_data *d = (struct blocking_data *)data; - - UNUSED_ARG(context); - GC_ASSERT(GC_is_initialized); - GC_ASSERT(GC_blocked_sp == NULL); -#ifdef SPARC - GC_blocked_sp = GC_save_regs_in_stack(); -#else - GC_blocked_sp = (ptr_t)&d; /* save approx. sp */ -#ifdef IA64 - GC_blocked_register_sp = GC_save_regs_in_stack(); -#elif defined(E2K) - (void)GC_save_regs_in_stack(); -#endif -#endif - - d->client_data = (d->fn)(d->client_data); - -#ifdef SPARC - GC_ASSERT(GC_blocked_sp != NULL); -#else - GC_ASSERT(GC_blocked_sp == (ptr_t)(&d)); -#endif -#if defined(CPPCHECK) - GC_noop1((word)GC_blocked_sp); -#endif - GC_blocked_sp = NULL; -} - -GC_API void GC_CALL GC_set_stackbottom(void *gc_thread_handle, - const struct GC_stack_base *sb) { - GC_ASSERT(sb->mem_base != NULL); - GC_ASSERT(NULL == gc_thread_handle || &GC_stackbottom == gc_thread_handle); - GC_ASSERT(NULL == GC_blocked_sp && NULL == GC_traced_stack_sect); /* for now */ - (void)gc_thread_handle; - - GC_stackbottom = (char *)sb->mem_base; -#ifdef IA64 - GC_register_stackbottom = (ptr_t)sb->reg_base; -#endif -} - -GC_API void *GC_CALL GC_get_my_stackbottom(struct GC_stack_base *sb) { - GC_ASSERT(GC_is_initialized); - sb->mem_base = GC_stackbottom; -#ifdef IA64 - sb->reg_base = GC_register_stackbottom; -#elif defined(E2K) - sb->reg_base = NULL; -#endif - return &GC_stackbottom; /* gc_thread_handle */ -} -#endif /* !THREADS */ - -GC_API void *GC_CALL GC_do_blocking(GC_fn_type fn, void *client_data) { - struct blocking_data my_data; - - my_data.fn = fn; - my_data.client_data = client_data; - GC_with_callee_saves_pushed(GC_do_blocking_inner, (ptr_t)(&my_data)); - return my_data.client_data; /* result */ -} - -#if !defined(NO_DEBUGGING) -GC_API void GC_CALL GC_dump(void) { - LOCK(); - GC_dump_named(NULL); - UNLOCK(); -} - -GC_API void GC_CALL GC_dump_named(const char *name) { -#ifndef NO_CLOCK - CLOCK_TYPE current_time; - - GET_TIME(current_time); -#endif - if (name != NULL) { - GC_printf("\n***GC Dump %s\n", name); - } else { - GC_printf("\n***GC Dump collection #%lu\n", (unsigned long)GC_gc_no); - } -#ifndef NO_CLOCK - /* Note that the time is wrapped in ~49 days if sizeof(long)==4. */ - GC_printf("Time since GC init: %lu ms\n", - MS_TIME_DIFF(current_time, GC_init_time)); -#endif - - GC_printf("\n***Static roots:\n"); - GC_print_static_roots(); - GC_printf("\n***Heap sections:\n"); - GC_print_heap_sects(); - GC_printf("\n***Free blocks:\n"); - GC_print_hblkfreelist(); - GC_printf("\n***Blocks in use:\n"); - GC_print_block_list(); -#ifndef GC_NO_FINALIZATION - GC_dump_finalization(); -#endif -} -#endif /* !NO_DEBUGGING */ - -static void GC_CALLBACK block_add_size(struct hblk *h, GC_word pbytes) { - hdr *hhdr = HDR(h); - *(word *)pbytes += (WORDS_TO_BYTES(hhdr->hb_sz) + HBLKSIZE - 1) & ~(word)(HBLKSIZE - 1); -} - -GC_API size_t GC_CALL GC_get_memory_use(void) { - word bytes = 0; - - LOCK(); - GC_apply_to_all_blocks(block_add_size, (word)(&bytes)); - UNLOCK(); - return (size_t)bytes; -} - -/* Getter functions for the public Read-only variables. */ - -/* GC_get_gc_no() is unsynchronized and should be typically called */ -/* inside the context of GC_call_with_alloc_lock() to prevent data */ -/* races (on multiprocessors). */ -GC_API GC_word GC_CALL GC_get_gc_no(void) { - return GC_gc_no; -} - -#ifndef PARALLEL_MARK -GC_API void GC_CALL GC_set_markers_count(unsigned markers) { - UNUSED_ARG(markers); -} -#endif - -GC_API int GC_CALL GC_get_parallel(void) { -#ifdef THREADS - return GC_parallel; -#else - return 0; -#endif -} - -#ifdef THREADS -GC_API void GC_CALL GC_alloc_lock(void) { - LOCK(); -} - -GC_API void GC_CALL GC_alloc_unlock(void) { - UNLOCK(); -} - -GC_INNER GC_on_thread_event_proc GC_on_thread_event = 0; - -GC_API void GC_CALL GC_set_on_thread_event(GC_on_thread_event_proc fn) { - /* fn may be 0 (means no event notifier). */ - LOCK(); - GC_on_thread_event = fn; - UNLOCK(); -} - -GC_API GC_on_thread_event_proc GC_CALL GC_get_on_thread_event(void) { - GC_on_thread_event_proc fn; - - LOCK(); - fn = GC_on_thread_event; - UNLOCK(); - return fn; -} -#endif /* THREADS */ - -/* Setter and getter functions for the public R/W function variables. */ -/* These functions are synchronized (like GC_set_warn_proc() and */ -/* GC_get_warn_proc()). */ - -GC_API void GC_CALL GC_set_oom_fn(GC_oom_func fn) { - GC_ASSERT(NONNULL_ARG_NOT_NULL(fn)); - LOCK(); - GC_oom_fn = fn; - UNLOCK(); -} - -GC_API GC_oom_func GC_CALL GC_get_oom_fn(void) { - GC_oom_func fn; - - LOCK(); - fn = GC_oom_fn; - UNLOCK(); - return fn; -} - -GC_API void GC_CALL GC_set_on_heap_resize(GC_on_heap_resize_proc fn) { - /* fn may be 0 (means no event notifier). */ - LOCK(); - GC_on_heap_resize = fn; - UNLOCK(); -} - -GC_API GC_on_heap_resize_proc GC_CALL GC_get_on_heap_resize(void) { - GC_on_heap_resize_proc fn; - - LOCK(); - fn = GC_on_heap_resize; - UNLOCK(); - return fn; -} - -GC_API void GC_CALL GC_set_finalizer_notifier(GC_finalizer_notifier_proc fn) { - /* fn may be 0 (means no finalizer notifier). */ - LOCK(); - GC_finalizer_notifier = fn; - UNLOCK(); -} - -GC_API GC_finalizer_notifier_proc GC_CALL GC_get_finalizer_notifier(void) { - GC_finalizer_notifier_proc fn; - - LOCK(); - fn = GC_finalizer_notifier; - UNLOCK(); - return fn; -} - -/* Setter and getter functions for the public numeric R/W variables. */ -/* It is safe to call these functions even before GC_INIT(). */ -/* These functions are unsynchronized and should be typically called */ -/* inside the context of GC_call_with_alloc_lock() (if called after */ -/* GC_INIT()) to prevent data races (unless it is guaranteed the */ -/* collector is not multi-threaded at that execution point). */ - -GC_API void GC_CALL GC_set_find_leak(int value) { - /* value is of boolean type. */ - GC_find_leak = value; -} - -GC_API int GC_CALL GC_get_find_leak(void) { - return GC_find_leak; -} - -GC_API void GC_CALL GC_set_all_interior_pointers(int value) { - GC_all_interior_pointers = value ? 1 : 0; - if (GC_is_initialized) { - /* It is not recommended to change GC_all_interior_pointers value */ - /* after GC is initialized but it seems GC could work correctly */ - /* even after switching the mode. */ - LOCK(); - GC_initialize_offsets(); /* NOTE: this resets manual offsets as well */ - if (!GC_all_interior_pointers) - GC_bl_init_no_interiors(); - UNLOCK(); - } -} - -GC_API int GC_CALL GC_get_all_interior_pointers(void) { - return GC_all_interior_pointers; -} - -GC_API void GC_CALL GC_set_finalize_on_demand(int value) { - GC_ASSERT(value != -1); /* -1 was used to retrieve old value in gc-7.2 */ - /* value is of boolean type. */ - GC_finalize_on_demand = value; -} - -GC_API int GC_CALL GC_get_finalize_on_demand(void) { - return GC_finalize_on_demand; -} - -GC_API void GC_CALL GC_set_java_finalization(int value) { - GC_ASSERT(value != -1); /* -1 was used to retrieve old value in gc-7.2 */ - /* value is of boolean type. */ - GC_java_finalization = value; -} - -GC_API int GC_CALL GC_get_java_finalization(void) { - return GC_java_finalization; -} - -GC_API void GC_CALL GC_set_dont_expand(int value) { - GC_ASSERT(value != -1); /* -1 was used to retrieve old value in gc-7.2 */ - /* value is of boolean type. */ - GC_dont_expand = value; -} - -GC_API int GC_CALL GC_get_dont_expand(void) { - return GC_dont_expand; -} - -GC_API void GC_CALL GC_set_no_dls(int value) { - GC_ASSERT(value != -1); /* -1 was used to retrieve old value in gc-7.2 */ - /* value is of boolean type. */ - GC_no_dls = value; -} - -GC_API int GC_CALL GC_get_no_dls(void) { - return GC_no_dls; -} - -GC_API void GC_CALL GC_set_non_gc_bytes(GC_word value) { - GC_non_gc_bytes = value; -} - -GC_API GC_word GC_CALL GC_get_non_gc_bytes(void) { - return GC_non_gc_bytes; -} - -GC_API void GC_CALL GC_set_free_space_divisor(GC_word value) { - GC_ASSERT(value > 0); - GC_free_space_divisor = value; -} - -GC_API GC_word GC_CALL GC_get_free_space_divisor(void) { - return GC_free_space_divisor; -} - -GC_API void GC_CALL GC_set_max_retries(GC_word value) { - GC_ASSERT((GC_signed_word)value != -1); - /* -1 was used to retrieve old value in gc-7.2 */ - GC_max_retries = value; -} - -GC_API GC_word GC_CALL GC_get_max_retries(void) { - return GC_max_retries; -} - -GC_API void GC_CALL GC_set_dont_precollect(int value) { - GC_ASSERT(value != -1); /* -1 was used to retrieve old value in gc-7.2 */ - /* value is of boolean type. */ - GC_dont_precollect = value; -} - -GC_API int GC_CALL GC_get_dont_precollect(void) { - return GC_dont_precollect; -} - -GC_API void GC_CALL GC_set_full_freq(int value) { - GC_ASSERT(value >= 0); - GC_full_freq = value; -} - -GC_API int GC_CALL GC_get_full_freq(void) { - return GC_full_freq; -} - -GC_API void GC_CALL GC_set_time_limit(unsigned long value) { - GC_ASSERT((long)value != -1L); - /* -1 was used to retrieve old value in gc-7.2 */ - GC_time_limit = value; -} - -GC_API unsigned long GC_CALL GC_get_time_limit(void) { - return GC_time_limit; -} - -GC_API void GC_CALL GC_set_force_unmap_on_gcollect(int value) { - GC_force_unmap_on_gcollect = (GC_bool)value; -} - -GC_API int GC_CALL GC_get_force_unmap_on_gcollect(void) { - return (int)GC_force_unmap_on_gcollect; -} - -GC_API void GC_CALL GC_abort_on_oom(void) { - GC_err_printf("Insufficient memory for the allocation\n"); - EXIT(); -} - -GC_API size_t GC_CALL GC_get_hblk_size(void) { - return (size_t)HBLKSIZE; -} - -#ifdef THREADS -GC_API void GC_CALL GC_stop_world_external(void) { - GC_ASSERT(GC_is_initialized); - LOCK(); -#ifdef THREAD_LOCAL_ALLOC - GC_ASSERT(!GC_world_stopped); -#endif - STOP_WORLD(); -#ifdef THREAD_LOCAL_ALLOC - GC_world_stopped = TRUE; -#endif -} - -GC_API void GC_CALL GC_start_world_external(void) { -#ifdef THREAD_LOCAL_ALLOC - GC_ASSERT(GC_world_stopped); - GC_world_stopped = FALSE; -#else - GC_ASSERT(GC_is_initialized); -#endif - START_WORLD(); - UNLOCK(); -} -#endif /* THREADS */ diff --git a/vendor/bdwgc/new_hblk.c b/vendor/bdwgc/new_hblk.c deleted file mode 100644 index f78b93ce..00000000 --- a/vendor/bdwgc/new_hblk.c +++ /dev/null @@ -1,190 +0,0 @@ -/* - * Copyright 1988, 1989 Hans-J. Boehm, Alan J. Demers - * Copyright (c) 1991-1994 by Xerox Corporation. All rights reserved. - * Copyright (c) 2000 by Hewlett-Packard Company. All rights reserved. - * - * THIS MATERIAL IS PROVIDED AS IS, WITH ABSOLUTELY NO WARRANTY EXPRESSED - * OR IMPLIED. ANY USE IS AT YOUR OWN RISK. - * - * Permission is hereby granted to use or copy this program - * for any purpose, provided the above notices are retained on all copies. - * Permission to modify the code and to distribute modified code is granted, - * provided the above notices are retained, and a notice that the code was - * modified is included with the above copyright notice. - */ - -#ifdef _WIN32 -#define WIN32_LEAN_AND_MEAN -#include -#endif -#include "private/gc_priv.h" - -/* - * This file contains the functions: - * ptr_t GC_build_flXXX(h, old_fl) - * void GC_new_hblk(size, kind) - */ - -#ifndef SMALL_CONFIG -/* Build a free list for size 2 (words) cleared objects inside */ -/* hblk h. Set the last link to be ofl. Return a pointer to the */ -/* first free list entry. */ -STATIC ptr_t GC_build_fl_clear2(struct hblk *h, ptr_t ofl) { - word *p = (word *)(h->hb_body); - word *lim = (word *)(h + 1); - - p[0] = (word)ofl; - p[1] = 0; - p[2] = (word)p; - p[3] = 0; - p += 4; - for (; (word)p < (word)lim; p += 4) { - p[0] = (word)(p - 2); - p[1] = 0; - p[2] = (word)p; - p[3] = 0; - } - return (ptr_t)(p - 2); -} - -/* The same for size 4 cleared objects. */ -STATIC ptr_t GC_build_fl_clear4(struct hblk *h, ptr_t ofl) { - word *p = (word *)(h->hb_body); - word *lim = (word *)(h + 1); - - p[0] = (word)ofl; - p[1] = 0; - p[2] = 0; - p[3] = 0; - p += 4; - for (; (word)p < (word)lim; p += 4) { - GC_PREFETCH_FOR_WRITE((ptr_t)(p + 64)); - p[0] = (word)(p - 4); - p[1] = 0; - CLEAR_DOUBLE(p + 2); - } - return (ptr_t)(p - 4); -} - -/* The same for size 2 uncleared objects. */ -STATIC ptr_t GC_build_fl2(struct hblk *h, ptr_t ofl) { - word *p = (word *)(h->hb_body); - word *lim = (word *)(h + 1); - - p[0] = (word)ofl; - p[2] = (word)p; - p += 4; - for (; (word)p < (word)lim; p += 4) { - p[0] = (word)(p - 2); - p[2] = (word)p; - } - return (ptr_t)(p - 2); -} - -/* The same for size 4 uncleared objects. */ -STATIC ptr_t GC_build_fl4(struct hblk *h, ptr_t ofl) { - word *p = (word *)(h->hb_body); - word *lim = (word *)(h + 1); - - p[0] = (word)ofl; - p[4] = (word)p; - p += 8; - for (; (word)p < (word)lim; p += 8) { - GC_PREFETCH_FOR_WRITE((ptr_t)(p + 64)); - p[0] = (word)(p - 4); - p[4] = (word)p; - } - return (ptr_t)(p - 4); -} -#endif /* !SMALL_CONFIG */ - -/* Build a free list for objects of size sz inside heap block h. */ -/* Clear objects inside h if clear is set. Add list to the end of */ -/* the free list we build. Return the new free list. */ -/* This could be called without the allocation lock, if we ensure that */ -/* there is no concurrent collection which might reclaim objects that */ -/* we have not yet allocated. */ -GC_INNER ptr_t GC_build_fl(struct hblk *h, size_t sz, GC_bool clear, - ptr_t list) { - word *p, *prev; - word *last_object; /* points to last object in new hblk */ - - /* Do a few prefetches here, just because it's cheap. */ - /* If we were more serious about it, these should go inside */ - /* the loops. But write prefetches usually don't seem to */ - /* matter much. */ - GC_PREFETCH_FOR_WRITE((ptr_t)h); - GC_PREFETCH_FOR_WRITE((ptr_t)h + 128); - GC_PREFETCH_FOR_WRITE((ptr_t)h + 256); - GC_PREFETCH_FOR_WRITE((ptr_t)h + 378); -#ifndef SMALL_CONFIG - /* Handle small objects sizes more efficiently. For larger objects */ - /* the difference is less significant. */ - switch (sz) { - case 2: - if (clear) { - return GC_build_fl_clear2(h, list); - } else { - return GC_build_fl2(h, list); - } - case 4: - if (clear) { - return GC_build_fl_clear4(h, list); - } else { - return GC_build_fl4(h, list); - } - default: - break; - } -#endif /* !SMALL_CONFIG */ - - /* Clear the page if necessary. */ - if (clear) BZERO(h, HBLKSIZE); - - /* Add objects to free list */ - p = (word *)(h->hb_body) + sz; /* second object in *h */ - prev = (word *)(h->hb_body); /* One object behind p */ - last_object = (word *)((char *)h + HBLKSIZE); - last_object -= sz; - /* Last place for last object to start */ - - /* make a list of all objects in *h with head as last object */ - while ((word)p <= (word)last_object) { - /* current object's link points to last object */ - obj_link(p) = (ptr_t)prev; - prev = p; - p += sz; - } - p -= sz; /* p now points to last object */ - - /* Put p (which is now head of list of objects in *h) as first */ - /* pointer in the appropriate free list for this size. */ - *(ptr_t *)h = list; - return (ptr_t)p; -} - -/* Allocate a new heapblock for small objects of size gran granules. */ -/* Add all of the heapblock's objects to the free list for objects */ -/* of that size. Set all mark bits if objects are uncollectible. */ -/* Will fail to do anything if we are out of memory. */ -GC_INNER void GC_new_hblk(size_t gran, int kind) { - struct hblk *h; /* the new heap block */ - GC_bool clear = GC_obj_kinds[kind].ok_init; - - GC_STATIC_ASSERT((sizeof(struct hblk)) == HBLKSIZE); - GC_ASSERT(I_HOLD_LOCK()); - - if (GC_debugging_started) clear = TRUE; - - /* Allocate a new heap block */ - h = GC_allochblk(GRANULES_TO_BYTES(gran), kind, 0 /* flags */, 0); - if (EXPECT(NULL == h, FALSE)) return; - - /* Mark all objects if appropriate. */ - if (IS_UNCOLLECTABLE(kind)) GC_set_hdr_marks(HDR(h)); - - /* Build the free list */ - GC_obj_kinds[kind].ok_freelist[gran] = - GC_build_fl(h, GRANULES_TO_WORDS(gran), clear, - (ptr_t)GC_obj_kinds[kind].ok_freelist[gran]); -} diff --git a/vendor/bdwgc/obj_map.c b/vendor/bdwgc/obj_map.c deleted file mode 100644 index c38c02e0..00000000 --- a/vendor/bdwgc/obj_map.c +++ /dev/null @@ -1,89 +0,0 @@ -/* - * Copyright 1988, 1989 Hans-J. Boehm, Alan J. Demers - * Copyright (c) 1991, 1992 by Xerox Corporation. All rights reserved. - * Copyright (c) 1999-2001 by Hewlett-Packard Company. All rights reserved. - * - * THIS MATERIAL IS PROVIDED AS IS, WITH ABSOLUTELY NO WARRANTY EXPRESSED - * OR IMPLIED. ANY USE IS AT YOUR OWN RISK. - * - * Permission is hereby granted to use or copy this program - * for any purpose, provided the above notices are retained on all copies. - * Permission to modify the code and to distribute modified code is granted, - * provided the above notices are retained, and a notice that the code was - * modified is included with the above copyright notice. - */ - -#ifdef _WIN32 -#define WIN32_LEAN_AND_MEAN -#include -#endif -#include "private/gc_priv.h" - -/* Routines for maintaining maps describing heap block - * layouts for various object sizes. Allows fast pointer validity checks - * and fast location of object start locations on machines (such as SPARC) - * with slow division. - */ - -/* Consider pointers that are offset bytes displaced from the beginning */ -/* of an object to be valid. */ - -GC_API void GC_CALL GC_register_displacement(size_t offset) { - LOCK(); - GC_register_displacement_inner(offset); - UNLOCK(); -} - -GC_INNER void GC_register_displacement_inner(size_t offset) { - GC_ASSERT(I_HOLD_LOCK()); - if (offset >= VALID_OFFSET_SZ) { - ABORT("Bad argument to GC_register_displacement"); - } - if (!GC_valid_offsets[offset]) { - GC_valid_offsets[offset] = TRUE; - GC_modws_valid_offsets[offset % sizeof(word)] = TRUE; - } -} - -#ifdef MARK_BIT_PER_GRANULE -/* Add a heap block map for objects of size granules to obj_map. */ -/* A size of 0 is used for large objects. Return FALSE on failure. */ -GC_INNER GC_bool GC_add_map_entry(size_t granules) { - unsigned displ; - unsigned short *new_map; - - GC_ASSERT(I_HOLD_LOCK()); - if (granules > BYTES_TO_GRANULES(MAXOBJBYTES)) granules = 0; - if (GC_obj_map[granules] != 0) return TRUE; - - new_map = (unsigned short *)GC_scratch_alloc(OBJ_MAP_LEN * sizeof(short)); - if (EXPECT(NULL == new_map, FALSE)) return FALSE; - - GC_COND_LOG_PRINTF( - "Adding block map for size of %u granules (%u bytes)\n", - (unsigned)granules, (unsigned)GRANULES_TO_BYTES(granules)); - if (granules == 0) { - for (displ = 0; displ < OBJ_MAP_LEN; displ++) { - new_map[displ] = 1; /* Nonzero to get us out of marker fast path. */ - } - } else { - for (displ = 0; displ < OBJ_MAP_LEN; displ++) { - new_map[displ] = (unsigned short)(displ % granules); - } - } - GC_obj_map[granules] = new_map; - return TRUE; -} -#endif /* MARK_BIT_PER_GRANULE */ - -GC_INNER void GC_initialize_offsets(void) { - unsigned i; - if (GC_all_interior_pointers) { - for (i = 0; i < VALID_OFFSET_SZ; ++i) - GC_valid_offsets[i] = TRUE; - } else { - BZERO(GC_valid_offsets, sizeof(GC_valid_offsets)); - for (i = 0; i < sizeof(word); ++i) - GC_modws_valid_offsets[i] = FALSE; - } -} diff --git a/vendor/bdwgc/os_dep.c b/vendor/bdwgc/os_dep.c deleted file mode 100644 index b9745adc..00000000 --- a/vendor/bdwgc/os_dep.c +++ /dev/null @@ -1,5091 +0,0 @@ -/* - * Copyright 1988, 1989 Hans-J. Boehm, Alan J. Demers - * Copyright (c) 1991-1995 by Xerox Corporation. All rights reserved. - * Copyright (c) 1996-1999 by Silicon Graphics. All rights reserved. - * Copyright (c) 1999 by Hewlett-Packard Company. All rights reserved. - * Copyright (c) 2008-2022 Ivan Maidanski - * - * THIS MATERIAL IS PROVIDED AS IS, WITH ABSOLUTELY NO WARRANTY EXPRESSED - * OR IMPLIED. ANY USE IS AT YOUR OWN RISK. - * - * Permission is hereby granted to use or copy this program - * for any purpose, provided the above notices are retained on all copies. - * Permission to modify the code and to distribute modified code is granted, - * provided the above notices are retained, and a notice that the code was - * modified is included with the above copyright notice. - */ - -#ifdef _WIN32 -#define WIN32_LEAN_AND_MEAN -#include -#endif -#include "private/gc_priv.h" - -#if !defined(OS2) && !defined(PCR) && !defined(AMIGA) && !defined(MACOS) && !defined(MSWINCE) && !defined(SN_TARGET_ORBIS) && !defined(SN_TARGET_PSP2) && !defined(__CC_ARM) -#include -#if !defined(MSWIN32) && !defined(MSWIN_XBOX1) -#include -#endif -#endif - -#if defined(MSWINCE) || defined(SN_TARGET_PS3) -#define SIGSEGV 0 /* value is irrelevant */ -#else -#include -#endif - -#if defined(UNIX_LIKE) || defined(CYGWIN32) || defined(NACL) || defined(SYMBIAN) -#include -#endif - -#if defined(LINUX) || defined(LINUX_STACKBOTTOM) -#include -#endif - -/* Blatantly OS dependent routines, except for those that are related */ -/* to dynamic loading. */ - -#ifdef AMIGA -#define GC_AMIGA_DEF -#include "extra/AmigaOS.c" -#undef GC_AMIGA_DEF -#endif - -#ifdef MACOS -#include -#endif - -#ifdef IRIX5 -#include /* for locking */ -#include -#endif - -#if defined(MMAP_SUPPORTED) || defined(ADD_HEAP_GUARD_PAGES) -#if defined(USE_MUNMAP) && !defined(USE_MMAP) && !defined(CPPCHECK) -#error Invalid config: USE_MUNMAP requires USE_MMAP -#endif -#include -#include -#include -#endif - -#if defined(ADD_HEAP_GUARD_PAGES) || defined(LINUX_STACKBOTTOM) || defined(MMAP_SUPPORTED) || defined(NEED_PROC_MAPS) -#include -#endif - -#ifdef DARWIN -/* for get_etext and friends */ -#include -#endif - -#ifdef DJGPP -/* Apparently necessary for djgpp 2.01. May cause problems with */ -/* other versions. */ -typedef long unsigned int caddr_t; -#endif - -#ifdef PCR -#include "il/PCR_IL.h" -#include "mm/PCR_MM.h" -#include "th/PCR_ThCtl.h" -#endif - -#if !defined(NO_EXECUTE_PERMISSION) -STATIC GC_bool GC_pages_executable = TRUE; -#else -STATIC GC_bool GC_pages_executable = FALSE; -#endif -#define IGNORE_PAGES_EXECUTABLE 1 -/* Undefined on GC_pages_executable real use. */ - -#if ((defined(LINUX_STACKBOTTOM) || defined(NEED_PROC_MAPS) || defined(PROC_VDB) || defined(SOFT_VDB)) && !defined(PROC_READ)) || defined(CPPCHECK) -#define PROC_READ read -/* Should probably call the real read, if read is wrapped. */ -#endif - -#if defined(LINUX_STACKBOTTOM) || defined(NEED_PROC_MAPS) -/* Repeatedly perform a read call until the buffer is filled */ -/* up, or we encounter EOF or an error. */ -STATIC ssize_t GC_repeat_read(int fd, char *buf, size_t count) { - size_t num_read = 0; - - ASSERT_CANCEL_DISABLED(); - while (num_read < count) { - ssize_t result = PROC_READ(fd, buf + num_read, count - num_read); - - if (result < 0) return result; - if (result == 0) break; - num_read += result; - } - return num_read; -} -#endif /* LINUX_STACKBOTTOM || NEED_PROC_MAPS */ - -#ifdef NEED_PROC_MAPS -/* We need to parse /proc/self/maps, either to find dynamic libraries, */ -/* and/or to find the register backing store base (IA64). Do it once */ -/* here. */ - -#ifdef THREADS -/* Determine the length of a file by incrementally reading it into a */ -/* buffer. This would be silly to use it on a file supporting lseek, */ -/* but Linux /proc files usually do not. */ -/* As of Linux 4.15.0, lseek(SEEK_END) fails for /proc/self/maps. */ -STATIC size_t GC_get_file_len(int f) { - size_t total = 0; - ssize_t result; -#define GET_FILE_LEN_BUF_SZ 500 - char buf[GET_FILE_LEN_BUF_SZ]; - - do { - result = PROC_READ(f, buf, sizeof(buf)); - if (result == -1) return 0; - total += result; - } while (result > 0); - return total; -} - -STATIC size_t GC_get_maps_len(void) { - int f = open("/proc/self/maps", O_RDONLY); - size_t result; - if (f < 0) return 0; /* treat missing file as empty */ - result = GC_get_file_len(f); - close(f); - return result; -} -#endif /* THREADS */ - -/* Copy the contents of /proc/self/maps to a buffer in our address */ -/* space. Return the address of the buffer. */ -GC_INNER const char *GC_get_maps(void) { - ssize_t result; - static char *maps_buf = NULL; - static size_t maps_buf_sz = 1; - size_t maps_size; -#ifdef THREADS - size_t old_maps_size = 0; -#endif - - /* The buffer is essentially static, so there must be a single client. */ - GC_ASSERT(I_HOLD_LOCK()); - - /* Note that in the presence of threads, the maps file can */ - /* essentially shrink asynchronously and unexpectedly as */ - /* threads that we already think of as dead release their */ - /* stacks. And there is no easy way to read the entire */ - /* file atomically. This is arguably a misfeature of the */ - /* /proc/self/maps interface. */ - /* Since we expect the file can grow asynchronously in rare */ - /* cases, it should suffice to first determine */ - /* the size (using read), and then to reread the file. */ - /* If the size is inconsistent we have to retry. */ - /* This only matters with threads enabled, and if we use */ - /* this to locate roots (not the default). */ - -#ifdef THREADS - /* Determine the initial size of /proc/self/maps. */ - maps_size = GC_get_maps_len(); - if (0 == maps_size) - ABORT("Cannot determine length of /proc/self/maps"); -#else - maps_size = 4000; /* Guess */ -#endif - - /* Read /proc/self/maps, growing maps_buf as necessary. */ - /* Note that we may not allocate conventionally, and */ - /* thus can't use stdio. */ - do { - int f; - - while (maps_size >= maps_buf_sz) { -#ifdef LINT2 - /* Workaround passing tainted maps_buf to a tainted sink. */ - GC_noop1((word)maps_buf); -#else - GC_scratch_recycle_no_gww(maps_buf, maps_buf_sz); -#endif - /* Grow only by powers of 2, since we leak "too small" buffers.*/ - while (maps_size >= maps_buf_sz) maps_buf_sz *= 2; - maps_buf = GC_scratch_alloc(maps_buf_sz); - if (NULL == maps_buf) - ABORT_ARG1("Insufficient space for /proc/self/maps buffer", - ", %lu bytes requested", (unsigned long)maps_buf_sz); -#ifdef THREADS - /* Recompute initial length, since we allocated. */ - /* This can only happen a few times per program */ - /* execution. */ - maps_size = GC_get_maps_len(); - if (0 == maps_size) - ABORT("Cannot determine length of /proc/self/maps"); -#endif - } - GC_ASSERT(maps_buf_sz >= maps_size + 1); - f = open("/proc/self/maps", O_RDONLY); - if (-1 == f) - ABORT_ARG1("Cannot open /proc/self/maps", - ": errno= %d", errno); -#ifdef THREADS - old_maps_size = maps_size; -#endif - maps_size = 0; - do { - result = GC_repeat_read(f, maps_buf, maps_buf_sz - 1); - if (result < 0) { - ABORT_ARG1("Failed to read /proc/self/maps", - ": errno= %d", errno); - } - maps_size += result; - } while ((size_t)result == maps_buf_sz - 1); - close(f); - if (0 == maps_size) - ABORT("Empty /proc/self/maps"); -#ifdef THREADS - if (maps_size > old_maps_size) { - /* This might be caused by e.g. thread creation. */ - WARN( - "Unexpected asynchronous /proc/self/maps growth" - " (to %" WARN_PRIuPTR " bytes)\n", - maps_size); - } -#endif - } while (maps_size >= maps_buf_sz -#ifdef THREADS - || maps_size < old_maps_size -#endif - ); - maps_buf[maps_size] = '\0'; - return maps_buf; -} - -/* - * GC_parse_map_entry parses an entry from /proc/self/maps so we can - * locate all writable data segments that belong to shared libraries. - * The format of one of these entries and the fields we care about - * is as follows: - * XXXXXXXX-XXXXXXXX r-xp 00000000 30:05 260537 name of mapping...\n - * ^^^^^^^^ ^^^^^^^^ ^^^^ ^^ - * start end prot maj_dev - * - * Note that since about august 2003 kernels, the columns no longer have - * fixed offsets on 64-bit kernels. Hence we no longer rely on fixed offsets - * anywhere, which is safer anyway. - */ - -/* Assign various fields of the first line in maps_ptr to (*start), */ -/* (*end), (*prot), (*maj_dev) and (*mapping_name). mapping_name may */ -/* be NULL. (*prot) and (*mapping_name) are assigned pointers into the */ -/* original buffer. */ -#if (defined(DYNAMIC_LOADING) && defined(USE_PROC_FOR_LIBRARIES)) || defined(IA64) || defined(INCLUDE_LINUX_THREAD_DESCR) || (defined(REDIRECT_MALLOC) && defined(GC_LINUX_THREADS)) -GC_INNER const char *GC_parse_map_entry(const char *maps_ptr, - ptr_t *start, ptr_t *end, - const char **prot, unsigned *maj_dev, - const char **mapping_name) { - const unsigned char *start_start, *end_start, *maj_dev_start; - const unsigned char *p; /* unsigned for isspace, isxdigit */ - - if (maps_ptr == NULL || *maps_ptr == '\0') { - return NULL; - } - - p = (const unsigned char *)maps_ptr; - while (isspace(*p)) ++p; - start_start = p; - GC_ASSERT(isxdigit(*start_start)); - *start = (ptr_t)strtoul((const char *)start_start, (char **)&p, 16); - GC_ASSERT(*p == '-'); - - ++p; - end_start = p; - GC_ASSERT(isxdigit(*end_start)); - *end = (ptr_t)strtoul((const char *)end_start, (char **)&p, 16); - GC_ASSERT(isspace(*p)); - - while (isspace(*p)) ++p; - GC_ASSERT(*p == 'r' || *p == '-'); - *prot = (const char *)p; - /* Skip past protection field to offset field */ - while (!isspace(*p)) ++p; - while (isspace(*p)) p++; - GC_ASSERT(isxdigit(*p)); - /* Skip past offset field, which we ignore */ - while (!isspace(*p)) ++p; - while (isspace(*p)) p++; - maj_dev_start = p; - GC_ASSERT(isxdigit(*maj_dev_start)); - *maj_dev = strtoul((const char *)maj_dev_start, NULL, 16); - - if (mapping_name != NULL) { - while (*p && *p != '\n' && *p != '/' && *p != '[') p++; - *mapping_name = (const char *)p; - } - while (*p && *p++ != '\n') - ; - return (const char *)p; -} -#endif /* REDIRECT_MALLOC || DYNAMIC_LOADING || IA64 || ... */ - -#if defined(IA64) || defined(INCLUDE_LINUX_THREAD_DESCR) -/* Try to read the backing store base from /proc/self/maps. */ -/* Return the bounds of the writable mapping with a 0 major device, */ -/* which includes the address passed as data. */ -/* Return FALSE if there is no such mapping. */ -GC_INNER GC_bool GC_enclosing_mapping(ptr_t addr, ptr_t *startp, - ptr_t *endp) { - const char *prot; - ptr_t my_start, my_end; - unsigned int maj_dev; - const char *maps_ptr = GC_get_maps(); - - for (;;) { - maps_ptr = GC_parse_map_entry(maps_ptr, &my_start, &my_end, - &prot, &maj_dev, 0); - if (NULL == maps_ptr) break; - - if (prot[1] == 'w' && maj_dev == 0 && (word)my_end > (word)addr && (word)my_start <= (word)addr) { - *startp = my_start; - *endp = my_end; - return TRUE; - } - } - return FALSE; -} -#endif /* IA64 || INCLUDE_LINUX_THREAD_DESCR */ - -#if defined(REDIRECT_MALLOC) && defined(GC_LINUX_THREADS) -/* Find the text(code) mapping for the library whose name, after */ -/* stripping the directory part, starts with nm. */ -GC_INNER GC_bool GC_text_mapping(char *nm, ptr_t *startp, ptr_t *endp) { - size_t nm_len = strlen(nm); - const char *prot, *map_path; - ptr_t my_start, my_end; - unsigned int maj_dev; - const char *maps_ptr = GC_get_maps(); - - for (;;) { - maps_ptr = GC_parse_map_entry(maps_ptr, &my_start, &my_end, - &prot, &maj_dev, &map_path); - if (NULL == maps_ptr) break; - - if (prot[0] == 'r' && prot[1] == '-' && prot[2] == 'x') { - const char *p = map_path; - - /* Set p to point just past last slash, if any. */ - while (*p != '\0' && *p != '\n' && *p != ' ' && *p != '\t') ++p; - while (*p != '/' && (word)p >= (word)map_path) --p; - ++p; - if (strncmp(nm, p, nm_len) == 0) { - *startp = my_start; - *endp = my_end; - return TRUE; - } - } - } - return FALSE; -} -#endif /* REDIRECT_MALLOC */ - -#ifdef IA64 -static ptr_t backing_store_base_from_proc(void) { - ptr_t my_start, my_end; - - GC_ASSERT(I_HOLD_LOCK()); - if (!GC_enclosing_mapping(GC_save_regs_in_stack(), &my_start, &my_end)) { - GC_COND_LOG_PRINTF("Failed to find backing store base from /proc\n"); - return 0; - } - return my_start; -} -#endif - -#endif /* NEED_PROC_MAPS */ - -#if defined(SEARCH_FOR_DATA_START) -/* The x86 case can be handled without a search. The Alpha case */ -/* used to be handled differently as well, but the rules changed */ -/* for recent Linux versions. This seems to be the easiest way to */ -/* cover all versions. */ - -#if defined(LINUX) || defined(HURD) -/* Some Linux distributions arrange to define __data_start. Some */ -/* define data_start as a weak symbol. The latter is technically */ -/* broken, since the user program may define data_start, in which */ -/* case we lose. Nonetheless, we try both, preferring __data_start.*/ -/* We assume gcc-compatible pragmas. */ -EXTERN_C_BEGIN -#pragma weak __data_start -#pragma weak data_start -extern int __data_start[], data_start[]; -EXTERN_C_END -#elif defined(NETBSD) -EXTERN_C_BEGIN -extern char **environ; -EXTERN_C_END -#endif - -ptr_t GC_data_start = NULL; - -GC_INNER void GC_init_linux_data_start(void) { - ptr_t data_end = DATAEND; - -#if (defined(LINUX) || defined(HURD)) && defined(USE_PROG_DATA_START) - /* Try the easy approaches first: */ - /* However, this may lead to wrong data start value if libgc */ - /* code is put into a shared library (directly or indirectly) */ - /* which is linked with -Bsymbolic-functions option. Thus, */ - /* the following is not used by default. */ - if (COVERT_DATAFLOW(__data_start) != 0) { - GC_data_start = (ptr_t)(__data_start); - } else { - GC_data_start = (ptr_t)(data_start); - } - if (COVERT_DATAFLOW(GC_data_start) != 0) { - if ((word)GC_data_start > (word)data_end) - ABORT_ARG2("Wrong __data_start/_end pair", - ": %p .. %p", (void *)GC_data_start, (void *)data_end); - return; - } -#ifdef DEBUG_ADD_DEL_ROOTS - GC_log_printf("__data_start not provided\n"); -#endif -#endif /* LINUX */ - - if (GC_no_dls) { - /* Not needed, avoids the SIGSEGV caused by */ - /* GC_find_limit which complicates debugging. */ - GC_data_start = data_end; /* set data root size to 0 */ - return; - } - -#ifdef NETBSD - /* This may need to be environ, without the underscore, for */ - /* some versions. */ - GC_data_start = (ptr_t)GC_find_limit(&environ, FALSE); -#else - GC_data_start = (ptr_t)GC_find_limit(data_end, FALSE); -#endif -} -#endif /* SEARCH_FOR_DATA_START */ - -#ifdef ECOS - -#ifndef ECOS_GC_MEMORY_SIZE -#define ECOS_GC_MEMORY_SIZE (448 * 1024) -#endif /* ECOS_GC_MEMORY_SIZE */ - -/* TODO: This is a simple way of allocating memory which is */ -/* compatible with ECOS early releases. Later releases use a more */ -/* sophisticated means of allocating memory than this simple static */ -/* allocator, but this method is at least bound to work. */ -static char ecos_gc_memory[ECOS_GC_MEMORY_SIZE]; -static char *ecos_gc_brk = ecos_gc_memory; - -static void *tiny_sbrk(ptrdiff_t increment) { - void *p = ecos_gc_brk; - ecos_gc_brk += increment; - if ((word)ecos_gc_brk > (word)(ecos_gc_memory + sizeof(ecos_gc_memory))) { - ecos_gc_brk -= increment; - return NULL; - } - return p; -} -#define sbrk tiny_sbrk -#endif /* ECOS */ - -#if defined(ADDRESS_SANITIZER) && (defined(UNIX_LIKE) || defined(NEED_FIND_LIMIT) || defined(MPROTECT_VDB)) && !defined(CUSTOM_ASAN_DEF_OPTIONS) -/* To tell ASan to allow GC to use its own SIGBUS/SEGV handlers. */ -/* The function is exported just to be visible to ASan library. */ -GC_API const char *__asan_default_options(void) { - return "allow_user_segv_handler=1"; -} -#endif - -#ifdef OPENBSD -static struct sigaction old_segv_act; -STATIC JMP_BUF GC_jmp_buf_openbsd; - -STATIC void GC_fault_handler_openbsd(int sig) { - UNUSED_ARG(sig); - LONGJMP(GC_jmp_buf_openbsd, 1); -} - -static volatile int firstpass; - -/* Return first addressable location > p or bound. */ -STATIC ptr_t GC_skip_hole_openbsd(ptr_t p, ptr_t bound) { - static volatile ptr_t result; - - struct sigaction act; - word pgsz; - - GC_ASSERT(I_HOLD_LOCK()); - pgsz = (word)sysconf(_SC_PAGESIZE); - GC_ASSERT((word)bound >= pgsz); - - act.sa_handler = GC_fault_handler_openbsd; - sigemptyset(&act.sa_mask); - act.sa_flags = SA_NODEFER | SA_RESTART; - /* act.sa_restorer is deprecated and should not be initialized. */ - sigaction(SIGSEGV, &act, &old_segv_act); - - firstpass = 1; - result = (ptr_t)((word)p & ~(pgsz - 1)); - if (SETJMP(GC_jmp_buf_openbsd) != 0 || firstpass) { - firstpass = 0; - if ((word)result >= (word)bound - pgsz) { - result = bound; - } else { - result += pgsz; /* no overflow expected */ - GC_noop1((word)(*result)); - } - } - - sigaction(SIGSEGV, &old_segv_act, 0); - return result; -} -#endif /* OPENBSD */ - -#ifdef OS2 - -#include - -#if !defined(__IBMC__) && !defined(__WATCOMC__) /* e.g. EMX */ - -struct exe_hdr { - unsigned short magic_number; - unsigned short padding[29]; - long new_exe_offset; -}; - -#define E_MAGIC(x) (x).magic_number -#define EMAGIC 0x5A4D -#define E_LFANEW(x) (x).new_exe_offset - -struct e32_exe { - unsigned char magic_number[2]; - unsigned char byte_order; - unsigned char word_order; - unsigned long exe_format_level; - unsigned short cpu; - unsigned short os; - unsigned long padding1[13]; - unsigned long object_table_offset; - unsigned long object_count; - unsigned long padding2[31]; -}; - -#define E32_MAGIC1(x) (x).magic_number[0] -#define E32MAGIC1 'L' -#define E32_MAGIC2(x) (x).magic_number[1] -#define E32MAGIC2 'X' -#define E32_BORDER(x) (x).byte_order -#define E32LEBO 0 -#define E32_WORDER(x) (x).word_order -#define E32LEWO 0 -#define E32_CPU(x) (x).cpu -#define E32CPU286 1 -#define E32_OBJTAB(x) (x).object_table_offset -#define E32_OBJCNT(x) (x).object_count - -struct o32_obj { - unsigned long size; - unsigned long base; - unsigned long flags; - unsigned long pagemap; - unsigned long mapsize; - unsigned long reserved; -}; - -#define O32_FLAGS(x) (x).flags -#define OBJREAD 0x0001L -#define OBJWRITE 0x0002L -#define OBJINVALID 0x0080L -#define O32_SIZE(x) (x).size -#define O32_BASE(x) (x).base - -#else /* IBM's compiler */ - -/* A kludge to get around what appears to be a header file bug */ -#ifndef WORD -#define WORD unsigned short -#endif -#ifndef DWORD -#define DWORD unsigned long -#endif - -#define EXE386 1 -#include -#include - -#endif /* __IBMC__ */ - -#define INCL_DOSERRORS -#define INCL_DOSEXCEPTIONS -#define INCL_DOSFILEMGR -#define INCL_DOSMEMMGR -#define INCL_DOSMISC -#define INCL_DOSMODULEMGR -#define INCL_DOSPROCESS -#include - -#endif /* OS/2 */ - -/* Find the page size. */ -GC_INNER size_t GC_page_size = 0; -#ifdef REAL_PAGESIZE_NEEDED -GC_INNER size_t GC_real_page_size = 0; -#endif - -#if defined(MSWIN32) || defined(MSWINCE) || defined(CYGWIN32) -#ifndef VER_PLATFORM_WIN32_CE -#define VER_PLATFORM_WIN32_CE 3 -#endif - -#if defined(MSWINCE) && defined(THREADS) -GC_INNER GC_bool GC_dont_query_stack_min = FALSE; -#endif - -GC_INNER SYSTEM_INFO GC_sysinfo; - -GC_INNER void GC_setpagesize(void) { - GetSystemInfo(&GC_sysinfo); -#ifdef ALT_PAGESIZE_USED - /* Allocations made with mmap() are aligned to the allocation */ - /* granularity, which (at least on Win64) is not the same as the */ - /* page size. Probably we could distinguish the allocation */ - /* granularity from the actual page size, but in practice there */ - /* is no good reason to make allocations smaller than */ - /* dwAllocationGranularity, so we just use it instead of the */ - /* actual page size here (as Cygwin itself does in many cases). */ - GC_page_size = (size_t)GC_sysinfo.dwAllocationGranularity; -#ifdef REAL_PAGESIZE_NEEDED - GC_real_page_size = (size_t)GC_sysinfo.dwPageSize; - GC_ASSERT(GC_page_size >= GC_real_page_size); -#endif -#else - GC_page_size = (size_t)GC_sysinfo.dwPageSize; -#endif -#if defined(MSWINCE) && !defined(_WIN32_WCE_EMULATION) - { - OSVERSIONINFO verInfo; - /* Check the current WinCE version. */ - verInfo.dwOSVersionInfoSize = sizeof(OSVERSIONINFO); - if (!GetVersionEx(&verInfo)) - ABORT("GetVersionEx failed"); - if (verInfo.dwPlatformId == VER_PLATFORM_WIN32_CE && - verInfo.dwMajorVersion < 6) { - /* Only the first 32 MB of address space belongs to the */ - /* current process (unless WinCE 6.0+ or emulation). */ - GC_sysinfo.lpMaximumApplicationAddress = (LPVOID)((word)32 << 20); -#ifdef THREADS - /* On some old WinCE versions, it's observed that */ - /* VirtualQuery calls don't work properly when used to */ - /* get thread current stack committed minimum. */ - if (verInfo.dwMajorVersion < 5) - GC_dont_query_stack_min = TRUE; -#endif - } - } -#endif -} - -#ifndef CYGWIN32 -#define is_writable(prot) ((prot) == PAGE_READWRITE || (prot) == PAGE_WRITECOPY || (prot) == PAGE_EXECUTE_READWRITE || (prot) == PAGE_EXECUTE_WRITECOPY) -/* Return the number of bytes that are writable starting at p. */ -/* The pointer p is assumed to be page aligned. */ -/* If base is not 0, *base becomes the beginning of the */ -/* allocation region containing p. */ -STATIC word GC_get_writable_length(ptr_t p, ptr_t *base) { - MEMORY_BASIC_INFORMATION buf; - word result; - word protect; - - result = VirtualQuery(p, &buf, sizeof(buf)); - if (result != sizeof(buf)) ABORT("Weird VirtualQuery result"); - if (base != 0) *base = (ptr_t)(buf.AllocationBase); - protect = (buf.Protect & ~(PAGE_GUARD | PAGE_NOCACHE)); - if (!is_writable(protect) || buf.State != MEM_COMMIT) return 0; - return buf.RegionSize; -} - -/* Should not acquire the GC lock as it is used by GC_DllMain. */ -GC_API int GC_CALL GC_get_stack_base(struct GC_stack_base *sb) { - ptr_t trunc_sp; - word size; - - /* Set page size if it is not ready (so client can use this */ - /* function even before GC is initialized). */ - if (!GC_page_size) GC_setpagesize(); - - trunc_sp = (ptr_t)((word)GC_approx_sp() & ~(GC_page_size - 1)); - /* FIXME: This won't work if called from a deeply recursive */ - /* client code (and the committed stack space has grown). */ - size = GC_get_writable_length(trunc_sp, 0); - GC_ASSERT(size != 0); - sb->mem_base = trunc_sp + size; - return GC_SUCCESS; -} -#else /* CYGWIN32 */ -/* An alternate version for Cygwin (adapted from Dave Korn's */ -/* gcc version of boehm-gc). */ -GC_API int GC_CALL GC_get_stack_base(struct GC_stack_base *sb) { -#ifdef X86_64 - sb->mem_base = ((NT_TIB *)NtCurrentTeb())->StackBase; -#else - void *_tlsbase; - - __asm__("movl %%fs:4, %0" - : "=r"(_tlsbase)); - sb->mem_base = _tlsbase; -#endif - return GC_SUCCESS; -} -#endif /* CYGWIN32 */ -#define HAVE_GET_STACK_BASE - -#else /* !MSWIN32 */ - -#ifdef OS2 -static int os2_getpagesize(void) { - ULONG result[1]; - - if (DosQuerySysInfo(QSV_PAGE_SIZE, QSV_PAGE_SIZE, - (void *)result, sizeof(ULONG)) != NO_ERROR) { - WARN("DosQuerySysInfo failed\n", 0); - result[0] = 4096; - } - return (int)result[0]; -} -#endif /* OS2 */ - -GC_INNER void GC_setpagesize(void) { -#ifdef ALT_PAGESIZE_USED -#ifdef REAL_PAGESIZE_NEEDED - GC_real_page_size = (size_t)GETPAGESIZE(); -#endif - /* It's acceptable to fake it. */ - GC_page_size = HBLKSIZE; -#else - GC_page_size = (size_t)GETPAGESIZE(); -#if !defined(CPPCHECK) - if (0 == GC_page_size) - ABORT("getpagesize failed"); -#endif -#endif -} -#endif /* !MSWIN32 */ - -#ifdef HAIKU -#include - -GC_API int GC_CALL GC_get_stack_base(struct GC_stack_base *sb) { - thread_info th; - get_thread_info(find_thread(NULL), &th); - sb->mem_base = th.stack_end; - return GC_SUCCESS; -} -#define HAVE_GET_STACK_BASE -#endif /* HAIKU */ - -#ifdef OS2 -GC_API int GC_CALL GC_get_stack_base(struct GC_stack_base *sb) { - PTIB ptib; /* thread information block */ - PPIB ppib; - if (DosGetInfoBlocks(&ptib, &ppib) != NO_ERROR) { - WARN("DosGetInfoBlocks failed\n", 0); - return GC_UNIMPLEMENTED; - } - sb->mem_base = ptib->tib_pstacklimit; - return GC_SUCCESS; -} -#define HAVE_GET_STACK_BASE -#endif /* OS2 */ - -#ifdef AMIGA -#define GC_AMIGA_SB -#include "extra/AmigaOS.c" -#undef GC_AMIGA_SB -#define GET_MAIN_STACKBASE_SPECIAL -#endif /* AMIGA */ - -#if defined(NEED_FIND_LIMIT) || defined(UNIX_LIKE) || (defined(WRAP_MARK_SOME) && defined(NO_SEH_AVAILABLE)) - -typedef void (*GC_fault_handler_t)(int); - -#ifdef USE_SEGV_SIGACT -#ifndef OPENBSD -static struct sigaction old_segv_act; -#endif -#ifdef USE_BUS_SIGACT -static struct sigaction old_bus_act; -#endif -#else -static GC_fault_handler_t old_segv_hand; -#ifdef HAVE_SIGBUS -static GC_fault_handler_t old_bus_hand; -#endif -#endif /* !USE_SEGV_SIGACT */ - -GC_INNER void GC_set_and_save_fault_handler(GC_fault_handler_t h) { -#ifdef USE_SEGV_SIGACT - struct sigaction act; - - act.sa_handler = h; -#ifdef SIGACTION_FLAGS_NODEFER_HACK - /* Was necessary for Solaris 2.3 and very temporary */ - /* NetBSD bugs. */ - act.sa_flags = SA_RESTART | SA_NODEFER; -#else - act.sa_flags = SA_RESTART; -#endif - - (void)sigemptyset(&act.sa_mask); - /* act.sa_restorer is deprecated and should not be initialized. */ -#ifdef GC_IRIX_THREADS - /* Older versions have a bug related to retrieving and */ - /* and setting a handler at the same time. */ - (void)sigaction(SIGSEGV, 0, &old_segv_act); - (void)sigaction(SIGSEGV, &act, 0); -#else - (void)sigaction(SIGSEGV, &act, &old_segv_act); -#ifdef USE_BUS_SIGACT - /* Pthreads doesn't exist under Irix 5.x, so we */ - /* don't have to worry in the threads case. */ - (void)sigaction(SIGBUS, &act, &old_bus_act); -#endif -#endif /* !GC_IRIX_THREADS */ -#else - old_segv_hand = signal(SIGSEGV, h); -#ifdef HAVE_SIGBUS - old_bus_hand = signal(SIGBUS, h); -#endif -#endif /* !USE_SEGV_SIGACT */ -#if defined(CPPCHECK) && defined(ADDRESS_SANITIZER) - GC_noop1((word)&__asan_default_options); -#endif -} -#endif /* NEED_FIND_LIMIT || UNIX_LIKE */ - -#if defined(NEED_FIND_LIMIT) || (defined(WRAP_MARK_SOME) && defined(NO_SEH_AVAILABLE)) || (defined(USE_PROC_FOR_LIBRARIES) && defined(THREADS)) -GC_INNER JMP_BUF GC_jmp_buf; - -STATIC void GC_fault_handler(int sig) { - UNUSED_ARG(sig); - LONGJMP(GC_jmp_buf, 1); -} - -GC_INNER void GC_setup_temporary_fault_handler(void) { - /* Handler is process-wide, so this should only happen in */ - /* one thread at a time. */ - GC_ASSERT(I_HOLD_LOCK()); - GC_set_and_save_fault_handler(GC_fault_handler); -} - -GC_INNER void GC_reset_fault_handler(void) { -#ifdef USE_SEGV_SIGACT - (void)sigaction(SIGSEGV, &old_segv_act, 0); -#ifdef USE_BUS_SIGACT - (void)sigaction(SIGBUS, &old_bus_act, 0); -#endif -#else - (void)signal(SIGSEGV, old_segv_hand); -#ifdef HAVE_SIGBUS - (void)signal(SIGBUS, old_bus_hand); -#endif -#endif -} -#endif /* NEED_FIND_LIMIT || USE_PROC_FOR_LIBRARIES || WRAP_MARK_SOME */ - -#if defined(NEED_FIND_LIMIT) || (defined(USE_PROC_FOR_LIBRARIES) && defined(THREADS)) -#define MIN_PAGE_SIZE 256 /* Smallest conceivable page size, in bytes. */ - -/* Return the first non-addressable location > p (up) or */ -/* the smallest location q s.t. [q,p) is addressable (!up). */ -/* We assume that p (up) or p-1 (!up) is addressable. */ -GC_ATTR_NO_SANITIZE_ADDR -STATIC ptr_t GC_find_limit_with_bound(ptr_t p, GC_bool up, ptr_t bound) { - static volatile ptr_t result; - /* Safer if static, since otherwise it may not be */ - /* preserved across the longjmp. Can safely be */ - /* static since it's only called with the */ - /* allocation lock held. */ - - GC_ASSERT(up ? (word)bound >= MIN_PAGE_SIZE - : (word)bound <= ~(word)MIN_PAGE_SIZE); - GC_ASSERT(I_HOLD_LOCK()); - GC_setup_temporary_fault_handler(); - if (SETJMP(GC_jmp_buf) == 0) { - result = (ptr_t)(((word)(p)) & ~(MIN_PAGE_SIZE - 1)); - for (;;) { - if (up) { - if ((word)result >= (word)bound - MIN_PAGE_SIZE) { - result = bound; - break; - } - result += MIN_PAGE_SIZE; /* no overflow expected */ - } else { - if ((word)result <= (word)bound + MIN_PAGE_SIZE) { - result = bound - MIN_PAGE_SIZE; - /* This is to compensate */ - /* further result increment (we */ - /* do not modify "up" variable */ - /* since it might be clobbered */ - /* by setjmp otherwise). */ - break; - } - result -= MIN_PAGE_SIZE; /* no underflow expected */ - } - GC_noop1((word)(*result)); - } - } - GC_reset_fault_handler(); - if (!up) { - result += MIN_PAGE_SIZE; - } - return result; -} - -void *GC_find_limit(void *p, int up) { - return GC_find_limit_with_bound((ptr_t)p, (GC_bool)up, - up ? (ptr_t)GC_WORD_MAX : 0); -} -#endif /* NEED_FIND_LIMIT || USE_PROC_FOR_LIBRARIES */ - -#ifdef HPUX_MAIN_STACKBOTTOM -#include -#include - -STATIC ptr_t GC_hpux_main_stack_base(void) { - struct pst_vm_status vm_status; - int i = 0; - - while (pstat_getprocvm(&vm_status, sizeof(vm_status), 0, i++) == 1) { - if (vm_status.pst_type == PS_STACK) - return (ptr_t)vm_status.pst_vaddr; - } - - /* Old way to get the stack bottom. */ -#ifdef STACK_GROWS_UP - return (ptr_t)GC_find_limit(GC_approx_sp(), /* up= */ FALSE); -#else /* not HP_PA */ - return (ptr_t)GC_find_limit(GC_approx_sp(), TRUE); -#endif -} -#endif /* HPUX_MAIN_STACKBOTTOM */ - -#ifdef HPUX_STACKBOTTOM - -#include -#include - -GC_INNER ptr_t GC_get_register_stack_base(void) { - struct pst_vm_status vm_status; - - int i = 0; - while (pstat_getprocvm(&vm_status, sizeof(vm_status), 0, i++) == 1) { - if (vm_status.pst_type == PS_RSESTACK) { - return (ptr_t)vm_status.pst_vaddr; - } - } - - /* old way to get the register stackbottom */ - GC_ASSERT(GC_stackbottom != NULL); - return (ptr_t)(((word)GC_stackbottom - BACKING_STORE_DISPLACEMENT - 1) & ~(BACKING_STORE_ALIGNMENT - 1)); -} - -#endif /* HPUX_STACK_BOTTOM */ - -#ifdef LINUX_STACKBOTTOM - -#include -#include - -#define STAT_SKIP 27 /* Number of fields preceding startstack */ - /* field in /proc/self/stat */ - -#ifdef USE_LIBC_PRIVATES -EXTERN_C_BEGIN -#pragma weak __libc_stack_end -extern ptr_t __libc_stack_end; -#ifdef IA64 -#pragma weak __libc_ia64_register_backing_store_base -extern ptr_t __libc_ia64_register_backing_store_base; -#endif -EXTERN_C_END -#endif - -#ifdef IA64 -GC_INNER ptr_t GC_get_register_stack_base(void) { - ptr_t result; - - GC_ASSERT(I_HOLD_LOCK()); -#ifdef USE_LIBC_PRIVATES - if (0 != &__libc_ia64_register_backing_store_base && 0 != __libc_ia64_register_backing_store_base) { - /* glibc 2.2.4 has a bug such that for dynamically linked */ - /* executables __libc_ia64_register_backing_store_base is */ - /* defined but uninitialized during constructor calls. */ - /* Hence we check for both nonzero address and value. */ - return __libc_ia64_register_backing_store_base; - } -#endif - result = backing_store_base_from_proc(); - if (0 == result) { - result = (ptr_t)GC_find_limit(GC_save_regs_in_stack(), FALSE); - /* This works better than a constant displacement heuristic. */ - } - return result; -} -#endif /* IA64 */ - -STATIC ptr_t GC_linux_main_stack_base(void) { - /* We read the stack bottom value from /proc/self/stat. We do this */ - /* using direct I/O system calls in order to avoid calling malloc */ - /* in case REDIRECT_MALLOC is defined. */ -#define STAT_BUF_SIZE 4096 - char stat_buf[STAT_BUF_SIZE]; - int f; - word result; - ssize_t i, buf_offset = 0, len; - - /* First try the easy way. This should work for glibc 2.2. */ - /* This fails in a prelinked ("prelink" command) executable */ - /* since the correct value of __libc_stack_end never */ - /* becomes visible to us. The second test works around */ - /* this. */ -#ifdef USE_LIBC_PRIVATES - if (0 != &__libc_stack_end && 0 != __libc_stack_end) { -#if defined(IA64) - /* Some versions of glibc set the address 16 bytes too */ - /* low while the initialization code is running. */ - if (((word)__libc_stack_end & 0xfff) + 0x10 < 0x1000) { - return __libc_stack_end + 0x10; - } /* Otherwise it's not safe to add 16 bytes and we fall */ - /* back to using /proc. */ -#elif defined(SPARC) - /* Older versions of glibc for 64-bit SPARC do not set this */ - /* variable correctly, it gets set to either zero or one. */ - if (__libc_stack_end != (ptr_t)(unsigned long)0x1) - return __libc_stack_end; -#else - return __libc_stack_end; -#endif - } -#endif - - f = open("/proc/self/stat", O_RDONLY); - if (-1 == f) - ABORT_ARG1("Could not open /proc/self/stat", ": errno= %d", errno); - len = GC_repeat_read(f, stat_buf, sizeof(stat_buf)); - if (len < 0) - ABORT_ARG1("Failed to read /proc/self/stat", - ": errno= %d", errno); - close(f); - - /* Skip the required number of fields. This number is hopefully */ - /* constant across all Linux implementations. */ - for (i = 0; i < STAT_SKIP; ++i) { - while (buf_offset < len && isspace(stat_buf[buf_offset++])) { - /* empty */ - } - while (buf_offset < len && !isspace(stat_buf[buf_offset++])) { - /* empty */ - } - } - /* Skip spaces. */ - while (buf_offset < len && isspace(stat_buf[buf_offset])) { - buf_offset++; - } - /* Find the end of the number and cut the buffer there. */ - for (i = 0; buf_offset + i < len; i++) { - if (!isdigit(stat_buf[buf_offset + i])) break; - } - if (buf_offset + i >= len) ABORT("Could not parse /proc/self/stat"); - stat_buf[buf_offset + i] = '\0'; - - result = (word)STRTOULL(&stat_buf[buf_offset], NULL, 10); - if (result < 0x100000 || (result & (sizeof(word) - 1)) != 0) - ABORT_ARG1("Absurd stack bottom value", - ": 0x%lx", (unsigned long)result); - return (ptr_t)result; -} -#endif /* LINUX_STACKBOTTOM */ - -#ifdef QNX_STACKBOTTOM -STATIC ptr_t GC_qnx_main_stack_base(void) { - /* TODO: this approach is not very exact but it works for the */ - /* tests, at least, unlike other available heuristics. */ - return (ptr_t)__builtin_frame_address(0); -} -#endif /* QNX_STACKBOTTOM */ - -#ifdef FREEBSD_STACKBOTTOM -/* This uses an undocumented sysctl call, but at least one expert */ -/* believes it will stay. */ - -#include -#include -#include - -STATIC ptr_t GC_freebsd_main_stack_base(void) { - int nm[2] = {CTL_KERN, KERN_USRSTACK}; - ptr_t base; - size_t len = sizeof(ptr_t); - int r = sysctl(nm, 2, &base, &len, NULL, 0); - if (r) ABORT("Error getting main stack base"); - return base; -} -#endif /* FREEBSD_STACKBOTTOM */ - -#if defined(ECOS) || defined(NOSYS) -ptr_t GC_get_main_stack_base(void) { - return STACKBOTTOM; -} -#define GET_MAIN_STACKBASE_SPECIAL -#elif defined(SYMBIAN) -EXTERN_C_BEGIN -extern int GC_get_main_symbian_stack_base(void); -EXTERN_C_END - -ptr_t GC_get_main_stack_base(void) { - return (ptr_t)GC_get_main_symbian_stack_base(); -} -#define GET_MAIN_STACKBASE_SPECIAL -#elif defined(EMSCRIPTEN) -#include - -ptr_t GC_get_main_stack_base(void) { - return (ptr_t)emscripten_stack_get_base(); -} -#define GET_MAIN_STACKBASE_SPECIAL -#elif !defined(AMIGA) && !defined(HAIKU) && !defined(OS2) && !defined(MSWIN32) && !defined(MSWINCE) && !defined(CYGWIN32) && !defined(GC_OPENBSD_THREADS) && (!defined(GC_SOLARIS_THREADS) || defined(_STRICT_STDC)) - -#if (defined(HAVE_PTHREAD_ATTR_GET_NP) || defined(HAVE_PTHREAD_GETATTR_NP)) && (defined(THREADS) || defined(USE_GET_STACKBASE_FOR_MAIN)) -#include -#ifdef HAVE_PTHREAD_NP_H -#include /* for pthread_attr_get_np() */ -#endif -#elif defined(DARWIN) && !defined(NO_PTHREAD_GET_STACKADDR_NP) -/* We could use pthread_get_stackaddr_np even in case of a */ -/* single-threaded gclib (there is no -lpthread on Darwin). */ -#include -#undef STACKBOTTOM -#define STACKBOTTOM (ptr_t) pthread_get_stackaddr_np(pthread_self()) -#endif - -ptr_t GC_get_main_stack_base(void) { - ptr_t result; -#if (defined(HAVE_PTHREAD_ATTR_GET_NP) || defined(HAVE_PTHREAD_GETATTR_NP)) && (defined(USE_GET_STACKBASE_FOR_MAIN) || (defined(THREADS) && !defined(REDIRECT_MALLOC))) - pthread_attr_t attr; - void *stackaddr; - size_t size; - -#ifdef HAVE_PTHREAD_ATTR_GET_NP - if (pthread_attr_init(&attr) == 0 && (pthread_attr_get_np(pthread_self(), &attr) == 0 - ? TRUE - : (pthread_attr_destroy(&attr), FALSE))) -#else /* HAVE_PTHREAD_GETATTR_NP */ - if (pthread_getattr_np(pthread_self(), &attr) == 0) -#endif - { - if (pthread_attr_getstack(&attr, &stackaddr, &size) == 0 && stackaddr != NULL) { - (void)pthread_attr_destroy(&attr); -#ifdef STACK_GROWS_DOWN - stackaddr = (char *)stackaddr + size; -#endif - return (ptr_t)stackaddr; - } - (void)pthread_attr_destroy(&attr); - } - WARN( - "pthread_getattr_np or pthread_attr_getstack failed" - " for main thread\n", - 0); -#endif -#ifdef STACKBOTTOM - result = STACKBOTTOM; -#else -#ifdef HEURISTIC1 -#define STACKBOTTOM_ALIGNMENT_M1 ((word)STACK_GRAN - 1) -#ifdef STACK_GROWS_DOWN - result = (ptr_t)(((word)GC_approx_sp() + STACKBOTTOM_ALIGNMENT_M1) & ~STACKBOTTOM_ALIGNMENT_M1); -#else - result = (ptr_t)((word)GC_approx_sp() & ~STACKBOTTOM_ALIGNMENT_M1); -#endif -#elif defined(HPUX_MAIN_STACKBOTTOM) - result = GC_hpux_main_stack_base(); -#elif defined(LINUX_STACKBOTTOM) - result = GC_linux_main_stack_base(); -#elif defined(QNX_STACKBOTTOM) - result = GC_qnx_main_stack_base(); -#elif defined(FREEBSD_STACKBOTTOM) - result = GC_freebsd_main_stack_base(); -#elif defined(HEURISTIC2) - { - ptr_t sp = GC_approx_sp(); -#ifdef STACK_GROWS_DOWN - result = (ptr_t)GC_find_limit(sp, TRUE); -#if defined(HEURISTIC2_LIMIT) && !defined(CPPCHECK) - if ((word)result > (word)HEURISTIC2_LIMIT && (word)sp < (word)HEURISTIC2_LIMIT) { - result = HEURISTIC2_LIMIT; - } -#endif -#else - result = (ptr_t)GC_find_limit(sp, FALSE); -#if defined(HEURISTIC2_LIMIT) && !defined(CPPCHECK) - if ((word)result < (word)HEURISTIC2_LIMIT && (word)sp > (word)HEURISTIC2_LIMIT) { - result = HEURISTIC2_LIMIT; - } -#endif -#endif - } -#elif defined(STACK_NOT_SCANNED) || defined(CPPCHECK) - result = NULL; -#else -#error None of HEURISTIC* and *STACKBOTTOM defined! -#endif -#if defined(STACK_GROWS_DOWN) && !defined(CPPCHECK) - if (NULL == result) - result = (ptr_t)(signed_word)(-sizeof(ptr_t)); -#endif -#endif -#if !defined(CPPCHECK) - GC_ASSERT((word)GC_approx_sp() HOTTER_THAN(word) result); -#endif - return result; -} -#define GET_MAIN_STACKBASE_SPECIAL -#endif /* !AMIGA && !HAIKU && !GC_OPENBSD_THREADS && !OS2 && !Windows */ - -#if (defined(HAVE_PTHREAD_ATTR_GET_NP) || defined(HAVE_PTHREAD_GETATTR_NP)) && defined(THREADS) && !defined(HAVE_GET_STACK_BASE) -#include -#ifdef HAVE_PTHREAD_NP_H -#include -#endif - -GC_API int GC_CALL GC_get_stack_base(struct GC_stack_base *b) { - pthread_attr_t attr; - size_t size; - -#ifdef HAVE_PTHREAD_ATTR_GET_NP - if (pthread_attr_init(&attr) != 0) - ABORT("pthread_attr_init failed"); - if (pthread_attr_get_np(pthread_self(), &attr) != 0) { - WARN("pthread_attr_get_np failed\n", 0); - (void)pthread_attr_destroy(&attr); - return GC_UNIMPLEMENTED; - } -#else /* HAVE_PTHREAD_GETATTR_NP */ - if (pthread_getattr_np(pthread_self(), &attr) != 0) { - WARN("pthread_getattr_np failed\n", 0); - return GC_UNIMPLEMENTED; - } -#endif - if (pthread_attr_getstack(&attr, &(b->mem_base), &size) != 0) { - ABORT("pthread_attr_getstack failed"); - } - (void)pthread_attr_destroy(&attr); -#ifdef STACK_GROWS_DOWN - b->mem_base = (char *)(b->mem_base) + size; -#endif -#ifdef IA64 - /* We could try backing_store_base_from_proc, but that's safe */ - /* only if no mappings are being asynchronously created. */ - /* Subtracting the size from the stack base doesn't work for at */ - /* least the main thread. */ - LOCK(); - { - IF_CANCEL(int cancel_state;) - ptr_t bsp; - ptr_t next_stack; - - DISABLE_CANCEL(cancel_state); - bsp = GC_save_regs_in_stack(); - next_stack = GC_greatest_stack_base_below(bsp); - if (0 == next_stack) { - b->reg_base = GC_find_limit(bsp, FALSE); - } else { - /* Avoid walking backwards into preceding memory stack and */ - /* growing it. */ - b->reg_base = GC_find_limit_with_bound(bsp, FALSE, next_stack); - } - RESTORE_CANCEL(cancel_state); - } - UNLOCK(); -#elif defined(E2K) - b->reg_base = NULL; -#endif - return GC_SUCCESS; -} -#define HAVE_GET_STACK_BASE -#endif /* THREADS && (HAVE_PTHREAD_ATTR_GET_NP || HAVE_PTHREAD_GETATTR_NP) */ - -#if defined(GC_DARWIN_THREADS) && !defined(NO_PTHREAD_GET_STACKADDR_NP) -#include - -GC_API int GC_CALL GC_get_stack_base(struct GC_stack_base *b) { - /* pthread_get_stackaddr_np() should return stack bottom (highest */ - /* stack address plus 1). */ - b->mem_base = pthread_get_stackaddr_np(pthread_self()); - GC_ASSERT((word)GC_approx_sp() HOTTER_THAN(word) b->mem_base); - return GC_SUCCESS; -} -#define HAVE_GET_STACK_BASE -#endif /* GC_DARWIN_THREADS */ - -#ifdef GC_OPENBSD_THREADS -#include -#include -#include - -/* Find the stack using pthread_stackseg_np(). */ -GC_API int GC_CALL GC_get_stack_base(struct GC_stack_base *sb) { - stack_t stack; - if (pthread_stackseg_np(pthread_self(), &stack)) - ABORT("pthread_stackseg_np(self) failed"); - sb->mem_base = stack.ss_sp; - return GC_SUCCESS; -} -#define HAVE_GET_STACK_BASE -#endif /* GC_OPENBSD_THREADS */ - -#if defined(GC_SOLARIS_THREADS) && !defined(_STRICT_STDC) - -#include -#include -#include - -/* These variables are used to cache ss_sp value for the primordial */ -/* thread (it's better not to call thr_stksegment() twice for this */ -/* thread - see JDK bug #4352906). */ -static pthread_t stackbase_main_self = 0; -/* 0 means stackbase_main_ss_sp value is unset. */ -static void *stackbase_main_ss_sp = NULL; - -GC_API int GC_CALL GC_get_stack_base(struct GC_stack_base *b) { - stack_t s; - pthread_t self = pthread_self(); - - if (self == stackbase_main_self) { - /* If the client calls GC_get_stack_base() from the main thread */ - /* then just return the cached value. */ - b->mem_base = stackbase_main_ss_sp; - GC_ASSERT(b->mem_base != NULL); - return GC_SUCCESS; - } - - if (thr_stksegment(&s)) { - /* According to the manual, the only failure error code returned */ - /* is EAGAIN meaning "the information is not available due to the */ - /* thread is not yet completely initialized or it is an internal */ - /* thread" - this shouldn't happen here. */ - ABORT("thr_stksegment failed"); - } - /* s.ss_sp holds the pointer to the stack bottom. */ - GC_ASSERT((word)GC_approx_sp() HOTTER_THAN(word) s.ss_sp); - - if (!stackbase_main_self && thr_main() != 0) { - /* Cache the stack bottom pointer for the primordial thread */ - /* (this is done during GC_init, so there is no race). */ - stackbase_main_ss_sp = s.ss_sp; - stackbase_main_self = self; - } - - b->mem_base = s.ss_sp; - return GC_SUCCESS; -} -#define HAVE_GET_STACK_BASE -#endif /* GC_SOLARIS_THREADS */ - -#ifdef GC_RTEMS_PTHREADS -GC_API int GC_CALL GC_get_stack_base(struct GC_stack_base *sb) { - sb->mem_base = rtems_get_stack_bottom(); - return GC_SUCCESS; -} -#define HAVE_GET_STACK_BASE -#endif /* GC_RTEMS_PTHREADS */ - -#ifndef HAVE_GET_STACK_BASE -#ifdef NEED_FIND_LIMIT -/* Retrieve the stack bottom. */ -/* Using the GC_find_limit version is risky. */ -/* On IA64, for example, there is no guard page between the */ -/* stack of one thread and the register backing store of the */ -/* next. Thus this is likely to identify way too large a */ -/* "stack" and thus at least result in disastrous performance. */ -/* TODO: Implement better strategies here. */ -GC_API int GC_CALL GC_get_stack_base(struct GC_stack_base *b) { - IF_CANCEL(int cancel_state;) - - LOCK(); - DISABLE_CANCEL(cancel_state); /* May be unnecessary? */ -#ifdef STACK_GROWS_DOWN - b->mem_base = GC_find_limit(GC_approx_sp(), TRUE); -#else - b->mem_base = GC_find_limit(GC_approx_sp(), FALSE); -#endif -#ifdef IA64 - b->reg_base = GC_find_limit(GC_save_regs_in_stack(), FALSE); -#elif defined(E2K) - b->reg_base = NULL; -#endif - RESTORE_CANCEL(cancel_state); - UNLOCK(); - return GC_SUCCESS; -} -#else -GC_API int GC_CALL GC_get_stack_base(struct GC_stack_base *b) { -#if defined(GET_MAIN_STACKBASE_SPECIAL) && !defined(THREADS) && !defined(IA64) - b->mem_base = GC_get_main_stack_base(); - return GC_SUCCESS; -#else - UNUSED_ARG(b); - return GC_UNIMPLEMENTED; -#endif -} -#endif /* !NEED_FIND_LIMIT */ -#endif /* !HAVE_GET_STACK_BASE */ - -#ifndef GET_MAIN_STACKBASE_SPECIAL -/* This is always called from the main thread. Default implementation. */ -ptr_t GC_get_main_stack_base(void) { - struct GC_stack_base sb; - - if (GC_get_stack_base(&sb) != GC_SUCCESS) - ABORT("GC_get_stack_base failed"); - GC_ASSERT((word)GC_approx_sp() HOTTER_THAN(word) sb.mem_base); - return (ptr_t)sb.mem_base; -} -#endif /* !GET_MAIN_STACKBASE_SPECIAL */ - -/* Register static data segment(s) as roots. If more data segments are */ -/* added later then they need to be registered at that point (as we do */ -/* with SunOS dynamic loading), or GC_mark_roots needs to check for */ -/* them (as we do with PCR). */ -#ifdef OS2 - -void GC_register_data_segments(void) { - PTIB ptib; - PPIB ppib; - HMODULE module_handle; -#define PBUFSIZ 512 - UCHAR path[PBUFSIZ]; - FILE *myexefile; - struct exe_hdr hdrdos; /* MSDOS header. */ - struct e32_exe hdr386; /* Real header for my executable */ - struct o32_obj seg; /* Current segment */ - int nsegs; - -#if defined(CPPCHECK) - hdrdos.padding[0] = 0; /* to prevent "field unused" warnings */ - hdr386.exe_format_level = 0; - hdr386.os = 0; - hdr386.padding1[0] = 0; - hdr386.padding2[0] = 0; - seg.pagemap = 0; - seg.mapsize = 0; - seg.reserved = 0; -#endif - if (DosGetInfoBlocks(&ptib, &ppib) != NO_ERROR) { - ABORT("DosGetInfoBlocks failed"); - } - module_handle = ppib->pib_hmte; - if (DosQueryModuleName(module_handle, PBUFSIZ, path) != NO_ERROR) { - ABORT("DosQueryModuleName failed"); - } - myexefile = fopen(path, "rb"); - if (myexefile == 0) { - ABORT_ARG1("Failed to open executable", ": %s", path); - } - if (fread((char *)(&hdrdos), 1, sizeof(hdrdos), myexefile) < sizeof(hdrdos)) { - ABORT_ARG1("Could not read MSDOS header", " from: %s", path); - } - if (E_MAGIC(hdrdos) != EMAGIC) { - ABORT_ARG1("Bad DOS magic number", " in file: %s", path); - } - if (fseek(myexefile, E_LFANEW(hdrdos), SEEK_SET) != 0) { - ABORT_ARG1("Bad DOS magic number", " in file: %s", path); - } - if (fread((char *)(&hdr386), 1, sizeof(hdr386), myexefile) < sizeof(hdr386)) { - ABORT_ARG1("Could not read OS/2 header", " from: %s", path); - } - if (E32_MAGIC1(hdr386) != E32MAGIC1 || E32_MAGIC2(hdr386) != E32MAGIC2) { - ABORT_ARG1("Bad OS/2 magic number", " in file: %s", path); - } - if (E32_BORDER(hdr386) != E32LEBO || E32_WORDER(hdr386) != E32LEWO) { - ABORT_ARG1("Bad byte order in executable", " file: %s", path); - } - if (E32_CPU(hdr386) == E32CPU286) { - ABORT_ARG1("GC cannot handle 80286 executables", ": %s", path); - } - if (fseek(myexefile, E_LFANEW(hdrdos) + E32_OBJTAB(hdr386), - SEEK_SET) != 0) { - ABORT_ARG1("Seek to object table failed", " in file: %s", path); - } - for (nsegs = E32_OBJCNT(hdr386); nsegs > 0; nsegs--) { - int flags; - if (fread((char *)(&seg), 1, sizeof(seg), myexefile) < sizeof(seg)) { - ABORT_ARG1("Could not read obj table entry", " from file: %s", path); - } - flags = O32_FLAGS(seg); - if (!(flags & OBJWRITE)) continue; - if (!(flags & OBJREAD)) continue; - if (flags & OBJINVALID) { - GC_err_printf("Object with invalid pages?\n"); - continue; - } - GC_add_roots_inner((ptr_t)O32_BASE(seg), - (ptr_t)(O32_BASE(seg) + O32_SIZE(seg)), FALSE); - } - (void)fclose(myexefile); -} - -#else /* !OS2 */ - -#if defined(GWW_VDB) -#ifndef MEM_WRITE_WATCH -#define MEM_WRITE_WATCH 0x200000 -#endif -#ifndef WRITE_WATCH_FLAG_RESET -#define WRITE_WATCH_FLAG_RESET 1 -#endif - -/* Since we can't easily check whether ULONG_PTR and SIZE_T are */ -/* defined in Win32 basetsd.h, we define own ULONG_PTR. */ -#define GC_ULONG_PTR word - -typedef UINT(WINAPI *GetWriteWatch_type)( - DWORD, PVOID, GC_ULONG_PTR /* SIZE_T */, - PVOID *, GC_ULONG_PTR *, PULONG); -static FARPROC GetWriteWatch_func; -static DWORD GetWriteWatch_alloc_flag; - -#define GC_GWW_AVAILABLE() (GetWriteWatch_func != 0) - -static void detect_GetWriteWatch(void) { - static GC_bool done; - HMODULE hK32; - if (done) - return; - -#if defined(MPROTECT_VDB) - { - char *str = GETENV("GC_USE_GETWRITEWATCH"); -#if defined(GC_PREFER_MPROTECT_VDB) - if (str == NULL || (*str == '0' && *(str + 1) == '\0')) { - /* GC_USE_GETWRITEWATCH is unset or set to "0". */ - done = TRUE; /* falling back to MPROTECT_VDB strategy. */ - /* This should work as if GWW_VDB is undefined. */ - return; - } -#else - if (str != NULL && *str == '0' && *(str + 1) == '\0') { - /* GC_USE_GETWRITEWATCH is set "0". */ - done = TRUE; /* falling back to MPROTECT_VDB strategy. */ - return; - } -#endif - } -#endif - -#ifdef MSWINRT_FLAVOR - { - MEMORY_BASIC_INFORMATION memInfo; - SIZE_T result = VirtualQuery((void *)(word)GetProcAddress, - &memInfo, sizeof(memInfo)); - if (result != sizeof(memInfo)) - ABORT("Weird VirtualQuery result"); - hK32 = (HMODULE)memInfo.AllocationBase; - } -#else - hK32 = GetModuleHandle(TEXT("kernel32.dll")); -#endif - if (hK32 != (HMODULE)0 && - (GetWriteWatch_func = GetProcAddress(hK32, "GetWriteWatch")) != 0) { - /* Also check whether VirtualAlloc accepts MEM_WRITE_WATCH, */ - /* as some versions of kernel32.dll have one but not the */ - /* other, making the feature completely broken. */ - void *page; - - GC_ASSERT(GC_page_size != 0); - page = VirtualAlloc(NULL, GC_page_size, MEM_WRITE_WATCH | MEM_RESERVE, - PAGE_READWRITE); - if (page != NULL) { - PVOID pages[16]; - GC_ULONG_PTR count = 16; - DWORD page_size; - /* Check that it actually works. In spite of some */ - /* documentation it actually seems to exist on Win2K. */ - /* This test may be unnecessary, but ... */ - if ((*(GetWriteWatch_type)(word)GetWriteWatch_func)( - WRITE_WATCH_FLAG_RESET, page, - GC_page_size, pages, &count, - &page_size) != 0) { - /* GetWriteWatch always fails. */ - GetWriteWatch_func = 0; - } else { - GetWriteWatch_alloc_flag = MEM_WRITE_WATCH; - } - VirtualFree(page, 0 /* dwSize */, MEM_RELEASE); - } else { - /* GetWriteWatch will be useless. */ - GetWriteWatch_func = 0; - } - } - done = TRUE; -} - -#else -#define GetWriteWatch_alloc_flag 0 -#endif /* !GWW_VDB */ - -#if defined(MSWIN32) || defined(MSWINCE) || defined(CYGWIN32) - -#ifdef MSWIN32 -/* Unfortunately, we have to handle win32s very differently from NT, */ -/* Since VirtualQuery has very different semantics. In particular, */ -/* under win32s a VirtualQuery call on an unmapped page returns an */ -/* invalid result. Under NT, GC_register_data_segments is a no-op */ -/* and all real work is done by GC_register_dynamic_libraries. Under */ -/* win32s, we cannot find the data segments associated with dll's. */ -/* We register the main data segment here. */ -GC_INNER GC_bool GC_no_win32_dlls = FALSE; -/* This used to be set for gcc, to avoid dealing with */ -/* the structured exception handling issues. But we now have */ -/* assembly code to do that right. */ - -GC_INNER GC_bool GC_wnt = FALSE; -/* This is a Windows NT derivative, i.e. NT, Win2K, XP or later. */ - -GC_INNER void GC_init_win32(void) { -#if defined(_WIN64) || (defined(_MSC_VER) && _MSC_VER >= 1800) - /* MS Visual Studio 2013 deprecates GetVersion, but on the other */ - /* hand it cannot be used to target pre-Win2K. */ - GC_wnt = TRUE; -#else - /* Set GC_wnt. If we're running under win32s, assume that no */ - /* DLLs will be loaded. I doubt anyone still runs win32s, but... */ - DWORD v = GetVersion(); - - GC_wnt = !(v & 0x80000000); - GC_no_win32_dlls |= ((!GC_wnt) && (v & 0xff) <= 3); -#endif -#ifdef USE_MUNMAP - if (GC_no_win32_dlls) { - /* Turn off unmapping for safety (since may not work well with */ - /* GlobalAlloc). */ - GC_unmap_threshold = 0; - } -#endif -} - -/* Return the smallest address a such that VirtualQuery */ -/* returns correct results for all addresses between a and start. */ -/* Assumes VirtualQuery returns correct information for start. */ -STATIC ptr_t GC_least_described_address(ptr_t start) { - MEMORY_BASIC_INFORMATION buf; - LPVOID limit = GC_sysinfo.lpMinimumApplicationAddress; - ptr_t p = (ptr_t)((word)start & ~(GC_page_size - 1)); - - GC_ASSERT(GC_page_size != 0); - for (;;) { - size_t result; - LPVOID q = (LPVOID)(p - GC_page_size); - - if ((word)q > (word)p /* underflow */ || (word)q < (word)limit) break; - result = VirtualQuery(q, &buf, sizeof(buf)); - if (result != sizeof(buf) || buf.AllocationBase == 0) break; - p = (ptr_t)(buf.AllocationBase); - } - return p; -} -#endif /* MSWIN32 */ - -#if defined(USE_WINALLOC) && !defined(REDIRECT_MALLOC) -/* We maintain a linked list of AllocationBase values that we know */ -/* correspond to malloc heap sections. Currently this is only called */ -/* during a GC. But there is some hope that for long running */ -/* programs we will eventually see most heap sections. */ - -/* In the long run, it would be more reliable to occasionally walk */ -/* the malloc heap with HeapWalk on the default heap. But that */ -/* apparently works only for NT-based Windows. */ - -STATIC size_t GC_max_root_size = 100000; /* Appr. largest root size. */ - -/* In the long run, a better data structure would also be nice ... */ -STATIC struct GC_malloc_heap_list { - void *allocation_base; - struct GC_malloc_heap_list *next; -} *GC_malloc_heap_l = 0; - -/* Is p the base of one of the malloc heap sections we already know */ -/* about? */ -STATIC GC_bool GC_is_malloc_heap_base(const void *p) { - struct GC_malloc_heap_list *q; - - for (q = GC_malloc_heap_l; q != NULL; q = q->next) { - if (q->allocation_base == p) return TRUE; - } - return FALSE; -} - -STATIC void *GC_get_allocation_base(void *p) { - MEMORY_BASIC_INFORMATION buf; - size_t result = VirtualQuery(p, &buf, sizeof(buf)); - if (result != sizeof(buf)) { - ABORT("Weird VirtualQuery result"); - } - return buf.AllocationBase; -} - -GC_INNER void GC_add_current_malloc_heap(void) { - struct GC_malloc_heap_list *new_l = (struct GC_malloc_heap_list *) - malloc(sizeof(struct GC_malloc_heap_list)); - void *candidate; - - if (NULL == new_l) return; - new_l->allocation_base = NULL; - /* to suppress maybe-uninitialized gcc warning */ - - candidate = GC_get_allocation_base(new_l); - if (GC_is_malloc_heap_base(candidate)) { - /* Try a little harder to find malloc heap. */ - size_t req_size = 10000; - do { - void *p = malloc(req_size); - if (0 == p) { - free(new_l); - return; - } - candidate = GC_get_allocation_base(p); - free(p); - req_size *= 2; - } while (GC_is_malloc_heap_base(candidate) && req_size < GC_max_root_size / 10 && req_size < 500000); - if (GC_is_malloc_heap_base(candidate)) { - free(new_l); - return; - } - } - GC_COND_LOG_PRINTF("Found new system malloc AllocationBase at %p\n", - candidate); - new_l->allocation_base = candidate; - new_l->next = GC_malloc_heap_l; - GC_malloc_heap_l = new_l; -} - -/* Free all the linked list nodes. Could be invoked at process exit */ -/* to avoid memory leak complains of a dynamic code analysis tool. */ -STATIC void GC_free_malloc_heap_list(void) { - struct GC_malloc_heap_list *q = GC_malloc_heap_l; - - GC_malloc_heap_l = NULL; - while (q != NULL) { - struct GC_malloc_heap_list *next = q->next; - free(q); - q = next; - } -} -#endif /* USE_WINALLOC && !REDIRECT_MALLOC */ - -/* Is p the start of either the malloc heap, or of one of our */ -/* heap sections? */ -GC_INNER GC_bool GC_is_heap_base(const void *p) { - int i; - -#if defined(USE_WINALLOC) && !defined(REDIRECT_MALLOC) - if (GC_root_size > GC_max_root_size) - GC_max_root_size = GC_root_size; - if (GC_is_malloc_heap_base(p)) - return TRUE; -#endif - for (i = 0; i < (int)GC_n_heap_bases; i++) { - if (GC_heap_bases[i] == p) return TRUE; - } - return FALSE; -} - -#ifdef MSWIN32 -STATIC void GC_register_root_section(ptr_t static_root) { - MEMORY_BASIC_INFORMATION buf; - LPVOID p; - char *base; - char *limit; - - GC_ASSERT(I_HOLD_LOCK()); - if (!GC_no_win32_dlls) return; - p = base = limit = GC_least_described_address(static_root); - while ((word)p < (word)GC_sysinfo.lpMaximumApplicationAddress) { - size_t result = VirtualQuery(p, &buf, sizeof(buf)); - char *new_limit; - DWORD protect; - - if (result != sizeof(buf) || buf.AllocationBase == 0 || GC_is_heap_base(buf.AllocationBase)) break; - new_limit = (char *)p + buf.RegionSize; - protect = buf.Protect; - if (buf.State == MEM_COMMIT && is_writable(protect)) { - if ((char *)p == limit) { - limit = new_limit; - } else { - if (base != limit) GC_add_roots_inner(base, limit, FALSE); - base = (char *)p; - limit = new_limit; - } - } - if ((word)p > (word)new_limit /* overflow */) break; - p = (LPVOID)new_limit; - } - if (base != limit) GC_add_roots_inner(base, limit, FALSE); -} -#endif /* MSWIN32 */ - -void GC_register_data_segments(void) { -#ifdef MSWIN32 - GC_register_root_section((ptr_t)&GC_pages_executable); - /* any other GC global variable would fit too. */ -#endif -} - -#else /* !OS2 && !Windows */ - -#if (defined(SVR4) || defined(AIX) || defined(DGUX) || (defined(LINUX) && defined(SPARC))) && !defined(PCR) -ptr_t GC_SysVGetDataStart(size_t max_page_size, ptr_t etext_addr) { - word text_end = ((word)(etext_addr) + sizeof(word) - 1) & ~(word)(sizeof(word) - 1); - /* etext rounded to word boundary */ - word next_page = ((text_end + (word)max_page_size - 1) & ~((word)max_page_size - 1)); - word page_offset = (text_end & ((word)max_page_size - 1)); - volatile ptr_t result = (char *)(next_page + page_offset); - /* Note that this isn't equivalent to just adding */ - /* max_page_size to &etext if &etext is at a page boundary */ - - GC_setup_temporary_fault_handler(); - if (SETJMP(GC_jmp_buf) == 0) { - /* Try writing to the address. */ -#ifdef AO_HAVE_fetch_and_add - volatile AO_t zero = 0; - (void)AO_fetch_and_add((volatile AO_t *)result, zero); -#else - /* Fallback to non-atomic fetch-and-store. */ - char v = *result; -#if defined(CPPCHECK) - GC_noop1((word)&v); -#endif - *result = v; -#endif - GC_reset_fault_handler(); - } else { - GC_reset_fault_handler(); - /* We got here via a longjmp. The address is not readable. */ - /* This is known to happen under Solaris 2.4 + gcc, which place */ - /* string constants in the text segment, but after etext. */ - /* Use plan B. Note that we now know there is a gap between */ - /* text and data segments, so plan A brought us something. */ - result = (char *)GC_find_limit(DATAEND, FALSE); - } - return (/* no volatile */ ptr_t)result; -} -#endif - -#ifdef DATASTART_USES_BSDGETDATASTART -/* It's unclear whether this should be identical to the above, or */ -/* whether it should apply to non-x86 architectures. */ -/* For now we don't assume that there is always an empty page after */ -/* etext. But in some cases there actually seems to be slightly more. */ -/* This also deals with holes between read-only data and writable data. */ -GC_INNER ptr_t GC_FreeBSDGetDataStart(size_t max_page_size, - ptr_t etext_addr) { - word text_end = ((word)(etext_addr) + sizeof(word) - 1) & ~(word)(sizeof(word) - 1); - /* etext rounded to word boundary */ - volatile word next_page = (text_end + (word)max_page_size - 1) & ~((word)max_page_size - 1); - volatile ptr_t result = (ptr_t)text_end; - GC_setup_temporary_fault_handler(); - if (SETJMP(GC_jmp_buf) == 0) { - /* Try reading at the address. */ - /* This should happen before there is another thread. */ - for (; next_page < (word)DATAEND; next_page += (word)max_page_size) - *(volatile char *)next_page; - GC_reset_fault_handler(); - } else { - GC_reset_fault_handler(); - /* As above, we go to plan B */ - result = (ptr_t)GC_find_limit(DATAEND, FALSE); - } - return result; -} -#endif /* DATASTART_USES_BSDGETDATASTART */ - -#ifdef AMIGA - -#define GC_AMIGA_DS -#include "extra/AmigaOS.c" -#undef GC_AMIGA_DS - -#elif defined(OPENBSD) - -/* Depending on arch alignment, there can be multiple holes */ -/* between DATASTART and DATAEND. Scan in DATASTART .. DATAEND */ -/* and register each region. */ -void GC_register_data_segments(void) { - ptr_t region_start = DATASTART; - - GC_ASSERT(I_HOLD_LOCK()); - if ((word)region_start - 1U >= (word)DATAEND) - ABORT_ARG2("Wrong DATASTART/END pair", - ": %p .. %p", (void *)region_start, (void *)DATAEND); - for (;;) { - ptr_t region_end = GC_find_limit_with_bound(region_start, TRUE, DATAEND); - - GC_add_roots_inner(region_start, region_end, FALSE); - if ((word)region_end >= (word)DATAEND) - break; - region_start = GC_skip_hole_openbsd(region_end, DATAEND); - } -} - -#else /* !OS2 && !Windows && !AMIGA && !OPENBSD */ - -#if !defined(PCR) && !defined(MACOS) && defined(REDIRECT_MALLOC) && defined(GC_SOLARIS_THREADS) -EXTERN_C_BEGIN -extern caddr_t sbrk(int); -EXTERN_C_END -#endif - -void GC_register_data_segments(void) { - GC_ASSERT(I_HOLD_LOCK()); -#if !defined(DYNAMIC_LOADING) && defined(GC_DONT_REGISTER_MAIN_STATIC_DATA) - /* Avoid even referencing DATASTART and DATAEND as they are */ - /* unnecessary and cause linker errors when bitcode is enabled. */ - /* GC_register_data_segments() is not called anyway. */ -#elif defined(PCR) - /* No-op. */ -#elif defined(MACOS) - { -#if defined(THINK_C) - extern void *GC_MacGetDataStart(void); - - /* Globals begin above stack and end at a5. */ - GC_add_roots_inner((ptr_t)GC_MacGetDataStart(), - (ptr_t)LMGetCurrentA5(), FALSE); -#elif defined(__MWERKS__) && defined(M68K) - extern void *GC_MacGetDataStart(void); -#if __option(far_data) - extern void *GC_MacGetDataEnd(void); - - /* Handle Far Globals (CW Pro 3) located after the QD globals. */ - GC_add_roots_inner((ptr_t)GC_MacGetDataStart(), - (ptr_t)GC_MacGetDataEnd(), FALSE); -#else - GC_add_roots_inner((ptr_t)GC_MacGetDataStart(), - (ptr_t)LMGetCurrentA5(), FALSE); -#endif -#elif defined(__MWERKS__) && defined(POWERPC) - extern char __data_start__[], __data_end__[]; - - GC_add_roots_inner((ptr_t)&__data_start__, - (ptr_t)&__data_end__, FALSE); -#endif - } -#elif defined(REDIRECT_MALLOC) && defined(GC_SOLARIS_THREADS) - /* As of Solaris 2.3, the Solaris threads implementation */ - /* allocates the data structure for the initial thread with */ - /* sbrk at process startup. It needs to be scanned, so that */ - /* we don't lose some malloc allocated data structures */ - /* hanging from it. We're on thin ice here ... */ - GC_ASSERT(DATASTART); - { - ptr_t p = (ptr_t)sbrk(0); - if ((word)DATASTART < (word)p) - GC_add_roots_inner(DATASTART, p, FALSE); - } -#else - if ((word)DATASTART - 1U >= (word)DATAEND) { - /* Subtract one to check also for NULL */ - /* without a compiler warning. */ - ABORT_ARG2("Wrong DATASTART/END pair", - ": %p .. %p", (void *)DATASTART, (void *)DATAEND); - } - GC_add_roots_inner(DATASTART, DATAEND, FALSE); -#ifdef GC_HAVE_DATAREGION2 - if ((word)DATASTART2 - 1U >= (word)DATAEND2) - ABORT_ARG2("Wrong DATASTART/END2 pair", - ": %p .. %p", (void *)DATASTART2, (void *)DATAEND2); - GC_add_roots_inner(DATASTART2, DATAEND2, FALSE); -#endif -#endif - /* Dynamic libraries are added at every collection, since they may */ - /* change. */ -} - -#endif /* !AMIGA */ -#endif /* !MSWIN32 && !MSWINCE */ -#endif /* !OS2 */ - -/* - * Auxiliary routines for obtaining memory from OS. - */ - -#if !defined(OS2) && !defined(PCR) && !defined(AMIGA) && !defined(USE_WINALLOC) && !defined(MACOS) && !defined(DOS4GW) && !defined(NINTENDO_SWITCH) && !defined(NONSTOP) && !defined(SN_TARGET_ORBIS) && !defined(SN_TARGET_PS3) && !defined(SN_TARGET_PSP2) && !defined(RTEMS) && !defined(__CC_ARM) - -#define SBRK_ARG_T ptrdiff_t - -#if defined(MMAP_SUPPORTED) - -#ifdef USE_MMAP_FIXED -#define GC_MMAP_FLAGS MAP_FIXED | MAP_PRIVATE -/* Seems to yield better performance on Solaris 2, but can */ -/* be unreliable if something is already mapped at the address. */ -#else -#define GC_MMAP_FLAGS MAP_PRIVATE -#endif - -#ifdef USE_MMAP_ANON -#define zero_fd -1 -#if defined(MAP_ANONYMOUS) && !defined(CPPCHECK) -#define OPT_MAP_ANON MAP_ANONYMOUS -#else -#define OPT_MAP_ANON MAP_ANON -#endif -#else -static int zero_fd = -1; -#define OPT_MAP_ANON 0 -#endif - -#ifndef MSWIN_XBOX1 -#if defined(SYMBIAN) && !defined(USE_MMAP_ANON) -EXTERN_C_BEGIN -extern char *GC_get_private_path_and_zero_file(void); -EXTERN_C_END -#endif - -STATIC ptr_t GC_unix_mmap_get_mem(size_t bytes) { - void *result; - static ptr_t last_addr = HEAP_START; - -#ifndef USE_MMAP_ANON - static GC_bool initialized = FALSE; - - if (!EXPECT(initialized, TRUE)) { -#ifdef SYMBIAN - char *path = GC_get_private_path_and_zero_file(); - if (path != NULL) { - zero_fd = open(path, O_RDWR | O_CREAT, 0644); - free(path); - } -#else - zero_fd = open("/dev/zero", O_RDONLY); -#endif - if (zero_fd == -1) - ABORT("Could not open /dev/zero"); - if (fcntl(zero_fd, F_SETFD, FD_CLOEXEC) == -1) - WARN("Could not set FD_CLOEXEC for /dev/zero\n", 0); - - initialized = TRUE; - } -#endif - - GC_ASSERT(GC_page_size != 0); - if (bytes & (GC_page_size - 1)) ABORT("Bad GET_MEM arg"); - result = mmap(last_addr, bytes, (PROT_READ | PROT_WRITE) | (GC_pages_executable ? PROT_EXEC : 0), - GC_MMAP_FLAGS | OPT_MAP_ANON, zero_fd, 0 /* offset */); -#undef IGNORE_PAGES_EXECUTABLE - - if (EXPECT(MAP_FAILED == result, FALSE)) { - if (HEAP_START == last_addr && GC_pages_executable && (EACCES == errno || EPERM == errno)) - ABORT("Cannot allocate executable pages"); - return NULL; - } - last_addr = (ptr_t)(((word)result + bytes + GC_page_size - 1) & ~(GC_page_size - 1)); -#if !defined(LINUX) - if (last_addr == 0) { - /* Oops. We got the end of the address space. This isn't */ - /* usable by arbitrary C code, since one-past-end pointers */ - /* don't work, so we discard it and try again. */ - munmap(result, ~GC_page_size - (size_t)result + 1); - /* Leave last page mapped, so we can't repeat. */ - return GC_unix_mmap_get_mem(bytes); - } -#else - GC_ASSERT(last_addr != 0); -#endif - if (((word)result % HBLKSIZE) != 0) - ABORT( - "GC_unix_get_mem: Memory returned by mmap is not aligned to HBLKSIZE."); - return (ptr_t)result; -} -#endif /* !MSWIN_XBOX1 */ - -#endif /* MMAP_SUPPORTED */ - -#if defined(USE_MMAP) -ptr_t GC_unix_get_mem(size_t bytes) { - return GC_unix_mmap_get_mem(bytes); -} -#else /* !USE_MMAP */ - -STATIC ptr_t GC_unix_sbrk_get_mem(size_t bytes) { - ptr_t result; -#ifdef IRIX5 - /* Bare sbrk isn't thread safe. Play by malloc rules. */ - /* The equivalent may be needed on other systems as well. */ - __LOCK_MALLOC(); -#endif - { - ptr_t cur_brk = (ptr_t)sbrk(0); - SBRK_ARG_T lsbs = (word)cur_brk & (GC_page_size - 1); - - GC_ASSERT(GC_page_size != 0); - if ((SBRK_ARG_T)bytes < 0) { - result = 0; /* too big */ - goto out; - } - if (lsbs != 0) { - if ((ptr_t)sbrk((SBRK_ARG_T)GC_page_size - lsbs) == (ptr_t)(-1)) { - result = 0; - goto out; - } - } -#ifdef ADD_HEAP_GUARD_PAGES - /* This is useful for catching severe memory overwrite problems that */ - /* span heap sections. It shouldn't otherwise be turned on. */ - { - ptr_t guard = (ptr_t)sbrk((SBRK_ARG_T)GC_page_size); - if (mprotect(guard, GC_page_size, PROT_NONE) != 0) - ABORT("ADD_HEAP_GUARD_PAGES: mprotect failed"); - } -#endif /* ADD_HEAP_GUARD_PAGES */ - result = (ptr_t)sbrk((SBRK_ARG_T)bytes); - if (result == (ptr_t)(-1)) result = 0; - } -out: -#ifdef IRIX5 - __UNLOCK_MALLOC(); -#endif - return result; -} - -ptr_t GC_unix_get_mem(size_t bytes) { -#if defined(MMAP_SUPPORTED) - /* By default, we try both sbrk and mmap, in that order. */ - static GC_bool sbrk_failed = FALSE; - ptr_t result = 0; - - if (GC_pages_executable) { - /* If the allocated memory should have the execute permission */ - /* then sbrk() cannot be used. */ - return GC_unix_mmap_get_mem(bytes); - } - if (!sbrk_failed) result = GC_unix_sbrk_get_mem(bytes); - if (0 == result) { - sbrk_failed = TRUE; - result = GC_unix_mmap_get_mem(bytes); - } - if (0 == result) { - /* Try sbrk again, in case sbrk memory became available. */ - result = GC_unix_sbrk_get_mem(bytes); - } - return result; -#else /* !MMAP_SUPPORTED */ - return GC_unix_sbrk_get_mem(bytes); -#endif -} - -#endif /* !USE_MMAP */ - -#endif /* UN*X */ - -#ifdef OS2 - -void *os2_alloc(size_t bytes) { - void *result; - - if (DosAllocMem(&result, bytes, (PAG_READ | PAG_WRITE | PAG_COMMIT) | (GC_pages_executable ? PAG_EXECUTE : 0)) != NO_ERROR) { - return NULL; - } - /* FIXME: What's the purpose of this recursion? (Probably, if */ - /* DosAllocMem returns memory at 0 address then just retry once.) */ - if (NULL == result) return os2_alloc(bytes); - return result; -} - -#endif /* OS2 */ - -#ifdef MSWIN_XBOX1 -ptr_t GC_durango_get_mem(size_t bytes) { - if (0 == bytes) return NULL; - return (ptr_t)VirtualAlloc(NULL, bytes, MEM_COMMIT | MEM_TOP_DOWN, - PAGE_READWRITE); -} -#elif defined(MSWINCE) -ptr_t GC_wince_get_mem(size_t bytes) { - ptr_t result = 0; /* initialized to prevent warning. */ - word i; - - GC_ASSERT(GC_page_size != 0); - bytes = ROUNDUP_PAGESIZE(bytes); - - /* Try to find reserved, uncommitted pages */ - for (i = 0; i < GC_n_heap_bases; i++) { - if (((word)(-(signed_word)GC_heap_lengths[i]) & (GC_sysinfo.dwAllocationGranularity - 1)) >= bytes) { - result = GC_heap_bases[i] + GC_heap_lengths[i]; - break; - } - } - - if (i == GC_n_heap_bases) { - /* Reserve more pages */ - size_t res_bytes = - SIZET_SAT_ADD(bytes, (size_t)GC_sysinfo.dwAllocationGranularity - 1) & ~((size_t)GC_sysinfo.dwAllocationGranularity - 1); - /* If we ever support MPROTECT_VDB here, we will probably need to */ - /* ensure that res_bytes is strictly > bytes, so that VirtualProtect */ - /* never spans regions. It seems to be OK for a VirtualFree */ - /* argument to span regions, so we should be OK for now. */ - result = (ptr_t)VirtualAlloc(NULL, res_bytes, - MEM_RESERVE | MEM_TOP_DOWN, - GC_pages_executable ? PAGE_EXECUTE_READWRITE : PAGE_READWRITE); - if (HBLKDISPL(result) != 0) ABORT("Bad VirtualAlloc result"); - /* If I read the documentation correctly, this can */ - /* only happen if HBLKSIZE > 64 KB or not a power of 2. */ - if (GC_n_heap_bases >= MAX_HEAP_SECTS) ABORT("Too many heap sections"); - if (result == NULL) return NULL; - GC_heap_bases[GC_n_heap_bases] = result; - GC_heap_lengths[GC_n_heap_bases] = 0; - GC_n_heap_bases++; - } - - /* Commit pages */ - result = (ptr_t)VirtualAlloc(result, bytes, MEM_COMMIT, - GC_pages_executable ? PAGE_EXECUTE_READWRITE : PAGE_READWRITE); -#undef IGNORE_PAGES_EXECUTABLE - - if (result != NULL) { - if (HBLKDISPL(result) != 0) ABORT("Bad VirtualAlloc result"); - GC_heap_lengths[i] += bytes; - } - return result; -} - -#elif defined(USE_WINALLOC) /* && !MSWIN_XBOX1 */ || defined(CYGWIN32) - -#ifdef USE_GLOBAL_ALLOC -#define GLOBAL_ALLOC_TEST 1 -#else -#define GLOBAL_ALLOC_TEST GC_no_win32_dlls -#endif - -#if (defined(GC_USE_MEM_TOP_DOWN) && defined(USE_WINALLOC)) || defined(CPPCHECK) -DWORD GC_mem_top_down = MEM_TOP_DOWN; -/* Use GC_USE_MEM_TOP_DOWN for better 64-bit */ -/* testing. Otherwise all addresses tend to */ -/* end up in first 4 GB, hiding bugs. */ -#else -#define GC_mem_top_down 0 -#endif /* !GC_USE_MEM_TOP_DOWN */ - -ptr_t GC_win32_get_mem(size_t bytes) { - ptr_t result; - -#ifndef USE_WINALLOC - result = GC_unix_get_mem(bytes); -#else -#if defined(MSWIN32) && !defined(MSWINRT_FLAVOR) - if (GLOBAL_ALLOC_TEST) { - /* VirtualAlloc doesn't like PAGE_EXECUTE_READWRITE. */ - /* There are also unconfirmed rumors of other */ - /* problems, so we dodge the issue. */ - result = (ptr_t)GlobalAlloc(0, SIZET_SAT_ADD(bytes, HBLKSIZE)); - /* Align it at HBLKSIZE boundary. */ - result = (ptr_t)(((word)result + HBLKSIZE - 1) & ~(word)(HBLKSIZE - 1)); - } else -#endif - /* else */ { - /* VirtualProtect only works on regions returned by a */ - /* single VirtualAlloc call. Thus we allocate one */ - /* extra page, which will prevent merging of blocks */ - /* in separate regions, and eliminate any temptation */ - /* to call VirtualProtect on a range spanning regions. */ - /* This wastes a small amount of memory, and risks */ - /* increased fragmentation. But better alternatives */ - /* would require effort. */ -#ifdef MPROTECT_VDB - /* We can't check for GC_incremental here (because */ - /* GC_enable_incremental() might be called some time */ - /* later after the GC initialization). */ -#ifdef GWW_VDB -#define VIRTUAL_ALLOC_PAD (GC_GWW_AVAILABLE() ? 0 : 1) -#else -#define VIRTUAL_ALLOC_PAD 1 -#endif -#else -#define VIRTUAL_ALLOC_PAD 0 -#endif - /* Pass the MEM_WRITE_WATCH only if GetWriteWatch-based */ - /* VDBs are enabled and the GetWriteWatch function is */ - /* available. Otherwise we waste resources or possibly */ - /* cause VirtualAlloc to fail (observed in Windows 2000 */ - /* SP2). */ - result = (ptr_t)VirtualAlloc(NULL, - SIZET_SAT_ADD(bytes, VIRTUAL_ALLOC_PAD), - GetWriteWatch_alloc_flag | (MEM_COMMIT | MEM_RESERVE) | GC_mem_top_down, - GC_pages_executable ? PAGE_EXECUTE_READWRITE : PAGE_READWRITE); -#undef IGNORE_PAGES_EXECUTABLE - } -#endif /* USE_WINALLOC */ - if (HBLKDISPL(result) != 0) ABORT("Bad VirtualAlloc result"); - /* If I read the documentation correctly, this can */ - /* only happen if HBLKSIZE > 64 KB or not a power of 2. */ - if (GC_n_heap_bases >= MAX_HEAP_SECTS) ABORT("Too many heap sections"); - if (result != NULL) GC_heap_bases[GC_n_heap_bases++] = result; - return result; -} -#endif /* USE_WINALLOC || CYGWIN32 */ - -#if defined(MSWIN32) || defined(MSWINCE) || defined(CYGWIN32) || defined(MSWIN_XBOX1) -GC_API void GC_CALL GC_win32_free_heap(void) { -#if defined(USE_WINALLOC) && !defined(REDIRECT_MALLOC) && !defined(MSWIN_XBOX1) - GC_free_malloc_heap_list(); -#endif -#if (defined(USE_WINALLOC) && !defined(MSWIN_XBOX1) && !defined(MSWINCE)) || defined(CYGWIN32) -#ifndef MSWINRT_FLAVOR -#ifndef CYGWIN32 - if (GLOBAL_ALLOC_TEST) -#endif - { - while (GC_n_heap_bases-- > 0) { -#ifdef CYGWIN32 - /* FIXME: Is it OK to use non-GC free() here? */ -#else - GlobalFree(GC_heap_bases[GC_n_heap_bases]); -#endif - GC_heap_bases[GC_n_heap_bases] = 0; - } - return; - } -#endif /* !MSWINRT_FLAVOR */ -#ifndef CYGWIN32 - /* Avoiding VirtualAlloc leak. */ - while (GC_n_heap_bases > 0) { - VirtualFree(GC_heap_bases[--GC_n_heap_bases], 0, MEM_RELEASE); - GC_heap_bases[GC_n_heap_bases] = 0; - } -#endif -#endif /* USE_WINALLOC || CYGWIN32 */ -} -#endif /* Windows */ - -#ifdef AMIGA -#define GC_AMIGA_AM -#include "extra/AmigaOS.c" -#undef GC_AMIGA_AM -#endif - -#if defined(HAIKU) -#ifdef GC_LEAK_DETECTOR_H -#undef posix_memalign /* to use the real one */ -#endif -ptr_t GC_haiku_get_mem(size_t bytes) { - void *mem; - - GC_ASSERT(GC_page_size != 0); - if (posix_memalign(&mem, GC_page_size, bytes) == 0) - return mem; - return NULL; -} -#endif /* HAIKU */ - -#if (defined(USE_MUNMAP) || defined(MPROTECT_VDB)) && !defined(USE_WINALLOC) -#define ABORT_ON_REMAP_FAIL(C_msg_prefix, start_addr, len) \ - ABORT_ARG3(C_msg_prefix " failed", \ - " at %p (length %lu), errno= %d", \ - (void *)(start_addr), (unsigned long)(len), errno) -#endif - -#ifdef USE_MUNMAP - -/* For now, this only works on Win32/WinCE and some Unix-like */ -/* systems. If you have something else, don't define */ -/* USE_MUNMAP. */ - -#if !defined(NN_PLATFORM_CTR) && !defined(MSWIN32) && !defined(MSWINCE) && !defined(MSWIN_XBOX1) -#include -#ifdef SN_TARGET_PS3 -#include -#else -#include -#endif -#include -#include -#endif - -/* Compute a page aligned starting address for the unmap */ -/* operation on a block of size bytes starting at start. */ -/* Return 0 if the block is too small to make this feasible. */ -STATIC ptr_t GC_unmap_start(ptr_t start, size_t bytes) { - ptr_t result = (ptr_t)(((word)start + GC_page_size - 1) & ~(GC_page_size - 1)); - - GC_ASSERT(GC_page_size != 0); - if ((word)(result + GC_page_size) > (word)(start + bytes)) return 0; - return result; -} - -/* We assume that GC_remap is called on exactly the same range */ -/* as a previous call to GC_unmap. It is safe to consistently */ -/* round the endpoints in both places. */ - -static void block_unmap_inner(ptr_t start_addr, size_t len) { - if (0 == start_addr) return; - -#ifdef USE_WINALLOC - /* Under Win32/WinCE we commit (map) and decommit (unmap) */ - /* memory using VirtualAlloc and VirtualFree. These functions */ - /* work on individual allocations of virtual memory, made */ - /* previously using VirtualAlloc with the MEM_RESERVE flag. */ - /* The ranges we need to (de)commit may span several of these */ - /* allocations; therefore we use VirtualQuery to check */ - /* allocation lengths, and split up the range as necessary. */ - while (len != 0) { - MEMORY_BASIC_INFORMATION mem_info; - word free_len; - - if (VirtualQuery(start_addr, &mem_info, sizeof(mem_info)) != sizeof(mem_info)) - ABORT("Weird VirtualQuery result"); - free_len = (len < mem_info.RegionSize) ? len : mem_info.RegionSize; - if (!VirtualFree(start_addr, free_len, MEM_DECOMMIT)) - ABORT("VirtualFree failed"); - GC_unmapped_bytes += free_len; - start_addr += free_len; - len -= free_len; - } -#else - if (len != 0) { -#ifdef SN_TARGET_PS3 - ps3_free_mem(start_addr, len); -#elif defined(AIX) || defined(CYGWIN32) || defined(HAIKU) || (defined(LINUX) && !defined(PREFER_MMAP_PROT_NONE)) || defined(HPUX) - /* On AIX, mmap(PROT_NONE) fails with ENOMEM unless the */ - /* environment variable XPG_SUS_ENV is set to ON. */ - /* On Cygwin, calling mmap() with the new protection flags on */ - /* an existing memory map with MAP_FIXED is broken. */ - /* However, calling mprotect() on the given address range */ - /* with PROT_NONE seems to work fine. */ - /* On Linux, low RLIMIT_AS value may lead to mmap failure. */ -#if defined(LINUX) && !defined(FORCE_MPROTECT_BEFORE_MADVISE) - /* On Linux, at least, madvise() should be sufficient. */ -#else - if (mprotect(start_addr, len, PROT_NONE)) - ABORT_ON_REMAP_FAIL("unmap: mprotect", start_addr, len); -#endif -#if !defined(CYGWIN32) - /* On Linux (and some other platforms probably), */ - /* mprotect(PROT_NONE) is just disabling access to */ - /* the pages but not returning them to OS. */ - if (madvise(start_addr, len, MADV_DONTNEED) == -1) - ABORT_ON_REMAP_FAIL("unmap: madvise", start_addr, len); -#endif -#else - /* We immediately remap it to prevent an intervening mmap() */ - /* from accidentally grabbing the same address space. */ - void *result = mmap(start_addr, len, PROT_NONE, - MAP_PRIVATE | MAP_FIXED | OPT_MAP_ANON, - zero_fd, 0 /* offset */); - - if (EXPECT(MAP_FAILED == result, FALSE)) - ABORT_ON_REMAP_FAIL("unmap: mmap", start_addr, len); - if (result != (void *)start_addr) - ABORT("unmap: mmap() result differs from start_addr"); -#if defined(CPPCHECK) || defined(LINT2) - /* Explicitly store the resource handle to a global variable. */ - GC_noop1((word)result); -#endif -#endif - GC_unmapped_bytes += len; - } -#endif -} - -GC_INNER void GC_unmap(ptr_t start, size_t bytes) { - ptr_t start_addr = GC_unmap_start(start, bytes); - ptr_t end_addr = GC_unmap_end(start, bytes); - - block_unmap_inner(start_addr, (size_t)(end_addr - start_addr)); -} - -GC_INNER void GC_remap(ptr_t start, size_t bytes) { - ptr_t start_addr = GC_unmap_start(start, bytes); - ptr_t end_addr = GC_unmap_end(start, bytes); - word len = end_addr - start_addr; - if (0 == start_addr) return; - - /* FIXME: Handle out-of-memory correctly (at least for Win32) */ -#ifdef USE_WINALLOC - while (len != 0) { - MEMORY_BASIC_INFORMATION mem_info; - word alloc_len; - ptr_t result; - - if (VirtualQuery(start_addr, &mem_info, sizeof(mem_info)) != sizeof(mem_info)) - ABORT("Weird VirtualQuery result"); - alloc_len = (len < mem_info.RegionSize) ? len : mem_info.RegionSize; - result = (ptr_t)VirtualAlloc(start_addr, alloc_len, MEM_COMMIT, - GC_pages_executable - ? PAGE_EXECUTE_READWRITE - : PAGE_READWRITE); - if (result != start_addr) { - if (GetLastError() == ERROR_NOT_ENOUGH_MEMORY || - GetLastError() == ERROR_OUTOFMEMORY) { - ABORT("Not enough memory to process remapping"); - } else { - ABORT("VirtualAlloc remapping failed"); - } - } -#ifdef LINT2 - GC_noop1((word)result); -#endif - GC_ASSERT(GC_unmapped_bytes >= alloc_len); - GC_unmapped_bytes -= alloc_len; - start_addr += alloc_len; - len -= alloc_len; - } -#undef IGNORE_PAGES_EXECUTABLE -#else - /* It was already remapped with PROT_NONE. */ - { -#if !defined(SN_TARGET_PS3) && !defined(FORCE_MPROTECT_BEFORE_MADVISE) && defined(LINUX) && !defined(PREFER_MMAP_PROT_NONE) - /* Nothing to unprotect as madvise() is just a hint. */ -#elif defined(NACL) || defined(NETBSD) - /* NaCl does not expose mprotect, but mmap should work fine. */ - /* In case of NetBSD, mprotect fails (unlike mmap) even */ - /* without PROT_EXEC if PaX MPROTECT feature is enabled. */ - void *result = mmap(start_addr, len, (PROT_READ | PROT_WRITE) | (GC_pages_executable ? PROT_EXEC : 0), - MAP_PRIVATE | MAP_FIXED | OPT_MAP_ANON, - zero_fd, 0 /* offset */); - if (EXPECT(MAP_FAILED == result, FALSE)) - ABORT_ON_REMAP_FAIL("remap: mmap", start_addr, len); - if (result != (void *)start_addr) - ABORT("remap: mmap() result differs from start_addr"); -#if defined(CPPCHECK) || defined(LINT2) - GC_noop1((word)result); -#endif -#undef IGNORE_PAGES_EXECUTABLE -#else - if (mprotect(start_addr, len, (PROT_READ | PROT_WRITE) | (GC_pages_executable ? PROT_EXEC : 0))) - ABORT_ON_REMAP_FAIL("remap: mprotect", start_addr, len); -#undef IGNORE_PAGES_EXECUTABLE -#endif /* !NACL */ - } - GC_ASSERT(GC_unmapped_bytes >= len); - GC_unmapped_bytes -= len; -#endif -} - -/* Two adjacent blocks have already been unmapped and are about to */ -/* be merged. Unmap the whole block. This typically requires */ -/* that we unmap a small section in the middle that was not previously */ -/* unmapped due to alignment constraints. */ -GC_INNER void GC_unmap_gap(ptr_t start1, size_t bytes1, ptr_t start2, - size_t bytes2) { - ptr_t start1_addr = GC_unmap_start(start1, bytes1); - ptr_t end1_addr = GC_unmap_end(start1, bytes1); - ptr_t start2_addr = GC_unmap_start(start2, bytes2); - ptr_t start_addr = end1_addr; - ptr_t end_addr = start2_addr; - - GC_ASSERT(start1 + bytes1 == start2); - if (0 == start1_addr) start_addr = GC_unmap_start(start1, bytes1 + bytes2); - if (0 == start2_addr) end_addr = GC_unmap_end(start1, bytes1 + bytes2); - block_unmap_inner(start_addr, (size_t)(end_addr - start_addr)); -} - -#endif /* USE_MUNMAP */ - -/* Routine for pushing any additional roots. In THREADS */ -/* environment, this is also responsible for marking from */ -/* thread stacks. */ -#ifndef THREADS - -#if defined(EMSCRIPTEN) && defined(EMSCRIPTEN_ASYNCIFY) -#include - -static void scan_regs_cb(void *begin, void *end) { - GC_push_all_stack((ptr_t)begin, (ptr_t)end); -} - -STATIC void GC_CALLBACK GC_default_push_other_roots(void) { - /* Note: this needs -sASYNCIFY linker flag. */ - emscripten_scan_registers(scan_regs_cb); -} - -#else -#define GC_default_push_other_roots 0 -#endif - -#else /* THREADS */ - -#ifdef PCR -PCR_ERes GC_push_thread_stack(PCR_Th_T *t, PCR_Any dummy) { - struct PCR_ThCtl_TInfoRep info; - PCR_ERes result; - - info.ti_stkLow = info.ti_stkHi = 0; - result = PCR_ThCtl_GetInfo(t, &info); - GC_push_all_stack((ptr_t)(info.ti_stkLow), (ptr_t)(info.ti_stkHi)); - return result; -} - -/* Push the contents of an old object. We treat this as stack */ -/* data only because that makes it robust against mark stack */ -/* overflow. */ -PCR_ERes GC_push_old_obj(void *p, size_t size, PCR_Any data) { - GC_push_all_stack((ptr_t)p, (ptr_t)p + size); - return PCR_ERes_okay; -} - -extern struct PCR_MM_ProcsRep *GC_old_allocator; -/* defined in pcr_interface.c. */ - -STATIC void GC_CALLBACK GC_default_push_other_roots(void) { - /* Traverse data allocated by previous memory managers. */ - if ((*(GC_old_allocator->mmp_enumerate))(PCR_Bool_false, - GC_push_old_obj, 0) != PCR_ERes_okay) { - ABORT("Old object enumeration failed"); - } - /* Traverse all thread stacks. */ - if (PCR_ERes_IsErr( - PCR_ThCtl_ApplyToAllOtherThreads(GC_push_thread_stack, 0)) || - PCR_ERes_IsErr(GC_push_thread_stack(PCR_Th_CurrThread(), 0))) { - ABORT("Thread stack marking failed"); - } -} - -#elif defined(SN_TARGET_PS3) -STATIC void GC_CALLBACK GC_default_push_other_roots(void) { - ABORT("GC_default_push_other_roots is not implemented"); -} - -void GC_push_thread_structures(void) { - ABORT("GC_push_thread_structures is not implemented"); -} - -#else /* GC_PTHREADS, or GC_WIN32_THREADS, etc. */ -STATIC void GC_CALLBACK GC_default_push_other_roots(void) { - GC_push_all_stacks(); -} -#endif - -#endif /* THREADS */ - -GC_push_other_roots_proc GC_push_other_roots = GC_default_push_other_roots; - -GC_API void GC_CALL GC_set_push_other_roots(GC_push_other_roots_proc fn) { - GC_push_other_roots = fn; -} - -GC_API GC_push_other_roots_proc GC_CALL GC_get_push_other_roots(void) { - return GC_push_other_roots; -} - -#if defined(SOFT_VDB) && !defined(NO_SOFT_VDB_LINUX_VER_RUNTIME_CHECK) || (defined(GLIBC_2_19_TSX_BUG) && defined(THREADS)) -GC_INNER int GC_parse_version(int *pminor, const char *pverstr) { - char *endp; - unsigned long value = strtoul(pverstr, &endp, 10); - int major = (int)value; - - if (major < 0 || (char *)pverstr == endp || (unsigned)major != value) { - /* Parse error. */ - return -1; - } - if (*endp != '.') { - /* No minor part. */ - *pminor = -1; - } else { - value = strtoul(endp + 1, &endp, 10); - *pminor = (int)value; - if (*pminor < 0 || (unsigned)(*pminor) != value) { - return -1; - } - } - return major; -} -#endif - -/* - * Routines for accessing dirty bits on virtual pages. - * There are six ways to maintain this information: - * DEFAULT_VDB: A simple dummy implementation that treats every page - * as possibly dirty. This makes incremental collection - * useless, but the implementation is still correct. - * Manual VDB: Stacks and static data are always considered dirty. - * Heap pages are considered dirty if GC_dirty(p) has been - * called on some pointer p pointing to somewhere inside - * an object on that page. A GC_dirty() call on a large - * object directly dirties only a single page, but for the - * manual VDB we are careful to treat an object with a dirty - * page as completely dirty. - * In order to avoid races, an object must be marked dirty - * after it is written, and a reference to the object - * must be kept on a stack or in a register in the interim. - * With threads enabled, an object directly reachable from the - * stack at the time of a collection is treated as dirty. - * In single-threaded mode, it suffices to ensure that no - * collection can take place between the pointer assignment - * and the GC_dirty() call. - * PCR_VDB: Use PPCRs virtual dirty bit facility. - * PROC_VDB: Use the /proc facility for reading dirty bits. Only - * works under some SVR4 variants. Even then, it may be - * too slow to be entirely satisfactory. Requires reading - * dirty bits for entire address space. Implementations tend - * to assume that the client is a (slow) debugger. - * SOFT_VDB: Use the /proc facility for reading soft-dirty PTEs. - * Works on Linux 3.18+ if the kernel is properly configured. - * The proposed implementation iterates over GC_heap_sects and - * GC_static_roots examining the soft-dirty bit of the words - * in /proc/self/pagemap corresponding to the pages of the - * sections; finally all soft-dirty bits of the process are - * cleared (by writing some special value to - * /proc/self/clear_refs file). In case the soft-dirty bit is - * not supported by the kernel, MPROTECT_VDB may be defined as - * a fallback strategy. - * MPROTECT_VDB:Protect pages and then catch the faults to keep track of - * dirtied pages. The implementation (and implementability) - * is highly system dependent. This usually fails when system - * calls write to a protected page. We prevent the read system - * call from doing so. It is the clients responsibility to - * make sure that other system calls are similarly protected - * or write only to the stack. - * GWW_VDB: Use the Win32 GetWriteWatch functions, if available, to - * read dirty bits. In case it is not available (because we - * are running on Windows 95, Windows 2000 or earlier), - * MPROTECT_VDB may be defined as a fallback strategy. - */ - -#if (defined(CHECKSUMS) && (defined(GWW_VDB) || defined(SOFT_VDB))) || defined(PROC_VDB) -/* Add all pages in pht2 to pht1. */ -STATIC void GC_or_pages(page_hash_table pht1, const word *pht2) { - unsigned i; - for (i = 0; i < PHT_SIZE; i++) pht1[i] |= pht2[i]; -} -#endif /* CHECKSUMS && (GWW_VDB || SOFT_VDB) || PROC_VDB */ - -#ifdef GWW_VDB - -#define GC_GWW_BUF_LEN (MAXHINCR * HBLKSIZE / 4096 /* x86 page size */) -/* Still susceptible to overflow, if there are very large allocations, */ -/* and everything is dirty. */ -static PVOID gww_buf[GC_GWW_BUF_LEN]; - -#ifndef MPROTECT_VDB -#define GC_gww_dirty_init GC_dirty_init -#endif - -GC_INNER GC_bool GC_gww_dirty_init(void) { - /* No assumption about the GC lock. */ - detect_GetWriteWatch(); - return GC_GWW_AVAILABLE(); -} - -GC_INLINE void GC_gww_read_dirty(GC_bool output_unneeded) { - word i; - - GC_ASSERT(I_HOLD_LOCK()); - if (!output_unneeded) - BZERO(GC_grungy_pages, sizeof(GC_grungy_pages)); - - for (i = 0; i != GC_n_heap_sects; ++i) { - GC_ULONG_PTR count; - - do { - PVOID *pages = gww_buf; - DWORD page_size; - - count = GC_GWW_BUF_LEN; - /* GetWriteWatch is documented as returning non-zero when it */ - /* fails, but the documentation doesn't explicitly say why it */ - /* would fail or what its behavior will be if it fails. It */ - /* does appear to fail, at least on recent Win2K instances, if */ - /* the underlying memory was not allocated with the appropriate */ - /* flag. This is common if GC_enable_incremental is called */ - /* shortly after GC initialization. To avoid modifying the */ - /* interface, we silently work around such a failure, it only */ - /* affects the initial (small) heap allocation. If there are */ - /* more dirty pages than will fit in the buffer, this is not */ - /* treated as a failure; we must check the page count in the */ - /* loop condition. Since each partial call will reset the */ - /* status of some pages, this should eventually terminate even */ - /* in the overflow case. */ - if ((*(GetWriteWatch_type)(word)GetWriteWatch_func)( - WRITE_WATCH_FLAG_RESET, - GC_heap_sects[i].hs_start, - GC_heap_sects[i].hs_bytes, - pages, &count, &page_size) != 0) { - static int warn_count = 0; - struct hblk *start = (struct hblk *)GC_heap_sects[i].hs_start; - static struct hblk *last_warned = 0; - size_t nblocks = divHBLKSZ(GC_heap_sects[i].hs_bytes); - - if (i != 0 && last_warned != start && warn_count++ < 5) { - last_warned = start; - WARN( - "GC_gww_read_dirty unexpectedly failed at %p:" - " Falling back to marking all pages dirty\n", - start); - } - if (!output_unneeded) { - unsigned j; - - for (j = 0; j < nblocks; ++j) { - word hash = PHT_HASH(start + j); - set_pht_entry_from_index(GC_grungy_pages, hash); - } - } - count = 1; /* Done with this section. */ - } else /* succeeded */ - if (!output_unneeded) { - PVOID *pages_end = pages + count; - - while (pages != pages_end) { - struct hblk *h = (struct hblk *)*pages++; - struct hblk *h_end = (struct hblk *)((char *)h + page_size); - do { - set_pht_entry_from_index(GC_grungy_pages, PHT_HASH(h)); - } while ((word)(++h) < (word)h_end); - } - } - } while (count == GC_GWW_BUF_LEN); - /* FIXME: It's unclear from Microsoft's documentation if this loop */ - /* is useful. We suspect the call just fails if the buffer fills */ - /* up. But that should still be handled correctly. */ - } - -#ifdef CHECKSUMS - GC_ASSERT(!output_unneeded); - GC_or_pages(GC_written_pages, GC_grungy_pages); -#endif -} - -#elif defined(SOFT_VDB) -static int clear_refs_fd = -1; -#define GC_GWW_AVAILABLE() (clear_refs_fd != -1) -#else -#define GC_GWW_AVAILABLE() FALSE -#endif /* !GWW_VDB && !SOFT_VDB */ - -#ifdef DEFAULT_VDB -/* The client asserts that unallocated pages in the heap are never */ -/* written. */ - -/* Initialize virtual dirty bit implementation. */ -GC_INNER GC_bool GC_dirty_init(void) { - GC_VERBOSE_LOG_PRINTF("Initializing DEFAULT_VDB...\n"); - /* GC_dirty_pages and GC_grungy_pages are already cleared. */ - return TRUE; -} -#endif /* DEFAULT_VDB */ - -#ifndef GC_DISABLE_INCREMENTAL -#if !defined(THREADS) || defined(HAVE_LOCKFREE_AO_OR) -#define async_set_pht_entry_from_index(db, index) \ - set_pht_entry_from_index_concurrent(db, index) -#elif defined(AO_HAVE_test_and_set_acquire) -/* We need to lock around the bitmap update (in the write fault */ -/* handler or GC_dirty) in order to avoid the risk of losing a bit. */ -/* We do this with a test-and-set spin lock if possible. */ -GC_INNER volatile AO_TS_t GC_fault_handler_lock = AO_TS_INITIALIZER; - -static void async_set_pht_entry_from_index(volatile page_hash_table db, - size_t index) { - GC_acquire_dirty_lock(); - set_pht_entry_from_index(db, index); - GC_release_dirty_lock(); -} -#else -#error No test_and_set operation: Introduces a race. -#endif /* THREADS && !AO_HAVE_test_and_set_acquire */ -#endif /* !GC_DISABLE_INCREMENTAL */ - -#ifdef MPROTECT_VDB -/* - * This implementation maintains dirty bits itself by catching write - * faults and keeping track of them. We assume nobody else catches - * SIGBUS or SIGSEGV. We assume no write faults occur in system calls. - * This means that clients must ensure that system calls don't write - * to the write-protected heap. Probably the best way to do this is to - * ensure that system calls write at most to pointer-free objects in the - * heap, and do even that only if we are on a platform on which those - * are not protected. Another alternative is to wrap system calls - * (see example for read below), but the current implementation holds - * applications. - * We assume the page size is a multiple of HBLKSIZE. - * We prefer them to be the same. We avoid protecting pointer-free - * objects only if they are the same. - */ -#ifdef DARWIN -/* Using vm_protect (mach syscall) over mprotect (BSD syscall) seems to - decrease the likelihood of some of the problems described below. */ -#include -STATIC mach_port_t GC_task_self = 0; -#define PROTECT_INNER(addr, len, allow_write, C_msg_prefix) \ - if (vm_protect(GC_task_self, (vm_address_t)(addr), (vm_size_t)(len), \ - FALSE, VM_PROT_READ | ((allow_write) ? VM_PROT_WRITE : 0) | (GC_pages_executable ? VM_PROT_EXECUTE : 0)) == KERN_SUCCESS) { \ - } else \ - ABORT(C_msg_prefix \ - "vm_protect() failed") - -#elif !defined(USE_WINALLOC) -#include -#include -#if !defined(AIX) && !defined(CYGWIN32) && !defined(HAIKU) -#include -#endif - -#define PROTECT_INNER(addr, len, allow_write, C_msg_prefix) \ - if (mprotect((caddr_t)(addr), (size_t)(len), \ - PROT_READ | ((allow_write) ? PROT_WRITE : 0) | (GC_pages_executable ? PROT_EXEC : 0)) >= 0) { \ - } else if (GC_pages_executable) { \ - ABORT_ON_REMAP_FAIL(C_msg_prefix \ - "mprotect vdb executable pages", \ - addr, len); \ - } else \ - ABORT_ON_REMAP_FAIL(C_msg_prefix "mprotect vdb", addr, len) -#undef IGNORE_PAGES_EXECUTABLE - -#else /* USE_WINALLOC */ -#ifndef MSWINCE -#include -#endif - -static DWORD protect_junk; -#define PROTECT_INNER(addr, len, allow_write, C_msg_prefix) \ - if (VirtualProtect(addr, len, \ - GC_pages_executable ? ((allow_write) ? PAGE_EXECUTE_READWRITE : PAGE_EXECUTE_READ) : (allow_write) ? PAGE_READWRITE \ - : PAGE_READONLY, \ - &protect_junk)) { \ - } else \ - ABORT_ARG1(C_msg_prefix "VirtualProtect failed", \ - ": errcode= 0x%X", (unsigned)GetLastError()) -#endif /* USE_WINALLOC */ - -#define PROTECT(addr, len) PROTECT_INNER(addr, len, FALSE, "") -#define UNPROTECT(addr, len) PROTECT_INNER(addr, len, TRUE, "un-") - -#if defined(MSWIN32) -typedef LPTOP_LEVEL_EXCEPTION_FILTER SIG_HNDLR_PTR; -#undef SIG_DFL -#define SIG_DFL (LPTOP_LEVEL_EXCEPTION_FILTER)((signed_word)-1) -#elif defined(MSWINCE) -typedef LONG(WINAPI *SIG_HNDLR_PTR)(struct _EXCEPTION_POINTERS *); -#undef SIG_DFL -#define SIG_DFL (SIG_HNDLR_PTR)(-1) -#elif defined(DARWIN) -typedef void (*SIG_HNDLR_PTR)(); -#else -typedef void (*SIG_HNDLR_PTR)(int, siginfo_t *, void *); -typedef void (*PLAIN_HNDLR_PTR)(int); -#endif - -#ifndef DARWIN -STATIC SIG_HNDLR_PTR GC_old_segv_handler = 0; -/* Also old MSWIN32 ACCESS_VIOLATION filter */ -#ifdef USE_BUS_SIGACT -STATIC SIG_HNDLR_PTR GC_old_bus_handler = 0; -STATIC GC_bool GC_old_bus_handler_used_si = FALSE; -#endif -#if !defined(MSWIN32) && !defined(MSWINCE) -STATIC GC_bool GC_old_segv_handler_used_si = FALSE; -#endif /* !MSWIN32 */ -#endif /* !DARWIN */ - -#ifdef THREADS -/* This function is used only by the fault handler. Potential data */ -/* race between this function and GC_install_header, GC_remove_header */ -/* should not be harmful because the added or removed header should */ -/* be already unprotected. */ -GC_ATTR_NO_SANITIZE_THREAD -static GC_bool is_header_found_async(void *addr) { -#ifdef HASH_TL - hdr *result; - GET_HDR((ptr_t)addr, result); - return result != NULL; -#else - return HDR_INNER(addr) != NULL; -#endif -} -#else -#define is_header_found_async(addr) (HDR(addr) != NULL) -#endif /* !THREADS */ - -#ifndef DARWIN - -#if !defined(MSWIN32) && !defined(MSWINCE) -#include -#ifdef USE_BUS_SIGACT -#define SIG_OK (sig == SIGBUS || sig == SIGSEGV) -#else -#define SIG_OK (sig == SIGSEGV) -/* Catch SIGSEGV but ignore SIGBUS. */ -#endif -#if defined(FREEBSD) -#ifndef SEGV_ACCERR -#define SEGV_ACCERR 2 -#endif -#if defined(AARCH64) || defined(ARM32) || defined(MIPS) || __FreeBSD__ >= 7 -#define CODE_OK (si->si_code == SEGV_ACCERR) -#elif defined(POWERPC) -#define AIM /* Pretend that we're AIM. */ -#include -#define CODE_OK (si->si_code == EXC_DSI || si->si_code == SEGV_ACCERR) -#else -#define CODE_OK (si->si_code == BUS_PAGE_FAULT || si->si_code == SEGV_ACCERR) -#endif -#elif defined(OSF1) -#define CODE_OK (si->si_code == 2 /* experimentally determined */) -#elif defined(IRIX5) -#define CODE_OK (si->si_code == EACCES) -#elif defined(AIX) || defined(CYGWIN32) || defined(HAIKU) || defined(HURD) -#define CODE_OK TRUE -#elif defined(LINUX) -#define CODE_OK TRUE -/* Empirically c.trapno == 14, on IA32, but is that useful? */ -/* Should probably consider alignment issues on other */ -/* architectures. */ -#elif defined(HPUX) -#define CODE_OK (si->si_code == SEGV_ACCERR || si->si_code == BUS_ADRERR || si->si_code == BUS_UNKNOWN || si->si_code == SEGV_UNKNOWN || si->si_code == BUS_OBJERR) -#elif defined(SUNOS5SIGS) -#define CODE_OK (si->si_code == SEGV_ACCERR) -#endif -#ifndef NO_GETCONTEXT -#include -#endif -STATIC void GC_write_fault_handler(int sig, siginfo_t *si, void *raw_sc) -#else -#define SIG_OK (exc_info->ExceptionRecord->ExceptionCode == STATUS_ACCESS_VIOLATION) -#define CODE_OK (exc_info->ExceptionRecord->ExceptionInformation[0] == 1) /* Write fault */ -STATIC LONG WINAPI GC_write_fault_handler( - struct _EXCEPTION_POINTERS *exc_info) -#endif /* MSWIN32 || MSWINCE */ -{ -#if !defined(MSWIN32) && !defined(MSWINCE) - char *addr = (char *)si->si_addr; -#else - char *addr = (char *)(exc_info->ExceptionRecord - ->ExceptionInformation[1]); -#endif - - if (SIG_OK && CODE_OK) { - struct hblk *h = (struct hblk *)((word)addr & ~(GC_page_size - 1)); - GC_bool in_allocd_block; - size_t i; - - GC_ASSERT(GC_page_size != 0); -#ifdef CHECKSUMS - GC_record_fault(h); -#endif -#ifdef SUNOS5SIGS - /* Address is only within the correct physical page. */ - in_allocd_block = FALSE; - for (i = 0; i < divHBLKSZ(GC_page_size); i++) { - if (is_header_found_async(&h[i])) { - in_allocd_block = TRUE; - break; - } - } -#else - in_allocd_block = is_header_found_async(addr); -#endif - if (!in_allocd_block) { - /* FIXME - We should make sure that we invoke the */ - /* old handler with the appropriate calling */ - /* sequence, which often depends on SA_SIGINFO. */ - - /* Heap blocks now begin and end on page boundaries */ - SIG_HNDLR_PTR old_handler; - -#if defined(MSWIN32) || defined(MSWINCE) - old_handler = GC_old_segv_handler; -#else - GC_bool used_si; - -#ifdef USE_BUS_SIGACT - if (sig == SIGBUS) { - old_handler = GC_old_bus_handler; - used_si = GC_old_bus_handler_used_si; - } else -#endif - /* else */ { - old_handler = GC_old_segv_handler; - used_si = GC_old_segv_handler_used_si; - } -#endif - - if (old_handler == (SIG_HNDLR_PTR)(signed_word)SIG_DFL) { -#if !defined(MSWIN32) && !defined(MSWINCE) - ABORT_ARG1("Unexpected segmentation fault outside heap", - " at %p", (void *)addr); -#else - return EXCEPTION_CONTINUE_SEARCH; -#endif - } else { - /* - * FIXME: This code should probably check if the - * old signal handler used the traditional style and - * if so call it using that style. - */ -#if defined(MSWIN32) || defined(MSWINCE) - return (*old_handler)(exc_info); -#else - if (used_si) - ((SIG_HNDLR_PTR)old_handler)(sig, si, raw_sc); - else - /* FIXME: should pass nonstandard args as well. */ - ((PLAIN_HNDLR_PTR)(signed_word)old_handler)(sig); - return; -#endif - } - } - UNPROTECT(h, GC_page_size); - /* We need to make sure that no collection occurs between */ - /* the UNPROTECT and the setting of the dirty bit. Otherwise */ - /* a write by a third thread might go unnoticed. Reversing */ - /* the order is just as bad, since we would end up unprotecting */ - /* a page in a GC cycle during which it's not marked. */ - /* Currently we do this by disabling the thread stopping */ - /* signals while this handler is running. An alternative might */ - /* be to record the fact that we're about to unprotect, or */ - /* have just unprotected a page in the GC's thread structure, */ - /* and then to have the thread stopping code set the dirty */ - /* flag, if necessary. */ - for (i = 0; i < divHBLKSZ(GC_page_size); i++) { - word index = PHT_HASH(h + i); - - async_set_pht_entry_from_index(GC_dirty_pages, index); - } - /* The write may not take place before dirty bits are read. */ - /* But then we'll fault again ... */ -#if defined(MSWIN32) || defined(MSWINCE) - return EXCEPTION_CONTINUE_EXECUTION; -#else - return; -#endif - } -#if defined(MSWIN32) || defined(MSWINCE) - return EXCEPTION_CONTINUE_SEARCH; -#else - ABORT_ARG1("Unexpected bus error or segmentation fault", - " at %p", (void *)addr); -#endif -} - -#if defined(GC_WIN32_THREADS) && !defined(CYGWIN32) -GC_INNER void GC_set_write_fault_handler(void) { - SetUnhandledExceptionFilter(GC_write_fault_handler); -} -#endif - -#ifdef SOFT_VDB -static GC_bool soft_dirty_init(void); -#endif - -GC_INNER GC_bool GC_dirty_init(void) { -#if !defined(MSWIN32) && !defined(MSWINCE) - struct sigaction act, oldact; -#endif - - GC_ASSERT(I_HOLD_LOCK()); -#if !defined(MSWIN32) && !defined(MSWINCE) - act.sa_flags = SA_RESTART | SA_SIGINFO; - act.sa_sigaction = GC_write_fault_handler; - (void)sigemptyset(&act.sa_mask); -#ifdef SIGNAL_BASED_STOP_WORLD - /* Arrange to postpone the signal while we are in a write fault */ - /* handler. This effectively makes the handler atomic w.r.t. */ - /* stopping the world for GC. */ - (void)sigaddset(&act.sa_mask, GC_get_suspend_signal()); -#endif -#endif /* !MSWIN32 */ - GC_VERBOSE_LOG_PRINTF( - "Initializing mprotect virtual dirty bit implementation\n"); - if (GC_page_size % HBLKSIZE != 0) { - ABORT("Page size not multiple of HBLKSIZE"); - } -#ifdef GWW_VDB - if (GC_gww_dirty_init()) { - GC_COND_LOG_PRINTF("Using GetWriteWatch()\n"); - return TRUE; - } -#elif defined(SOFT_VDB) - if (soft_dirty_init()) { - GC_COND_LOG_PRINTF("Using soft-dirty bit feature\n"); - return TRUE; - } -#endif -#ifdef MSWIN32 - GC_old_segv_handler = SetUnhandledExceptionFilter( - GC_write_fault_handler); - if (GC_old_segv_handler != NULL) { - GC_COND_LOG_PRINTF("Replaced other UnhandledExceptionFilter\n"); - } else { - GC_old_segv_handler = SIG_DFL; - } -#elif defined(MSWINCE) - /* MPROTECT_VDB is unsupported for WinCE at present. */ - /* FIXME: implement it (if possible). */ -#else - /* act.sa_restorer is deprecated and should not be initialized. */ -#if defined(GC_IRIX_THREADS) - sigaction(SIGSEGV, 0, &oldact); - sigaction(SIGSEGV, &act, 0); -#else - { - int res = sigaction(SIGSEGV, &act, &oldact); - if (res != 0) ABORT("Sigaction failed"); - } -#endif - if (oldact.sa_flags & SA_SIGINFO) { - GC_old_segv_handler = oldact.sa_sigaction; - GC_old_segv_handler_used_si = TRUE; - } else { - GC_old_segv_handler = (SIG_HNDLR_PTR)(signed_word)oldact.sa_handler; - GC_old_segv_handler_used_si = FALSE; - } - if (GC_old_segv_handler == (SIG_HNDLR_PTR)(signed_word)SIG_IGN) { - WARN("Previously ignored segmentation violation!?\n", 0); - GC_old_segv_handler = (SIG_HNDLR_PTR)(signed_word)SIG_DFL; - } - if (GC_old_segv_handler != (SIG_HNDLR_PTR)(signed_word)SIG_DFL) { - GC_VERBOSE_LOG_PRINTF("Replaced other SIGSEGV handler\n"); - } -#ifdef USE_BUS_SIGACT - sigaction(SIGBUS, &act, &oldact); - if ((oldact.sa_flags & SA_SIGINFO) != 0) { - GC_old_bus_handler = oldact.sa_sigaction; - GC_old_bus_handler_used_si = TRUE; - } else { - GC_old_bus_handler = (SIG_HNDLR_PTR)(signed_word)oldact.sa_handler; - } - if (GC_old_bus_handler == (SIG_HNDLR_PTR)(signed_word)SIG_IGN) { - WARN("Previously ignored bus error!?\n", 0); - GC_old_bus_handler = (SIG_HNDLR_PTR)(signed_word)SIG_DFL; - } else if (GC_old_bus_handler != (SIG_HNDLR_PTR)(signed_word)SIG_DFL) { - GC_VERBOSE_LOG_PRINTF("Replaced other SIGBUS handler\n"); - } -#endif -#endif /* !MSWIN32 && !MSWINCE */ -#if defined(CPPCHECK) && defined(ADDRESS_SANITIZER) - GC_noop1((word)&__asan_default_options); -#endif - return TRUE; -} -#endif /* !DARWIN */ - -GC_API int GC_CALL GC_incremental_protection_needs(void) { - GC_ASSERT(GC_is_initialized); -#if defined(GWW_VDB) || defined(SOFT_VDB) - /* Only if the incremental mode is already switched on. */ - if (GC_GWW_AVAILABLE()) - return GC_PROTECTS_NONE; -#endif - if (GC_page_size == HBLKSIZE) { - return GC_PROTECTS_POINTER_HEAP; - } else { - return GC_PROTECTS_POINTER_HEAP | GC_PROTECTS_PTRFREE_HEAP; - } -} -#define HAVE_INCREMENTAL_PROTECTION_NEEDS - -#define IS_PTRFREE(hhdr) ((hhdr)->hb_descr == 0) -#define PAGE_ALIGNED(x) !((word)(x) & (GC_page_size - 1)) - -STATIC void GC_protect_heap(void) { - unsigned i; - GC_bool protect_all = - (0 != (GC_incremental_protection_needs() & GC_PROTECTS_PTRFREE_HEAP)); - - GC_ASSERT(GC_page_size != 0); - for (i = 0; i < GC_n_heap_sects; i++) { - ptr_t start = GC_heap_sects[i].hs_start; - size_t len = GC_heap_sects[i].hs_bytes; - - if (protect_all) { - PROTECT(start, len); - } else { - struct hblk *current; - struct hblk *current_start; /* Start of block to be protected. */ - struct hblk *limit; - - GC_ASSERT(PAGE_ALIGNED(len)); - GC_ASSERT(PAGE_ALIGNED(start)); - current_start = current = (struct hblk *)start; - limit = (struct hblk *)(start + len); - while ((word)current < (word)limit) { - hdr *hhdr; - word nhblks; - GC_bool is_ptrfree; - - GC_ASSERT(PAGE_ALIGNED(current)); - GET_HDR(current, hhdr); - if (IS_FORWARDING_ADDR_OR_NIL(hhdr)) { - /* This can happen only if we're at the beginning of a */ - /* heap segment, and a block spans heap segments. */ - /* We will handle that block as part of the preceding */ - /* segment. */ - GC_ASSERT(current_start == current); - current_start = ++current; - continue; - } - if (HBLK_IS_FREE(hhdr)) { - GC_ASSERT(PAGE_ALIGNED(hhdr->hb_sz)); - nhblks = divHBLKSZ(hhdr->hb_sz); - is_ptrfree = TRUE; /* dirty on alloc */ - } else { - nhblks = OBJ_SZ_TO_BLOCKS(hhdr->hb_sz); - is_ptrfree = IS_PTRFREE(hhdr); - } - if (is_ptrfree) { - if ((word)current_start < (word)current) { - PROTECT(current_start, (ptr_t)current - (ptr_t)current_start); - } - current_start = (current += nhblks); - } else { - current += nhblks; - } - } - if ((word)current_start < (word)current) { - PROTECT(current_start, (ptr_t)current - (ptr_t)current_start); - } - } - } -} - -/* - * Acquiring the allocation lock here is dangerous, since this - * can be called from within GC_call_with_alloc_lock, and the cord - * package does so. On systems that allow nested lock acquisition, this - * happens to work. - */ - -/* We no longer wrap read by default, since that was causing too many */ -/* problems. It is preferred that the client instead avoids writing */ -/* to the write-protected heap with a system call. */ -#endif /* MPROTECT_VDB */ - -#if !defined(THREADS) && (defined(PROC_VDB) || defined(SOFT_VDB)) -static pid_t saved_proc_pid; /* pid used to compose /proc file names */ -#endif - -#ifdef PROC_VDB -/* This implementation assumes a Solaris 2.X like /proc */ -/* pseudo-file-system from which we can read page modified bits. This */ -/* facility is far from optimal (e.g. we would like to get the info for */ -/* only some of the address space), but it avoids intercepting system */ -/* calls. */ - -#include -#include -#include -#include -#include - -#ifdef GC_NO_SYS_FAULT_H -/* This exists only to check PROC_VDB code compilation (on Linux). */ -#define PG_MODIFIED 1 -struct prpageheader { - int dummy[2]; /* pr_tstamp */ - unsigned long pr_nmap; - unsigned long pr_npage; -}; -struct prasmap { - char *pr_vaddr; - size_t pr_npage; - char dummy1[64 + 8]; /* pr_mapname, pr_offset */ - unsigned pr_mflags; - unsigned pr_pagesize; - int dummy2[2]; -}; -#else -#include -#include -#endif - -#define INITIAL_BUF_SZ 16384 -STATIC size_t GC_proc_buf_size = INITIAL_BUF_SZ; -STATIC char *GC_proc_buf = NULL; -STATIC int GC_proc_fd = -1; - -static GC_bool proc_dirty_open_files(void) { - char buf[40]; - pid_t pid = getpid(); - - (void)snprintf(buf, sizeof(buf), "/proc/%ld/pagedata", (long)pid); - buf[sizeof(buf) - 1] = '\0'; - GC_proc_fd = open(buf, O_RDONLY); - if (-1 == GC_proc_fd) { - WARN("/proc open failed; cannot enable GC incremental mode\n", 0); - return FALSE; - } - if (syscall(SYS_fcntl, GC_proc_fd, F_SETFD, FD_CLOEXEC) == -1) - WARN("Could not set FD_CLOEXEC for /proc\n", 0); -#ifndef THREADS - saved_proc_pid = pid; /* updated on success only */ -#endif - return TRUE; -} - -#ifdef CAN_HANDLE_FORK -GC_INNER void GC_dirty_update_child(void) { - if (-1 == GC_proc_fd) - return; /* GC incremental mode is off */ - - close(GC_proc_fd); - if (!proc_dirty_open_files()) - GC_incremental = FALSE; /* should be safe to turn it off */ -} -#endif /* CAN_HANDLE_FORK */ - -GC_INNER GC_bool GC_dirty_init(void) { - GC_ASSERT(I_HOLD_LOCK()); - if (GC_bytes_allocd != 0 || GC_bytes_allocd_before_gc != 0) { - memset(GC_written_pages, 0xff, sizeof(page_hash_table)); - GC_VERBOSE_LOG_PRINTF( - "Allocated %lu bytes: all pages may have been written\n", - (unsigned long)(GC_bytes_allocd + GC_bytes_allocd_before_gc)); - } - if (!proc_dirty_open_files()) - return FALSE; - GC_proc_buf = GC_scratch_alloc(GC_proc_buf_size); - if (GC_proc_buf == NULL) - ABORT("Insufficient space for /proc read"); - return TRUE; -} - -GC_INLINE void GC_proc_read_dirty(GC_bool output_unneeded) { - int nmaps; - char *bufp = GC_proc_buf; - int i; - - GC_ASSERT(I_HOLD_LOCK()); -#ifndef THREADS - /* If the current pid differs from the saved one, then we are in */ - /* the forked (child) process, the current /proc file should be */ - /* closed, the new one should be opened with the updated path. */ - /* Note, this is not needed for multi-threaded case because */ - /* fork_child_proc() reopens the file right after fork. */ - if (getpid() != saved_proc_pid && (-1 == GC_proc_fd /* no need to retry */ - || (close(GC_proc_fd), !proc_dirty_open_files()))) { - /* Failed to reopen the file. Punt! */ - if (!output_unneeded) - memset(GC_grungy_pages, 0xff, sizeof(page_hash_table)); - memset(GC_written_pages, 0xff, sizeof(page_hash_table)); - return; - } -#endif - - BZERO(GC_grungy_pages, sizeof(GC_grungy_pages)); - if (PROC_READ(GC_proc_fd, bufp, GC_proc_buf_size) <= 0) { - /* Retry with larger buffer. */ - size_t new_size = 2 * GC_proc_buf_size; - char *new_buf; - - WARN("/proc read failed (buffer size is %" WARN_PRIuPTR " bytes)\n", - GC_proc_buf_size); - new_buf = GC_scratch_alloc(new_size); - if (new_buf != 0) { - GC_scratch_recycle_no_gww(bufp, GC_proc_buf_size); - GC_proc_buf = bufp = new_buf; - GC_proc_buf_size = new_size; - } - if (PROC_READ(GC_proc_fd, bufp, GC_proc_buf_size) <= 0) { - WARN("Insufficient space for /proc read\n", 0); - /* Punt: */ - if (!output_unneeded) - memset(GC_grungy_pages, 0xff, sizeof(page_hash_table)); - memset(GC_written_pages, 0xff, sizeof(page_hash_table)); - return; - } - } - - /* Copy dirty bits into GC_grungy_pages */ - nmaps = ((struct prpageheader *)bufp)->pr_nmap; -#ifdef DEBUG_DIRTY_BITS - GC_log_printf("Proc VDB read: pr_nmap= %u, pr_npage= %lu\n", - nmaps, ((struct prpageheader *)bufp)->pr_npage); -#endif -#if defined(GC_NO_SYS_FAULT_H) && defined(CPPCHECK) - GC_noop1(((struct prpageheader *)bufp)->dummy[0]); -#endif - bufp += sizeof(struct prpageheader); - for (i = 0; i < nmaps; i++) { - struct prasmap *map = (struct prasmap *)bufp; - ptr_t vaddr = (ptr_t)(map->pr_vaddr); - unsigned long npages = map->pr_npage; - unsigned pagesize = map->pr_pagesize; - ptr_t limit; - -#if defined(GC_NO_SYS_FAULT_H) && defined(CPPCHECK) - GC_noop1(map->dummy1[0] + map->dummy2[0]); -#endif -#ifdef DEBUG_DIRTY_BITS - GC_log_printf( - "pr_vaddr= %p, npage= %lu, mflags= 0x%x, pagesize= 0x%x\n", - (void *)vaddr, npages, map->pr_mflags, pagesize); -#endif - - bufp += sizeof(struct prasmap); - limit = vaddr + pagesize * npages; - for (; (word)vaddr < (word)limit; vaddr += pagesize) { - if ((*bufp++) & PG_MODIFIED) { - struct hblk *h; - ptr_t next_vaddr = vaddr + pagesize; -#ifdef DEBUG_DIRTY_BITS - GC_log_printf("dirty page at: %p\n", (void *)vaddr); -#endif - for (h = (struct hblk *)vaddr; - (word)h < (word)next_vaddr; h++) { - word index = PHT_HASH(h); - - set_pht_entry_from_index(GC_grungy_pages, index); - } - } - } - bufp = (char *)(((word)bufp + (sizeof(long) - 1)) & ~(word)(sizeof(long) - 1)); - } -#ifdef DEBUG_DIRTY_BITS - GC_log_printf("Proc VDB read done\n"); -#endif - - /* Update GC_written_pages (even if output_unneeded). */ - GC_or_pages(GC_written_pages, GC_grungy_pages); -} - -#endif /* PROC_VDB */ - -#ifdef SOFT_VDB -#ifndef VDB_BUF_SZ -#define VDB_BUF_SZ 16384 -#endif - -static int open_proc_fd(pid_t pid, const char *proc_filename, int mode) { - int f; - char buf[40]; - - (void)snprintf(buf, sizeof(buf), "/proc/%ld/%s", (long)pid, - proc_filename); - buf[sizeof(buf) - 1] = '\0'; - f = open(buf, mode); - if (-1 == f) { - WARN("/proc/self/%s open failed; cannot enable GC incremental mode\n", - proc_filename); - } else if (fcntl(f, F_SETFD, FD_CLOEXEC) == -1) { - WARN("Could not set FD_CLOEXEC for /proc\n", 0); - } - return f; -} - -#include /* for uint64_t */ - -typedef uint64_t pagemap_elem_t; - -static pagemap_elem_t *soft_vdb_buf; -static int pagemap_fd; - -static GC_bool soft_dirty_open_files(void) { - pid_t pid = getpid(); - - clear_refs_fd = open_proc_fd(pid, "clear_refs", O_WRONLY); - if (-1 == clear_refs_fd) - return FALSE; - pagemap_fd = open_proc_fd(pid, "pagemap", O_RDONLY); - if (-1 == pagemap_fd) { - close(clear_refs_fd); - clear_refs_fd = -1; - return FALSE; - } -#ifndef THREADS - saved_proc_pid = pid; /* updated on success only */ -#endif - return TRUE; -} - -#ifdef CAN_HANDLE_FORK -GC_INNER void GC_dirty_update_child(void) { - if (-1 == clear_refs_fd) - return; /* GC incremental mode is off */ - - close(clear_refs_fd); - close(pagemap_fd); - if (!soft_dirty_open_files()) - GC_incremental = FALSE; -} -#endif /* CAN_HANDLE_FORK */ - -/* The bit 55 of the 64-bit qword of pagemap file is the soft-dirty one. */ -#define PM_SOFTDIRTY_MASK ((pagemap_elem_t)1 << 55) - -static GC_bool detect_soft_dirty_supported(ptr_t vaddr) { - off_t fpos; - pagemap_elem_t buf[1]; - - *vaddr = 1; /* make it dirty */ - - /* Read the relevant PTE from the pagemap file. */ - GC_ASSERT(GC_page_size != 0); - fpos = (off_t)((word)vaddr / GC_page_size * sizeof(pagemap_elem_t)); - if (lseek(pagemap_fd, fpos, SEEK_SET) == (off_t)(-1)) - return FALSE; - if (PROC_READ(pagemap_fd, buf, sizeof(buf)) != (int)sizeof(buf)) - return FALSE; - - /* Is the soft-dirty bit set? */ - return (buf[0] & PM_SOFTDIRTY_MASK) != 0; -} - -#ifndef NO_SOFT_VDB_LINUX_VER_RUNTIME_CHECK -#include /* for strcmp() */ -#include - -/* Ensure the linux (kernel) major/minor version is as given or higher. */ -static GC_bool ensure_min_linux_ver(int major, int minor) { - struct utsname info; - int actual_major; - int actual_minor = -1; - - if (uname(&info) == -1) { - return FALSE; /* uname() failed, should not happen actually. */ - } - if (strcmp(info.sysname, "Linux")) { - WARN("Cannot ensure Linux version as running on other OS: %s\n", - info.sysname); - return FALSE; - } - actual_major = GC_parse_version(&actual_minor, info.release); - return actual_major > major || (actual_major == major && actual_minor >= minor); -} -#endif - -#ifdef MPROTECT_VDB -static GC_bool soft_dirty_init(void) -#else -GC_INNER GC_bool GC_dirty_init(void) -#endif -{ -#ifdef MPROTECT_VDB - char *str = GETENV("GC_USE_GETWRITEWATCH"); -#ifdef GC_PREFER_MPROTECT_VDB - if (str == NULL || (*str == '0' && *(str + 1) == '\0')) - return FALSE; /* the environment variable is unset or set to "0" */ -#else - if (str != NULL && *str == '0' && *(str + 1) == '\0') - return FALSE; /* the environment variable is set "0" */ -#endif -#endif - GC_ASSERT(I_HOLD_LOCK()); - GC_ASSERT(NULL == soft_vdb_buf); -#ifndef NO_SOFT_VDB_LINUX_VER_RUNTIME_CHECK - if (!ensure_min_linux_ver(3, 18)) { - GC_COND_LOG_PRINTF( - "Running on old kernel lacking correct soft-dirty bit support\n"); - return FALSE; - } -#endif - if (!soft_dirty_open_files()) - return FALSE; - soft_vdb_buf = (pagemap_elem_t *)GC_scratch_alloc(VDB_BUF_SZ); - if (NULL == soft_vdb_buf) - ABORT("Insufficient space for /proc pagemap buffer"); - if (!detect_soft_dirty_supported((ptr_t)soft_vdb_buf)) { - GC_COND_LOG_PRINTF("Soft-dirty bit is not supported by kernel\n"); - /* Release the resources. */ - GC_scratch_recycle_no_gww(soft_vdb_buf, VDB_BUF_SZ); - soft_vdb_buf = NULL; - close(clear_refs_fd); - clear_refs_fd = -1; - close(pagemap_fd); - return FALSE; - } - return TRUE; -} - -static off_t pagemap_buf_fpos; /* valid only if pagemap_buf_len > 0 */ -static size_t pagemap_buf_len; - -/* Read bytes from /proc/self/pagemap at given file position. */ -/* len - the maximum number of bytes to read; (*pres) - amount of */ -/* bytes actually read, always bigger than 0 but never exceeds len; */ -/* next_fpos_hint - the file position of the next bytes block to read */ -/* ahead if possible (0 means no information provided). */ -static const pagemap_elem_t *pagemap_buffered_read(size_t *pres, - off_t fpos, size_t len, - off_t next_fpos_hint) { - ssize_t res; - size_t ofs; - - GC_ASSERT(len > 0); - if (pagemap_buf_fpos <= fpos && fpos < pagemap_buf_fpos + (off_t)pagemap_buf_len) { - /* The requested data is already in the buffer. */ - ofs = (size_t)(fpos - pagemap_buf_fpos); - res = (ssize_t)(pagemap_buf_fpos + pagemap_buf_len - fpos); - } else { - off_t aligned_pos = fpos & ~(GC_page_size < VDB_BUF_SZ - ? GC_page_size - 1 - : VDB_BUF_SZ - 1); - - for (;;) { - size_t count; - - if ((0 == pagemap_buf_len || pagemap_buf_fpos + (off_t)pagemap_buf_len != aligned_pos) && lseek(pagemap_fd, aligned_pos, SEEK_SET) == (off_t)(-1)) - ABORT_ARG2("Failed to lseek /proc/self/pagemap", - ": offset= %lu, errno= %d", (unsigned long)fpos, errno); - - /* How much to read at once? */ - ofs = (size_t)(fpos - aligned_pos); - GC_ASSERT(ofs < VDB_BUF_SZ); - if (next_fpos_hint > aligned_pos && next_fpos_hint - aligned_pos < VDB_BUF_SZ) { - count = VDB_BUF_SZ; - } else { - count = len + ofs; - if (count > VDB_BUF_SZ) - count = VDB_BUF_SZ; - } - - GC_ASSERT(count % sizeof(pagemap_elem_t) == 0); - res = PROC_READ(pagemap_fd, soft_vdb_buf, count); - if (res > (ssize_t)ofs) - break; - if (res <= 0) - ABORT_ARG1("Failed to read /proc/self/pagemap", - ": errno= %d", res < 0 ? errno : 0); - /* Retry (once) w/o page-alignment. */ - aligned_pos = fpos; - } - - /* Save the buffer (file window) position and size. */ - pagemap_buf_fpos = aligned_pos; - pagemap_buf_len = (size_t)res; - res -= (ssize_t)ofs; - } - - GC_ASSERT(ofs % sizeof(pagemap_elem_t) == 0); - *pres = (size_t)res < len ? (size_t)res : len; - return &soft_vdb_buf[ofs / sizeof(pagemap_elem_t)]; -} - -static void soft_set_grungy_pages(ptr_t vaddr /* start */, ptr_t limit, - ptr_t next_start_hint) { - GC_ASSERT(GC_page_size != 0); - while ((word)vaddr < (word)limit) { - size_t res; - word limit_buf; - const pagemap_elem_t *bufp = pagemap_buffered_read(&res, - (off_t)((word)vaddr / GC_page_size * sizeof(pagemap_elem_t)), - (size_t)((((word)limit - (word)vaddr + GC_page_size - 1) / GC_page_size) * sizeof(pagemap_elem_t)), - (off_t)((word)next_start_hint / GC_page_size * sizeof(pagemap_elem_t))); - - if (res % sizeof(pagemap_elem_t) != 0) { - /* Punt: */ - memset(GC_grungy_pages, 0xff, sizeof(page_hash_table)); - WARN("Incomplete read of pagemap, not multiple of entry size\n", 0); - break; - } - - limit_buf = ((word)vaddr & ~(GC_page_size - 1)) + (res / sizeof(pagemap_elem_t)) * GC_page_size; - for (; (word)vaddr < limit_buf; vaddr += GC_page_size, bufp++) - if ((*bufp & PM_SOFTDIRTY_MASK) != 0) { - struct hblk *h; - ptr_t next_vaddr = vaddr + GC_page_size; - - /* If the bit is set, the respective PTE was written to */ - /* since clearing the soft-dirty bits. */ -#ifdef DEBUG_DIRTY_BITS - GC_log_printf("dirty page at: %p\n", (void *)vaddr); -#endif - for (h = (struct hblk *)vaddr; (word)h < (word)next_vaddr; h++) { - word index = PHT_HASH(h); - set_pht_entry_from_index(GC_grungy_pages, index); - } - } - /* Read the next portion of pagemap file if incomplete. */ - } -} - -GC_INLINE void GC_soft_read_dirty(GC_bool output_unneeded) { - ssize_t res; - - GC_ASSERT(I_HOLD_LOCK()); -#ifndef THREADS - /* Similar as for GC_proc_read_dirty. */ - if (getpid() != saved_proc_pid && (-1 == clear_refs_fd /* no need to retry */ - || (close(clear_refs_fd), close(pagemap_fd), - !soft_dirty_open_files()))) { - /* Failed to reopen the files. */ - if (!output_unneeded) { - /* Punt: */ - memset(GC_grungy_pages, 0xff, sizeof(page_hash_table)); -#ifdef CHECKSUMS - memset(GC_written_pages, 0xff, sizeof(page_hash_table)); -#endif - } - return; - } -#endif - - if (!output_unneeded) { - word i; - - BZERO(GC_grungy_pages, sizeof(GC_grungy_pages)); - pagemap_buf_len = 0; /* invalidate soft_vdb_buf */ - - for (i = 0; i != GC_n_heap_sects; ++i) { - ptr_t vaddr = GC_heap_sects[i].hs_start; - - soft_set_grungy_pages(vaddr, vaddr + GC_heap_sects[i].hs_bytes, - i < GC_n_heap_sects - 1 ? GC_heap_sects[i + 1].hs_start : NULL); - } -#ifdef CHECKSUMS - GC_or_pages(GC_written_pages, GC_grungy_pages); -#endif - -#ifndef NO_VDB_FOR_STATIC_ROOTS - for (i = 0; (int)i < n_root_sets; ++i) { - soft_set_grungy_pages(GC_static_roots[i].r_start, - GC_static_roots[i].r_end, - (int)i < n_root_sets - 1 ? GC_static_roots[i + 1].r_start : NULL); - } -#endif - } - - /* Clear soft-dirty bits from the task's PTEs. */ - res = write(clear_refs_fd, "4\n", 2); - if (res != 2) - ABORT_ARG1("Failed to write to /proc/self/clear_refs", - ": errno= %d", res < 0 ? errno : 0); -} -#endif /* SOFT_VDB */ - -#ifdef PCR_VDB - -#include "vd/PCR_VD.h" - -#define NPAGES (32 * 1024) /* 128 MB */ - -PCR_VD_DB GC_grungy_bits[NPAGES]; - -STATIC ptr_t GC_vd_base = NULL; -/* Address corresponding to GC_grungy_bits[0] */ -/* HBLKSIZE aligned. */ - -GC_INNER GC_bool GC_dirty_init(void) { - /* For the time being, we assume the heap generally grows up */ - GC_vd_base = GC_heap_sects[0].hs_start; - if (GC_vd_base == 0) { - ABORT("Bad initial heap segment"); - } - if (PCR_VD_Start(HBLKSIZE, GC_vd_base, NPAGES * HBLKSIZE) != PCR_ERes_okay) { - ABORT("Dirty bit initialization failed"); - } - return TRUE; -} -#endif /* PCR_VDB */ - -#ifndef GC_DISABLE_INCREMENTAL -GC_INNER GC_bool GC_manual_vdb = FALSE; - -/* Manually mark the page containing p as dirty. Logically, this */ -/* dirties the entire object. */ -GC_INNER void GC_dirty_inner(const void *p) { - word index = PHT_HASH(p); - -#if defined(MPROTECT_VDB) - /* Do not update GC_dirty_pages if it should be followed by the */ - /* page unprotection. */ - GC_ASSERT(GC_manual_vdb); -#endif - async_set_pht_entry_from_index(GC_dirty_pages, index); -} - -/* Retrieve system dirty bits for the heap to a local buffer (unless */ -/* output_unneeded). Restore the systems notion of which pages are */ -/* dirty. We assume that either the world is stopped or it is OK to */ -/* lose dirty bits while it's happening (as in GC_enable_incremental).*/ -GC_INNER void GC_read_dirty(GC_bool output_unneeded) { - GC_ASSERT(I_HOLD_LOCK()); - if (GC_manual_vdb -#if defined(MPROTECT_VDB) - || !GC_GWW_AVAILABLE() -#endif - ) { - if (!output_unneeded) - BCOPY((/* no volatile */ void *)GC_dirty_pages, GC_grungy_pages, - sizeof(GC_dirty_pages)); - BZERO((/* no volatile */ void *)GC_dirty_pages, - sizeof(GC_dirty_pages)); -#ifdef MPROTECT_VDB - if (!GC_manual_vdb) - GC_protect_heap(); -#endif - return; - } - -#ifdef GWW_VDB - GC_gww_read_dirty(output_unneeded); -#elif defined(PROC_VDB) - GC_proc_read_dirty(output_unneeded); -#elif defined(SOFT_VDB) - GC_soft_read_dirty(output_unneeded); -#elif defined(PCR_VDB) - /* lazily enable dirty bits on newly added heap sects */ - { - static int onhs = 0; - int nhs = GC_n_heap_sects; - for (; onhs < nhs; onhs++) { - PCR_VD_WriteProtectEnable( - GC_heap_sects[onhs].hs_start, - GC_heap_sects[onhs].hs_bytes); - } - } - if (PCR_VD_Clear(GC_vd_base, NPAGES * HBLKSIZE, GC_grungy_bits) != PCR_ERes_okay) { - ABORT("Dirty bit read failed"); - } -#endif -} - -#if !defined(NO_VDB_FOR_STATIC_ROOTS) && !defined(PROC_VDB) -GC_INNER GC_bool GC_is_vdb_for_static_roots(void) { - if (GC_manual_vdb) return FALSE; -#if defined(MPROTECT_VDB) - /* Currently used only in conjunction with SOFT_VDB. */ - return GC_GWW_AVAILABLE(); -#else - GC_ASSERT(GC_incremental); - return TRUE; -#endif -} -#endif - -/* Is the HBLKSIZE sized page at h marked dirty in the local buffer? */ -/* If the actual page size is different, this returns TRUE if any */ -/* of the pages overlapping h are dirty. This routine may err on the */ -/* side of labeling pages as dirty (and this implementation does). */ -GC_INNER GC_bool GC_page_was_dirty(struct hblk *h) { - word index; - -#ifdef PCR_VDB - if (!GC_manual_vdb) { - if ((word)h < (word)GC_vd_base || (word)h >= (word)(GC_vd_base + NPAGES * HBLKSIZE)) { - return TRUE; - } - return GC_grungy_bits[h - (struct hblk *)GC_vd_base] & PCR_VD_DB_dirtyBit; - } -#elif defined(DEFAULT_VDB) - if (!GC_manual_vdb) - return TRUE; -#elif defined(PROC_VDB) - /* Unless manual VDB is on, the bitmap covers all process memory. */ - if (GC_manual_vdb) -#endif - { - if (NULL == HDR(h)) - return TRUE; - } - index = PHT_HASH(h); - return get_pht_entry_from_index(GC_grungy_pages, index); -} - -#if defined(CHECKSUMS) || defined(PROC_VDB) -/* Could any valid GC heap pointer ever have been written to this page? */ -GC_INNER GC_bool GC_page_was_ever_dirty(struct hblk *h) { -#if defined(GWW_VDB) || defined(PROC_VDB) || defined(SOFT_VDB) - word index; - -#ifdef MPROTECT_VDB - if (!GC_GWW_AVAILABLE()) - return TRUE; -#endif -#if defined(PROC_VDB) - if (GC_manual_vdb) -#endif - { - if (NULL == HDR(h)) - return TRUE; - } - index = PHT_HASH(h); - return get_pht_entry_from_index(GC_written_pages, index); -#else - /* TODO: implement me for MANUAL_VDB. */ - (void)h; - return TRUE; -#endif -} -#endif /* CHECKSUMS || PROC_VDB */ - -/* We expect block h to be written shortly. Ensure that all pages */ -/* containing any part of the n hblks starting at h are no longer */ -/* protected. If is_ptrfree is false, also ensure that they will */ -/* subsequently appear to be dirty. Not allowed to call GC_printf */ -/* (and the friends) here, see Win32 GC_stop_world for the details. */ -GC_INNER void GC_remove_protection(struct hblk *h, word nblocks, - GC_bool is_ptrfree) { -#ifdef PCR_VDB - (void)is_ptrfree; - if (!GC_auto_incremental) - return; - PCR_VD_WriteProtectDisable(h, nblocks * HBLKSIZE); - PCR_VD_WriteProtectEnable(h, nblocks * HBLKSIZE); -#elif defined(MPROTECT_VDB) - struct hblk *h_trunc; /* Truncated to page boundary */ - struct hblk *h_end; /* Page boundary following block end */ - struct hblk *current; - - if (!GC_auto_incremental || GC_GWW_AVAILABLE()) - return; - GC_ASSERT(GC_page_size != 0); - h_trunc = (struct hblk *)((word)h & ~(GC_page_size - 1)); - h_end = (struct hblk *)(((word)(h + nblocks) + GC_page_size - 1) & ~(GC_page_size - 1)); - if (h_end == h_trunc + 1 && - get_pht_entry_from_index(GC_dirty_pages, PHT_HASH(h_trunc))) { - /* already marked dirty, and hence unprotected. */ - return; - } - for (current = h_trunc; (word)current < (word)h_end; ++current) { - word index = PHT_HASH(current); - - if (!is_ptrfree || (word)current < (word)h || (word)current >= (word)(h + nblocks)) { - async_set_pht_entry_from_index(GC_dirty_pages, index); - } - } - UNPROTECT(h_trunc, (ptr_t)h_end - (ptr_t)h_trunc); -#else - /* Ignore write hints. They don't help us here. */ - (void)h; - (void)nblocks; - (void)is_ptrfree; -#endif -} -#endif /* !GC_DISABLE_INCREMENTAL */ - -#if defined(MPROTECT_VDB) && defined(DARWIN) -/* The following sources were used as a "reference" for this exception - handling code: - 1. Apple's mach/xnu documentation - 2. Timothy J. Wood's "Mach Exception Handlers 101" post to the - omnigroup's macosx-dev list. - www.omnigroup.com/mailman/archive/macosx-dev/2000-June/014178.html - 3. macosx-nat.c from Apple's GDB source code. -*/ - -/* The bug that caused all this trouble should now be fixed. This should - eventually be removed if all goes well. */ - -/* #define BROKEN_EXCEPTION_HANDLING */ - -#include -#include -#include -#include -#include - -EXTERN_C_BEGIN - -/* Some of the following prototypes are missing in any header, although */ -/* they are documented. Some are in mach/exc.h file. */ -extern boolean_t -exc_server(mach_msg_header_t *, mach_msg_header_t *); - -extern kern_return_t - exception_raise(mach_port_t, mach_port_t, mach_port_t, exception_type_t, - exception_data_t, mach_msg_type_number_t); - -extern kern_return_t -exception_raise_state(mach_port_t, mach_port_t, mach_port_t, exception_type_t, - exception_data_t, mach_msg_type_number_t, - thread_state_flavor_t *, thread_state_t, - mach_msg_type_number_t, thread_state_t, - mach_msg_type_number_t *); - -extern kern_return_t -exception_raise_state_identity(mach_port_t, mach_port_t, mach_port_t, - exception_type_t, exception_data_t, - mach_msg_type_number_t, thread_state_flavor_t *, - thread_state_t, mach_msg_type_number_t, - thread_state_t, mach_msg_type_number_t *); - -GC_API_OSCALL kern_return_t -catch_exception_raise(mach_port_t exception_port, mach_port_t thread, - mach_port_t task, exception_type_t exception, - exception_data_t code, - mach_msg_type_number_t code_count); - -GC_API_OSCALL kern_return_t -catch_exception_raise_state(mach_port_name_t exception_port, - int exception, exception_data_t code, - mach_msg_type_number_t codeCnt, int flavor, - thread_state_t old_state, int old_stateCnt, - thread_state_t new_state, int new_stateCnt); - -GC_API_OSCALL kern_return_t -catch_exception_raise_state_identity(mach_port_name_t exception_port, - mach_port_t thread, mach_port_t task, int exception, - exception_data_t code, mach_msg_type_number_t codeCnt, - int flavor, thread_state_t old_state, int old_stateCnt, - thread_state_t new_state, int new_stateCnt); - -EXTERN_C_END - -/* These should never be called, but just in case... */ -GC_API_OSCALL kern_return_t -catch_exception_raise_state(mach_port_name_t exception_port, int exception, - exception_data_t code, - mach_msg_type_number_t codeCnt, int flavor, - thread_state_t old_state, int old_stateCnt, - thread_state_t new_state, int new_stateCnt) { - UNUSED_ARG(exception_port); - UNUSED_ARG(exception); - UNUSED_ARG(code); - UNUSED_ARG(codeCnt); - UNUSED_ARG(flavor); - UNUSED_ARG(old_state); - UNUSED_ARG(old_stateCnt); - UNUSED_ARG(new_state); - UNUSED_ARG(new_stateCnt); - ABORT_RET("Unexpected catch_exception_raise_state invocation"); - return KERN_INVALID_ARGUMENT; -} - -GC_API_OSCALL kern_return_t -catch_exception_raise_state_identity(mach_port_name_t exception_port, - mach_port_t thread, mach_port_t task, - int exception, exception_data_t code, - mach_msg_type_number_t codeCnt, - int flavor, thread_state_t old_state, - int old_stateCnt, - thread_state_t new_state, - int new_stateCnt) { - UNUSED_ARG(exception_port); - UNUSED_ARG(thread); - UNUSED_ARG(task); - UNUSED_ARG(exception); - UNUSED_ARG(code); - UNUSED_ARG(codeCnt); - UNUSED_ARG(flavor); - UNUSED_ARG(old_state); - UNUSED_ARG(old_stateCnt); - UNUSED_ARG(new_state); - UNUSED_ARG(new_stateCnt); - ABORT_RET("Unexpected catch_exception_raise_state_identity invocation"); - return KERN_INVALID_ARGUMENT; -} - -#define MAX_EXCEPTION_PORTS 16 - -static struct { - mach_msg_type_number_t count; - exception_mask_t masks[MAX_EXCEPTION_PORTS]; - exception_handler_t ports[MAX_EXCEPTION_PORTS]; - exception_behavior_t behaviors[MAX_EXCEPTION_PORTS]; - thread_state_flavor_t flavors[MAX_EXCEPTION_PORTS]; -} GC_old_exc_ports; - -STATIC struct ports_s { - void (*volatile os_callback[3])(void); - mach_port_t exception; -#if defined(THREADS) - mach_port_t reply; -#endif -} GC_ports = { - {/* This is to prevent stripping these routines as dead. */ - (void (*)(void))catch_exception_raise, - (void (*)(void))catch_exception_raise_state, - (void (*)(void))catch_exception_raise_state_identity}, -#ifdef THREADS - 0, /* for 'exception' */ -#endif - 0}; - -typedef struct { - mach_msg_header_t head; -} GC_msg_t; - -typedef enum { - GC_MP_NORMAL, - GC_MP_DISCARDING, - GC_MP_STOPPED -} GC_mprotect_state_t; - -#ifdef THREADS -/* FIXME: 1 and 2 seem to be safe to use in the msgh_id field, but it */ -/* is not documented. Use the source and see if they should be OK. */ -#define ID_STOP 1 -#define ID_RESUME 2 - -/* This value is only used on the reply port. */ -#define ID_ACK 3 - -STATIC GC_mprotect_state_t GC_mprotect_state = GC_MP_NORMAL; - -/* The following should ONLY be called when the world is stopped. */ -STATIC void GC_mprotect_thread_notify(mach_msg_id_t id) { - struct buf_s { - GC_msg_t msg; - mach_msg_trailer_t trailer; - } buf; - mach_msg_return_t r; - - /* remote, local */ - buf.msg.head.msgh_bits = MACH_MSGH_BITS(MACH_MSG_TYPE_MAKE_SEND, 0); - buf.msg.head.msgh_size = sizeof(buf.msg); - buf.msg.head.msgh_remote_port = GC_ports.exception; - buf.msg.head.msgh_local_port = MACH_PORT_NULL; - buf.msg.head.msgh_id = id; - - r = mach_msg(&buf.msg.head, MACH_SEND_MSG | MACH_RCV_MSG | MACH_RCV_LARGE, - sizeof(buf.msg), sizeof(buf), GC_ports.reply, - MACH_MSG_TIMEOUT_NONE, MACH_PORT_NULL); - if (r != MACH_MSG_SUCCESS) - ABORT("mach_msg failed in GC_mprotect_thread_notify"); - if (buf.msg.head.msgh_id != ID_ACK) - ABORT("Invalid ack in GC_mprotect_thread_notify"); -} - -/* Should only be called by the mprotect thread */ -STATIC void GC_mprotect_thread_reply(void) { - GC_msg_t msg; - mach_msg_return_t r; - /* remote, local */ - - msg.head.msgh_bits = MACH_MSGH_BITS(MACH_MSG_TYPE_MAKE_SEND, 0); - msg.head.msgh_size = sizeof(msg); - msg.head.msgh_remote_port = GC_ports.reply; - msg.head.msgh_local_port = MACH_PORT_NULL; - msg.head.msgh_id = ID_ACK; - - r = mach_msg(&msg.head, MACH_SEND_MSG, sizeof(msg), 0, MACH_PORT_NULL, - MACH_MSG_TIMEOUT_NONE, MACH_PORT_NULL); - if (r != MACH_MSG_SUCCESS) - ABORT("mach_msg failed in GC_mprotect_thread_reply"); -} - -GC_INNER void GC_mprotect_stop(void) { - GC_mprotect_thread_notify(ID_STOP); -} - -GC_INNER void GC_mprotect_resume(void) { - GC_mprotect_thread_notify(ID_RESUME); -} - -#else -/* The compiler should optimize away any GC_mprotect_state computations */ -#define GC_mprotect_state GC_MP_NORMAL -#endif /* !THREADS */ - -struct mp_reply_s { - mach_msg_header_t head; - char data[256]; -}; - -struct mp_msg_s { - mach_msg_header_t head; - mach_msg_body_t msgh_body; - char data[1024]; -}; - -STATIC void *GC_mprotect_thread(void *arg) { - mach_msg_return_t r; - /* These two structures contain some private kernel data. We don't */ - /* need to access any of it so we don't bother defining a proper */ - /* struct. The correct definitions are in the xnu source code. */ - struct mp_reply_s reply; - struct mp_msg_s msg; - mach_msg_id_t id; - - if ((word)arg == GC_WORD_MAX) return 0; /* to prevent a compiler warning */ -#if defined(CPPCHECK) - reply.data[0] = 0; /* to prevent "field unused" warnings */ - msg.data[0] = 0; -#endif - -#if defined(HAVE_PTHREAD_SETNAME_NP_WITHOUT_TID) - (void)pthread_setname_np("GC-mprotect"); -#endif -#if defined(THREADS) && !defined(GC_NO_THREADS_DISCOVERY) - GC_darwin_register_self_mach_handler(); -#endif - - for (;;) { - r = mach_msg(&msg.head, MACH_RCV_MSG | MACH_RCV_LARGE | (GC_mprotect_state == GC_MP_DISCARDING ? MACH_RCV_TIMEOUT : 0), - 0, sizeof(msg), GC_ports.exception, - GC_mprotect_state == GC_MP_DISCARDING ? 0 - : MACH_MSG_TIMEOUT_NONE, - MACH_PORT_NULL); - id = r == MACH_MSG_SUCCESS ? msg.head.msgh_id : -1; - -#if defined(THREADS) - if (GC_mprotect_state == GC_MP_DISCARDING) { - if (r == MACH_RCV_TIMED_OUT) { - GC_mprotect_state = GC_MP_STOPPED; - GC_mprotect_thread_reply(); - continue; - } - if (r == MACH_MSG_SUCCESS && (id == ID_STOP || id == ID_RESUME)) - ABORT("Out of order mprotect thread request"); - } -#endif /* THREADS */ - - if (r != MACH_MSG_SUCCESS) { - ABORT_ARG2("mach_msg failed", - ": errcode= %d (%s)", (int)r, mach_error_string(r)); - } - - switch (id) { -#if defined(THREADS) - case ID_STOP: - if (GC_mprotect_state != GC_MP_NORMAL) - ABORT("Called mprotect_stop when state wasn't normal"); - GC_mprotect_state = GC_MP_DISCARDING; - break; - case ID_RESUME: - if (GC_mprotect_state != GC_MP_STOPPED) - ABORT("Called mprotect_resume when state wasn't stopped"); - GC_mprotect_state = GC_MP_NORMAL; - GC_mprotect_thread_reply(); - break; -#endif /* THREADS */ - default: - /* Handle the message (calls catch_exception_raise) */ - if (!exc_server(&msg.head, &reply.head)) - ABORT("exc_server failed"); - /* Send the reply */ - r = mach_msg(&reply.head, MACH_SEND_MSG, reply.head.msgh_size, 0, - MACH_PORT_NULL, MACH_MSG_TIMEOUT_NONE, - MACH_PORT_NULL); - if (r != MACH_MSG_SUCCESS) { - /* This will fail if the thread dies, but the thread */ - /* shouldn't die... */ -#ifdef BROKEN_EXCEPTION_HANDLING - GC_err_printf( - "mach_msg failed with %d %s while sending " - "exc reply\n", - (int)r, mach_error_string(r)); -#else - ABORT("mach_msg failed while sending exception reply"); -#endif - } - } /* switch */ - } /* for(;;) */ -} - -/* All this SIGBUS code shouldn't be necessary. All protection faults should - be going through the mach exception handler. However, it seems a SIGBUS is - occasionally sent for some unknown reason. Even more odd, it seems to be - meaningless and safe to ignore. */ -#ifdef BROKEN_EXCEPTION_HANDLING - -/* Updates to this aren't atomic, but the SIGBUS'es seem pretty rare. */ -/* Even if this doesn't get updated property, it isn't really a problem. */ -STATIC int GC_sigbus_count = 0; - -STATIC void GC_darwin_sigbus(int num, siginfo_t *sip, void *context) { - if (num != SIGBUS) - ABORT("Got a non-sigbus signal in the sigbus handler"); - - /* Ugh... some seem safe to ignore, but too many in a row probably means - trouble. GC_sigbus_count is reset for each mach exception that is - handled */ - if (GC_sigbus_count >= 8) { - ABORT("Got more than 8 SIGBUSs in a row!"); - } else { - GC_sigbus_count++; - WARN("Ignoring SIGBUS\n", 0); - } -} -#endif /* BROKEN_EXCEPTION_HANDLING */ - -GC_INNER GC_bool GC_dirty_init(void) { - kern_return_t r; - mach_port_t me; - pthread_t thread; - pthread_attr_t attr; - exception_mask_t mask; - - GC_ASSERT(I_HOLD_LOCK()); -#ifdef CAN_HANDLE_FORK - if (GC_handle_fork) { - /* To both support GC incremental mode and GC functions usage in */ - /* the forked child, pthread_atfork should be used to install */ - /* handlers that switch off GC_incremental in the child */ - /* gracefully (unprotecting all pages and clearing */ - /* GC_mach_handler_thread). For now, we just disable incremental */ - /* mode if fork() handling is requested by the client. */ - WARN( - "Can't turn on GC incremental mode as fork()" - " handling requested\n", - 0); - return FALSE; - } -#endif - - GC_VERBOSE_LOG_PRINTF( - "Initializing mach/darwin mprotect" - " virtual dirty bit implementation\n"); -#ifdef BROKEN_EXCEPTION_HANDLING - WARN("Enabling workarounds for various darwin exception handling bugs\n", - 0); -#endif - if (GC_page_size % HBLKSIZE != 0) { - ABORT("Page size not multiple of HBLKSIZE"); - } - - GC_task_self = me = mach_task_self(); - - r = mach_port_allocate(me, MACH_PORT_RIGHT_RECEIVE, &GC_ports.exception); - /* TODO: WARN and return FALSE in case of a failure. */ - if (r != KERN_SUCCESS) - ABORT("mach_port_allocate failed (exception port)"); - - r = mach_port_insert_right(me, GC_ports.exception, GC_ports.exception, - MACH_MSG_TYPE_MAKE_SEND); - if (r != KERN_SUCCESS) - ABORT("mach_port_insert_right failed (exception port)"); - -#if defined(THREADS) - r = mach_port_allocate(me, MACH_PORT_RIGHT_RECEIVE, &GC_ports.reply); - if (r != KERN_SUCCESS) - ABORT("mach_port_allocate failed (reply port)"); -#endif - - /* The exceptions we want to catch */ - mask = EXC_MASK_BAD_ACCESS; - - r = task_get_exception_ports(me, mask, GC_old_exc_ports.masks, - &GC_old_exc_ports.count, GC_old_exc_ports.ports, - GC_old_exc_ports.behaviors, - GC_old_exc_ports.flavors); - if (r != KERN_SUCCESS) - ABORT("task_get_exception_ports failed"); - - r = task_set_exception_ports(me, mask, GC_ports.exception, EXCEPTION_DEFAULT, - GC_MACH_THREAD_STATE); - if (r != KERN_SUCCESS) - ABORT("task_set_exception_ports failed"); - if (pthread_attr_init(&attr) != 0) - ABORT("pthread_attr_init failed"); - if (pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED) != 0) - ABORT("pthread_attr_setdetachedstate failed"); - -#undef pthread_create - /* This will call the real pthread function, not our wrapper */ - if (pthread_create(&thread, &attr, GC_mprotect_thread, NULL) != 0) - ABORT("pthread_create failed"); - (void)pthread_attr_destroy(&attr); - - /* Setup the sigbus handler for ignoring the meaningless SIGBUSs */ -#ifdef BROKEN_EXCEPTION_HANDLING - { - struct sigaction sa, oldsa; - sa.sa_handler = (SIG_HNDLR_PTR)GC_darwin_sigbus; - sigemptyset(&sa.sa_mask); - sa.sa_flags = SA_RESTART | SA_SIGINFO; - /* sa.sa_restorer is deprecated and should not be initialized. */ - if (sigaction(SIGBUS, &sa, &oldsa) < 0) - ABORT("sigaction failed"); - if (oldsa.sa_handler != (SIG_HNDLR_PTR)(signed_word)SIG_DFL) { - GC_VERBOSE_LOG_PRINTF("Replaced other SIGBUS handler\n"); - } - } -#endif /* BROKEN_EXCEPTION_HANDLING */ -#if defined(CPPCHECK) - GC_noop1((word)GC_ports.os_callback[0]); -#endif - return TRUE; -} - -/* The source code for Apple's GDB was used as a reference for the */ -/* exception forwarding code. This code is similar to be GDB code only */ -/* because there is only one way to do it. */ -STATIC kern_return_t GC_forward_exception(mach_port_t thread, mach_port_t task, - exception_type_t exception, - exception_data_t data, - mach_msg_type_number_t data_count) { - unsigned int i; - kern_return_t r; - mach_port_t port; - exception_behavior_t behavior; - thread_state_flavor_t flavor; - - thread_state_data_t thread_state; - mach_msg_type_number_t thread_state_count = THREAD_STATE_MAX; - - for (i = 0; i < GC_old_exc_ports.count; i++) - if (GC_old_exc_ports.masks[i] & (1 << exception)) - break; - if (i == GC_old_exc_ports.count) - ABORT("No handler for exception!"); - - port = GC_old_exc_ports.ports[i]; - behavior = GC_old_exc_ports.behaviors[i]; - flavor = GC_old_exc_ports.flavors[i]; - - if (behavior == EXCEPTION_STATE || behavior == EXCEPTION_STATE_IDENTITY) { - r = thread_get_state(thread, flavor, thread_state, &thread_state_count); - if (r != KERN_SUCCESS) - ABORT("thread_get_state failed in forward_exception"); - } - - switch (behavior) { - case EXCEPTION_STATE: - r = exception_raise_state(port, thread, task, exception, data, data_count, - &flavor, thread_state, thread_state_count, - thread_state, &thread_state_count); - break; - case EXCEPTION_STATE_IDENTITY: - r = exception_raise_state_identity(port, thread, task, exception, data, - data_count, &flavor, thread_state, - thread_state_count, thread_state, - &thread_state_count); - break; - /* case EXCEPTION_DEFAULT: */ /* default signal handlers */ - default: /* user-supplied signal handlers */ - r = exception_raise(port, thread, task, exception, data, data_count); - } - - if (behavior == EXCEPTION_STATE || behavior == EXCEPTION_STATE_IDENTITY) { - r = thread_set_state(thread, flavor, thread_state, thread_state_count); - if (r != KERN_SUCCESS) - ABORT("thread_set_state failed in forward_exception"); - } - return r; -} - -#define FWD() GC_forward_exception(thread, task, exception, code, code_count) - -#ifdef ARM32 -#define DARWIN_EXC_STATE ARM_EXCEPTION_STATE -#define DARWIN_EXC_STATE_COUNT ARM_EXCEPTION_STATE_COUNT -#define DARWIN_EXC_STATE_T arm_exception_state_t -#define DARWIN_EXC_STATE_DAR THREAD_FLD_NAME(far) -#elif defined(AARCH64) -#define DARWIN_EXC_STATE ARM_EXCEPTION_STATE64 -#define DARWIN_EXC_STATE_COUNT ARM_EXCEPTION_STATE64_COUNT -#define DARWIN_EXC_STATE_T arm_exception_state64_t -#define DARWIN_EXC_STATE_DAR THREAD_FLD_NAME(far) -#elif defined(POWERPC) -#if CPP_WORDSZ == 32 -#define DARWIN_EXC_STATE PPC_EXCEPTION_STATE -#define DARWIN_EXC_STATE_COUNT PPC_EXCEPTION_STATE_COUNT -#define DARWIN_EXC_STATE_T ppc_exception_state_t -#else -#define DARWIN_EXC_STATE PPC_EXCEPTION_STATE64 -#define DARWIN_EXC_STATE_COUNT PPC_EXCEPTION_STATE64_COUNT -#define DARWIN_EXC_STATE_T ppc_exception_state64_t -#endif -#define DARWIN_EXC_STATE_DAR THREAD_FLD_NAME(dar) -#elif defined(I386) || defined(X86_64) -#if CPP_WORDSZ == 32 -#if defined(i386_EXCEPTION_STATE_COUNT) && !defined(x86_EXCEPTION_STATE32_COUNT) -/* Use old naming convention for 32-bit x86. */ -#define DARWIN_EXC_STATE i386_EXCEPTION_STATE -#define DARWIN_EXC_STATE_COUNT i386_EXCEPTION_STATE_COUNT -#define DARWIN_EXC_STATE_T i386_exception_state_t -#else -#define DARWIN_EXC_STATE x86_EXCEPTION_STATE32 -#define DARWIN_EXC_STATE_COUNT x86_EXCEPTION_STATE32_COUNT -#define DARWIN_EXC_STATE_T x86_exception_state32_t -#endif -#else -#define DARWIN_EXC_STATE x86_EXCEPTION_STATE64 -#define DARWIN_EXC_STATE_COUNT x86_EXCEPTION_STATE64_COUNT -#define DARWIN_EXC_STATE_T x86_exception_state64_t -#endif -#define DARWIN_EXC_STATE_DAR THREAD_FLD_NAME(faultvaddr) -#elif !defined(CPPCHECK) -#error FIXME for non-arm/ppc/x86 darwin -#endif - -/* This violates the namespace rules but there isn't anything that can */ -/* be done about it. The exception handling stuff is hard coded to */ -/* call this. catch_exception_raise, catch_exception_raise_state and */ -/* and catch_exception_raise_state_identity are called from OS. */ -GC_API_OSCALL kern_return_t -catch_exception_raise(mach_port_t exception_port, mach_port_t thread, - mach_port_t task, exception_type_t exception, - exception_data_t code, mach_msg_type_number_t code_count) { - kern_return_t r; - char *addr; - thread_state_flavor_t flavor = DARWIN_EXC_STATE; - mach_msg_type_number_t exc_state_count = DARWIN_EXC_STATE_COUNT; - DARWIN_EXC_STATE_T exc_state; - - UNUSED_ARG(exception_port); - UNUSED_ARG(task); - if (exception != EXC_BAD_ACCESS || code[0] != KERN_PROTECTION_FAILURE) { -#ifdef DEBUG_EXCEPTION_HANDLING - /* We aren't interested, pass it on to the old handler */ - GC_log_printf("Exception: 0x%x Code: 0x%x 0x%x in catch...\n", - exception, code_count > 0 ? code[0] : -1, - code_count > 1 ? code[1] : -1); -#else - UNUSED_ARG(code_count); -#endif - return FWD(); - } - - r = thread_get_state(thread, flavor, (natural_t *)&exc_state, - &exc_state_count); - if (r != KERN_SUCCESS) { - /* The thread is supposed to be suspended while the exception */ - /* handler is called. This shouldn't fail. */ -#ifdef BROKEN_EXCEPTION_HANDLING - GC_err_printf("thread_get_state failed in catch_exception_raise\n"); - return KERN_SUCCESS; -#else - ABORT("thread_get_state failed in catch_exception_raise"); -#endif - } - - /* This is the address that caused the fault */ - addr = (char *)exc_state.DARWIN_EXC_STATE_DAR; - if (!is_header_found_async(addr)) { - /* Ugh... just like the SIGBUS problem above, it seems we get */ - /* a bogus KERN_PROTECTION_FAILURE every once and a while. We wait */ - /* till we get a bunch in a row before doing anything about it. */ - /* If a "real" fault ever occurs it'll just keep faulting over and */ - /* over and we'll hit the limit pretty quickly. */ -#ifdef BROKEN_EXCEPTION_HANDLING - static char *last_fault; - static int last_fault_count; - - if (addr != last_fault) { - last_fault = addr; - last_fault_count = 0; - } - if (++last_fault_count < 32) { - if (last_fault_count == 1) - WARN("Ignoring KERN_PROTECTION_FAILURE at %p\n", addr); - return KERN_SUCCESS; - } - - GC_err_printf("Unexpected KERN_PROTECTION_FAILURE at %p; aborting...\n", - (void *)addr); - /* Can't pass it along to the signal handler because that is */ - /* ignoring SIGBUS signals. We also shouldn't call ABORT here as */ - /* signals don't always work too well from the exception handler. */ - EXIT(); -#else /* BROKEN_EXCEPTION_HANDLING */ - /* Pass it along to the next exception handler - (which should call SIGBUS/SIGSEGV) */ - return FWD(); -#endif /* !BROKEN_EXCEPTION_HANDLING */ - } - -#ifdef BROKEN_EXCEPTION_HANDLING - /* Reset the number of consecutive SIGBUSs */ - GC_sigbus_count = 0; -#endif - - GC_ASSERT(GC_page_size != 0); - if (GC_mprotect_state == GC_MP_NORMAL) { /* common case */ - struct hblk *h = (struct hblk *)((word)addr & ~(GC_page_size - 1)); - size_t i; - - UNPROTECT(h, GC_page_size); - for (i = 0; i < divHBLKSZ(GC_page_size); i++) { - word index = PHT_HASH(h + i); - async_set_pht_entry_from_index(GC_dirty_pages, index); - } - } else if (GC_mprotect_state == GC_MP_DISCARDING) { - /* Lie to the thread for now. No sense UNPROTECT()ing the memory - when we're just going to PROTECT() it again later. The thread - will just fault again once it resumes */ - } else { - /* Shouldn't happen, i don't think */ - GC_err_printf("KERN_PROTECTION_FAILURE while world is stopped\n"); - return FWD(); - } - return KERN_SUCCESS; -} -#undef FWD - -#ifndef NO_DESC_CATCH_EXCEPTION_RAISE -/* These symbols should have REFERENCED_DYNAMICALLY (0x10) bit set to */ -/* let strip know they are not to be stripped. */ -__asm__(".desc _catch_exception_raise, 0x10"); -__asm__(".desc _catch_exception_raise_state, 0x10"); -__asm__(".desc _catch_exception_raise_state_identity, 0x10"); -#endif - -#endif /* DARWIN && MPROTECT_VDB */ - -#ifndef HAVE_INCREMENTAL_PROTECTION_NEEDS -GC_API int GC_CALL GC_incremental_protection_needs(void) { - GC_ASSERT(GC_is_initialized); - return GC_PROTECTS_NONE; -} -#endif /* !HAVE_INCREMENTAL_PROTECTION_NEEDS */ - -#ifdef ECOS -/* Undo sbrk() redirection. */ -#undef sbrk -#endif - -/* If value is non-zero then allocate executable memory. */ -GC_API void GC_CALL GC_set_pages_executable(int value) { - GC_ASSERT(!GC_is_initialized); - /* Even if IGNORE_PAGES_EXECUTABLE is defined, GC_pages_executable is */ - /* touched here to prevent a compiler warning. */ - GC_pages_executable = (GC_bool)(value != 0); -} - -/* Returns non-zero if the GC-allocated memory is executable. */ -/* GC_get_pages_executable is defined after all the places */ -/* where GC_get_pages_executable is undefined. */ -GC_API int GC_CALL GC_get_pages_executable(void) { -#ifdef IGNORE_PAGES_EXECUTABLE - return 1; /* Always allocate executable memory. */ -#else - return (int)GC_pages_executable; -#endif -} - -/* Call stack save code for debugging. Should probably be in */ -/* mach_dep.c, but that requires reorganization. */ - -/* I suspect the following works for most *nix x86 variants, so */ -/* long as the frame pointer is explicitly stored. In the case of gcc, */ -/* compiler flags (e.g. -fomit-frame-pointer) determine whether it is. */ -#if defined(I386) && defined(LINUX) && defined(SAVE_CALL_CHAIN) -struct frame { - struct frame *fr_savfp; - long fr_savpc; -#if NARGS > 0 - long fr_arg[NARGS]; /* All the arguments go here. */ -#endif -}; -#endif - -#if defined(SPARC) -#if defined(LINUX) -#if defined(SAVE_CALL_CHAIN) -struct frame { - long fr_local[8]; - long fr_arg[6]; - struct frame *fr_savfp; - long fr_savpc; -#ifndef __arch64__ - char *fr_stret; -#endif - long fr_argd[6]; - long fr_argx[0]; -}; -#endif -#elif defined(DRSNX) -#include -#elif defined(OPENBSD) -#include -#elif defined(FREEBSD) || defined(NETBSD) -#include -#else -#include -#endif -#if NARGS > 6 -#error We only know how to get the first 6 arguments -#endif -#endif /* SPARC */ - -#ifdef NEED_CALLINFO -/* Fill in the pc and argument information for up to NFRAMES of my */ -/* callers. Ignore my frame and my callers frame. */ -#ifdef LINUX -#include -#endif -#endif /* NEED_CALLINFO */ - -#if defined(GC_HAVE_BUILTIN_BACKTRACE) -#ifdef _MSC_VER -EXTERN_C_BEGIN -int backtrace(void *addresses[], int count); -char **backtrace_symbols(void *const addresses[], int count); -EXTERN_C_END -#else -#include -#endif -#endif /* GC_HAVE_BUILTIN_BACKTRACE */ - -#ifdef SAVE_CALL_CHAIN - -#if NARGS == 0 && NFRAMES % 2 == 0 /* No padding */ \ - && defined(GC_HAVE_BUILTIN_BACKTRACE) - -#ifdef REDIRECT_MALLOC -/* Deal with possible malloc calls in backtrace by omitting */ -/* the infinitely recursing backtrace. */ -#ifdef THREADS -__thread /* If your compiler doesn't understand this */ - /* you could use something like pthread_getspecific. */ -#endif - GC_bool GC_in_save_callers = FALSE; -#endif - -GC_INNER void GC_save_callers(struct callinfo info[NFRAMES]) { - void *tmp_info[NFRAMES + 1]; - int npcs, i; -#define IGNORE_FRAMES 1 - - /* We retrieve NFRAMES+1 pc values, but discard the first, since it */ - /* points to our own frame. */ -#ifdef REDIRECT_MALLOC - if (GC_in_save_callers) { - info[0].ci_pc = (word)(&GC_save_callers); - for (i = 1; i < NFRAMES; ++i) info[i].ci_pc = 0; - return; - } - GC_in_save_callers = TRUE; -#endif - - GC_ASSERT(I_HOLD_LOCK()); - /* backtrace may call dl_iterate_phdr which is also */ - /* used by GC_register_dynamic_libraries, and */ - /* dl_iterate_phdr is not guaranteed to be reentrant. */ - - GC_STATIC_ASSERT(sizeof(struct callinfo) == sizeof(void *)); - npcs = backtrace((void **)tmp_info, NFRAMES + IGNORE_FRAMES); - if (npcs > IGNORE_FRAMES) - BCOPY(&tmp_info[IGNORE_FRAMES], info, - (npcs - IGNORE_FRAMES) * sizeof(void *)); - for (i = npcs - IGNORE_FRAMES; i < NFRAMES; ++i) info[i].ci_pc = 0; -#ifdef REDIRECT_MALLOC - GC_in_save_callers = FALSE; -#endif -} - -#else /* No builtin backtrace; do it ourselves */ - -#if (defined(OPENBSD) || defined(NETBSD) || defined(FREEBSD)) && defined(SPARC) -#define FR_SAVFP fr_fp -#define FR_SAVPC fr_pc -#else -#define FR_SAVFP fr_savfp -#define FR_SAVPC fr_savpc -#endif - -#if defined(SPARC) && (defined(__arch64__) || defined(__sparcv9)) -#define BIAS 2047 -#else -#define BIAS 0 -#endif - -GC_INNER void GC_save_callers(struct callinfo info[NFRAMES]) { - struct frame *frame; - struct frame *fp; - int nframes = 0; -#ifdef I386 - /* We assume this is turned on only with gcc as the compiler. */ - asm("movl %%ebp,%0" - : "=r"(frame)); - fp = frame; -#else - frame = (struct frame *)GC_save_regs_in_stack(); - fp = (struct frame *)((long)frame->FR_SAVFP + BIAS); -#endif - - for (; !((word)fp HOTTER_THAN(word) frame) -#ifndef THREADS - && !((word)GC_stackbottom HOTTER_THAN(word) fp) -#elif defined(STACK_GROWS_UP) - && fp != NULL -#endif - && nframes < NFRAMES; - fp = (struct frame *)((long)fp->FR_SAVFP + BIAS), nframes++) { -#if NARGS > 0 - int i; -#endif - - info[nframes].ci_pc = fp->FR_SAVPC; -#if NARGS > 0 - for (i = 0; i < NARGS; i++) { - info[nframes].ci_arg[i] = ~(fp->fr_arg[i]); - } -#endif /* NARGS > 0 */ - } - if (nframes < NFRAMES) info[nframes].ci_pc = 0; -} - -#endif /* No builtin backtrace */ - -#endif /* SAVE_CALL_CHAIN */ - -#ifdef NEED_CALLINFO - -/* Print info to stderr. We do NOT hold the allocation lock. */ -GC_INNER void GC_print_callers(struct callinfo info[NFRAMES]) { - int i; - static int reentry_count = 0; - - /* FIXME: This should probably use a different lock, so that we */ - /* become callable with or without the allocation lock. */ - LOCK(); - ++reentry_count; - UNLOCK(); - -#if NFRAMES == 1 - GC_err_printf("\tCaller at allocation:\n"); -#else - GC_err_printf("\tCall chain at allocation:\n"); -#endif - for (i = 0; i < NFRAMES; i++) { -#if defined(LINUX) && !defined(SMALL_CONFIG) - GC_bool stop = FALSE; -#endif - - if (0 == info[i].ci_pc) - break; -#if NARGS > 0 - { - int j; - - GC_err_printf("\t\targs: "); - for (j = 0; j < NARGS; j++) { - if (j != 0) GC_err_printf(", "); - GC_err_printf("%d (0x%X)", ~(info[i].ci_arg[j]), - ~(info[i].ci_arg[j])); - } - GC_err_printf("\n"); - } -#endif - if (reentry_count > 1) { - /* We were called during an allocation during */ - /* a previous GC_print_callers call; punt. */ - GC_err_printf("\t\t##PC##= 0x%lx\n", - (unsigned long)info[i].ci_pc); - continue; - } - { - char buf[40]; - char *name; -#if defined(GC_HAVE_BUILTIN_BACKTRACE) && !defined(GC_BACKTRACE_SYMBOLS_BROKEN) - char **sym_name = - backtrace_symbols((void **)(&(info[i].ci_pc)), 1); - if (sym_name != NULL) { - name = sym_name[0]; - } else -#endif - /* else */ { - (void)snprintf(buf, sizeof(buf), "##PC##= 0x%lx", - (unsigned long)info[i].ci_pc); - buf[sizeof(buf) - 1] = '\0'; - name = buf; - } -#if defined(LINUX) && !defined(SMALL_CONFIG) - /* Try for a line number. */ - do { - FILE *pipe; -#define EXE_SZ 100 - static char exe_name[EXE_SZ]; -#define CMD_SZ 200 - char cmd_buf[CMD_SZ]; -#define RESULT_SZ 200 - static char result_buf[RESULT_SZ]; - size_t result_len; - char *old_preload; -#define PRELOAD_SZ 200 - char preload_buf[PRELOAD_SZ]; - static GC_bool found_exe_name = FALSE; - static GC_bool will_fail = FALSE; - - /* Try to get it via a hairy and expensive scheme. */ - /* First we get the name of the executable: */ - if (will_fail) - break; - if (!found_exe_name) { - int ret_code = readlink("/proc/self/exe", exe_name, EXE_SZ); - - if (ret_code < 0 || ret_code >= EXE_SZ || exe_name[0] != '/') { - will_fail = TRUE; /* Don't try again. */ - break; - } - exe_name[ret_code] = '\0'; - found_exe_name = TRUE; - } - /* Then we use popen to start addr2line -e */ - /* There are faster ways to do this, but hopefully this */ - /* isn't time critical. */ - (void)snprintf(cmd_buf, sizeof(cmd_buf), - "/usr/bin/addr2line -f -e %s 0x%lx", - exe_name, (unsigned long)info[i].ci_pc); - cmd_buf[sizeof(cmd_buf) - 1] = '\0'; - old_preload = GETENV("LD_PRELOAD"); - if (0 != old_preload) { - size_t old_len = strlen(old_preload); - if (old_len >= PRELOAD_SZ) { - will_fail = TRUE; - break; - } - BCOPY(old_preload, preload_buf, old_len + 1); - unsetenv("LD_PRELOAD"); - } - pipe = popen(cmd_buf, "r"); - if (0 != old_preload && 0 != setenv("LD_PRELOAD", preload_buf, 0)) { - WARN("Failed to reset LD_PRELOAD\n", 0); - } - if (NULL == pipe) { - will_fail = TRUE; - break; - } - result_len = fread(result_buf, 1, RESULT_SZ - 1, pipe); - (void)pclose(pipe); - if (0 == result_len) { - will_fail = TRUE; - break; - } - if (result_buf[result_len - 1] == '\n') --result_len; - result_buf[result_len] = 0; - if (result_buf[0] == '?' || (result_buf[result_len - 2] == ':' && result_buf[result_len - 1] == '0')) - break; - /* Get rid of embedded newline, if any. Test for "main" */ - { - char *nl = strchr(result_buf, '\n'); - if (nl != NULL && (word)nl < (word)(result_buf + result_len)) { - *nl = ':'; - } - if (strncmp(result_buf, "main", - nl != NULL - ? (size_t)((word)nl /* a cppcheck workaround */ - - COVERT_DATAFLOW(result_buf)) - : result_len) == 0) { - stop = TRUE; - } - } - if (result_len < RESULT_SZ - 25) { - /* Add in hex address */ - (void)snprintf(&result_buf[result_len], - sizeof(result_buf) - result_len, - " [0x%lx]", (unsigned long)info[i].ci_pc); - result_buf[sizeof(result_buf) - 1] = '\0'; - } -#if defined(CPPCHECK) - GC_noop1((unsigned char)name[0]); - /* name computed previously is discarded */ -#endif - name = result_buf; - } while (0); -#endif /* LINUX */ - GC_err_printf("\t\t%s\n", name); -#if defined(GC_HAVE_BUILTIN_BACKTRACE) && !defined(GC_BACKTRACE_SYMBOLS_BROKEN) - if (sym_name != NULL) - free(sym_name); /* May call GC_[debug_]free; that's OK */ -#endif - } -#if defined(LINUX) && !defined(SMALL_CONFIG) - if (stop) - break; -#endif - } - LOCK(); - --reentry_count; - UNLOCK(); -} - -#endif /* NEED_CALLINFO */ - -#if defined(LINUX) && defined(__ELF__) && !defined(SMALL_CONFIG) -/* Dump /proc/self/maps to GC_stderr, to enable looking up names for */ -/* addresses in FIND_LEAK output. */ -void GC_print_address_map(void) { - const char *maps = GC_get_maps(); - - GC_err_printf("---------- Begin address map ----------\n"); - GC_err_puts(maps); - GC_err_printf("---------- End address map ----------\n"); -} -#endif /* LINUX && ELF */ diff --git a/vendor/bdwgc/private/darwin_semaphore.h b/vendor/bdwgc/private/darwin_semaphore.h deleted file mode 100644 index ed4ae948..00000000 --- a/vendor/bdwgc/private/darwin_semaphore.h +++ /dev/null @@ -1,86 +0,0 @@ -/* - * Copyright (c) 1994 by Xerox Corporation. All rights reserved. - * Copyright (c) 1996 by Silicon Graphics. All rights reserved. - * Copyright (c) 1998 by Fergus Henderson. All rights reserved. - * Copyright (c) 2000-2009 by Hewlett-Packard Development Company. - * All rights reserved. - * - * THIS MATERIAL IS PROVIDED AS IS, WITH ABSOLUTELY NO WARRANTY EXPRESSED - * OR IMPLIED. ANY USE IS AT YOUR OWN RISK. - * - * Permission is hereby granted to use or copy this program - * for any purpose, provided the above notices are retained on all copies. - * Permission to modify the code and to distribute modified code is granted, - * provided the above notices are retained, and a notice that the code was - * modified is included with the above copyright notice. - */ - -#ifndef GC_DARWIN_SEMAPHORE_H -#define GC_DARWIN_SEMAPHORE_H - -#if !defined(GC_DARWIN_THREADS) && !defined(GC_WIN32_THREADS) -#error darwin_semaphore.h included for improper target -#endif - -#ifdef __cplusplus -extern "C" { -#endif - -/* This is a very simple semaphore implementation based on pthreads. */ -/* It is not async-signal safe. But this is not a problem because */ -/* signals are not used to suspend threads on the target. */ - -typedef struct { - pthread_mutex_t mutex; - pthread_cond_t cond; - int value; -} sem_t; - -GC_INLINE int sem_init(sem_t *sem, int pshared, int value) { - if (pshared != 0) { - errno = EPERM; /* unsupported */ - return -1; - } - sem->value = value; - if (pthread_mutex_init(&sem->mutex, NULL) != 0) - return -1; - if (pthread_cond_init(&sem->cond, NULL) != 0) { - (void)pthread_mutex_destroy(&sem->mutex); - return -1; - } - return 0; -} - -GC_INLINE int sem_post(sem_t *sem) { - if (pthread_mutex_lock(&sem->mutex) != 0) - return -1; - sem->value++; - if (pthread_cond_signal(&sem->cond) != 0) { - (void)pthread_mutex_unlock(&sem->mutex); - return -1; - } - return pthread_mutex_unlock(&sem->mutex) != 0 ? -1 : 0; -} - -GC_INLINE int sem_wait(sem_t *sem) { - if (pthread_mutex_lock(&sem->mutex) != 0) - return -1; - while (sem->value == 0) { - if (pthread_cond_wait(&sem->cond, &sem->mutex) != 0) { - (void)pthread_mutex_unlock(&sem->mutex); - return -1; - } - } - sem->value--; - return pthread_mutex_unlock(&sem->mutex) != 0 ? -1 : 0; -} - -GC_INLINE int sem_destroy(sem_t *sem) { - return pthread_cond_destroy(&sem->cond) != 0 || pthread_mutex_destroy(&sem->mutex) != 0 ? -1 : 0; -} - -#ifdef __cplusplus -} /* extern "C" */ -#endif - -#endif diff --git a/vendor/bdwgc/private/dbg_mlc.h b/vendor/bdwgc/private/dbg_mlc.h deleted file mode 100644 index 91721a61..00000000 --- a/vendor/bdwgc/private/dbg_mlc.h +++ /dev/null @@ -1,180 +0,0 @@ -/* - * Copyright 1988, 1989 Hans-J. Boehm, Alan J. Demers - * Copyright (c) 1991-1995 by Xerox Corporation. All rights reserved. - * Copyright (c) 1997 by Silicon Graphics. All rights reserved. - * Copyright (c) 1999 by Hewlett-Packard Company. All rights reserved. - * - * THIS MATERIAL IS PROVIDED AS IS, WITH ABSOLUTELY NO WARRANTY EXPRESSED - * OR IMPLIED. ANY USE IS AT YOUR OWN RISK. - * - * Permission is hereby granted to use or copy this program - * for any purpose, provided the above notices are retained on all copies. - * Permission to modify the code and to distribute modified code is granted, - * provided the above notices are retained, and a notice that the code was - * modified is included with the above copyright notice. - */ - -/* - * This is mostly an internal header file. Typical clients should - * not use it. Clients that define their own object kinds with - * debugging allocators will probably want to include this, however. - * No attempt is made to keep the namespace clean. This should not be - * included from header files that are frequently included by clients. - */ - -#ifndef GC_DBG_MLC_H -#define GC_DBG_MLC_H - -#include "gc_priv.h" -#ifdef KEEP_BACK_PTRS -#include "gc/gc_backptr.h" -#endif - -EXTERN_C_BEGIN - -#if CPP_WORDSZ == 32 -#define START_FLAG (word)0xfedcedcb -#define END_FLAG (word)0xbcdecdef -#else -#define START_FLAG GC_WORD_C(0xFEDCEDCBfedcedcb) -#define END_FLAG GC_WORD_C(0xBCDECDEFbcdecdef) -#endif -/* Stored both one past the end of user object, and one before */ -/* the end of the object as seen by the allocator. */ - -#if defined(KEEP_BACK_PTRS) || defined(PRINT_BLACK_LIST) || defined(MAKE_BACK_GRAPH) -/* Pointer "source"s that aren't real locations. */ -/* Used in oh_back_ptr fields and as "source" */ -/* argument to some marking functions. */ -#define NOT_MARKED (ptr_t)0 -#define MARKED_FOR_FINALIZATION ((ptr_t)(word)2) -/* Object was marked because it is finalizable. */ -#define MARKED_FROM_REGISTER ((ptr_t)(word)4) -/* Object was marked from a register. Hence the */ -/* source of the reference doesn't have an address. */ -#endif /* KEEP_BACK_PTRS || PRINT_BLACK_LIST */ - -/* Object header */ -typedef struct { -#if defined(KEEP_BACK_PTRS) || defined(MAKE_BACK_GRAPH) - /* We potentially keep two different kinds of back */ - /* pointers. KEEP_BACK_PTRS stores a single back */ - /* pointer in each reachable object to allow reporting */ - /* of why an object was retained. MAKE_BACK_GRAPH */ - /* builds a graph containing the inverse of all */ - /* "points-to" edges including those involving */ - /* objects that have just become unreachable. This */ - /* allows detection of growing chains of unreachable */ - /* objects. It may be possible to eventually combine */ - /* both, but for now we keep them separate. Both */ - /* kinds of back pointers are hidden using the */ - /* following macros. In both cases, the plain version */ - /* is constrained to have an least significant bit of 1, */ - /* to allow it to be distinguished from a free list */ - /* link. This means the plain version must have an */ - /* lsb of 0. */ - /* Note that blocks dropped by black-listing will */ - /* also have the lsb clear once debugging has */ - /* started. */ - /* We're careful never to overwrite a value with lsb 0. */ -#if ALIGNMENT == 1 - /* Fudge back pointer to be even. */ -#define HIDE_BACK_PTR(p) GC_HIDE_POINTER(~1 & (word)(p)) -#else -#define HIDE_BACK_PTR(p) GC_HIDE_POINTER(p) -#endif -#ifdef KEEP_BACK_PTRS - GC_hidden_pointer oh_back_ptr; -#endif -#ifdef MAKE_BACK_GRAPH - GC_hidden_pointer oh_bg_ptr; -#endif -#if defined(KEEP_BACK_PTRS) != defined(MAKE_BACK_GRAPH) - /* Keep double-pointer-sized alignment. */ - word oh_dummy; -#endif -#endif - const char *oh_string; /* object descriptor string (file name) */ - signed_word oh_int; /* object descriptor integer (line number) */ -#ifdef NEED_CALLINFO - struct callinfo oh_ci[NFRAMES]; -#endif -#ifndef SHORT_DBG_HDRS - word oh_sz; /* Original malloc arg. */ - word oh_sf; /* start flag */ -#endif /* SHORT_DBG_HDRS */ -} oh; -/* The size of the above structure is assumed not to de-align things, */ -/* and to be a multiple of the word length. */ - -#ifdef SHORT_DBG_HDRS -#define DEBUG_BYTES (sizeof(oh)) -#define UNCOLLECTABLE_DEBUG_BYTES DEBUG_BYTES -#else -/* Add space for END_FLAG, but use any extra space that was already */ -/* added to catch off-the-end pointers. */ -/* For uncollectible objects, the extra byte is not added. */ -#define UNCOLLECTABLE_DEBUG_BYTES (sizeof(oh) + sizeof(word)) -#define DEBUG_BYTES (UNCOLLECTABLE_DEBUG_BYTES - EXTRA_BYTES) -#endif - -/* Round bytes to words without adding extra byte at end. */ -#define SIMPLE_ROUNDED_UP_WORDS(n) BYTES_TO_WORDS((n) + WORDS_TO_BYTES(1) - 1) - -/* ADD_CALL_CHAIN stores a (partial) call chain into an object */ -/* header; it should be called with the allocation lock held. */ -/* PRINT_CALL_CHAIN prints the call chain stored in an object */ -/* to stderr. It requires that we do not hold the lock. */ -#if defined(SAVE_CALL_CHAIN) -struct callinfo; -GC_INNER void GC_save_callers(struct callinfo info[NFRAMES]); -GC_INNER void GC_print_callers(struct callinfo info[NFRAMES]); -#define ADD_CALL_CHAIN(base, ra) GC_save_callers(((oh *)(base))->oh_ci) -#define PRINT_CALL_CHAIN(base) GC_print_callers(((oh *)(base))->oh_ci) -#elif defined(GC_ADD_CALLER) -struct callinfo; -GC_INNER void GC_print_callers(struct callinfo info[NFRAMES]); -#define ADD_CALL_CHAIN(base, ra) ((oh *)(base))->oh_ci[0].ci_pc = (ra) -#define PRINT_CALL_CHAIN(base) GC_print_callers(((oh *)(base))->oh_ci) -#else -#define ADD_CALL_CHAIN(base, ra) -#define PRINT_CALL_CHAIN(base) -#endif - -#ifdef GC_ADD_CALLER -#define OPT_RA ra, -#else -#define OPT_RA -#endif - -/* Check whether object with base pointer p has debugging info */ -/* p is assumed to point to a legitimate object in our part */ -/* of the heap. */ -#ifdef SHORT_DBG_HDRS -#define GC_has_other_debug_info(p) 1 -#else -GC_INNER int GC_has_other_debug_info(ptr_t p); -#endif - -#if defined(KEEP_BACK_PTRS) || defined(MAKE_BACK_GRAPH) -#if defined(SHORT_DBG_HDRS) && !defined(CPPCHECK) -#error Non-ptr stored in object results in GC_HAS_DEBUG_INFO malfunction -/* We may mistakenly conclude that p has a debugging wrapper. */ -#endif -#if defined(PARALLEL_MARK) && defined(KEEP_BACK_PTRS) -#define GC_HAS_DEBUG_INFO(p) \ - ((AO_load((volatile AO_t *)(p)) & 1) != 0 && GC_has_other_debug_info(p) > 0) -/* Atomic load is used as GC_store_back_pointer */ -/* stores oh_back_ptr atomically (p might point */ -/* to the field); this prevents a TSan warning. */ -#else -#define GC_HAS_DEBUG_INFO(p) \ - ((*(word *)(p)&1) && GC_has_other_debug_info(p) > 0) -#endif -#else -#define GC_HAS_DEBUG_INFO(p) (GC_has_other_debug_info(p) > 0) -#endif /* !KEEP_BACK_PTRS && !MAKE_BACK_GRAPH */ - -EXTERN_C_END - -#endif /* GC_DBG_MLC_H */ diff --git a/vendor/bdwgc/private/gc/gc.h b/vendor/bdwgc/private/gc/gc.h deleted file mode 100644 index f6ed927f..00000000 --- a/vendor/bdwgc/private/gc/gc.h +++ /dev/null @@ -1,2248 +0,0 @@ -/* - * Copyright (c) 1988-1989 Hans-J. Boehm, Alan J. Demers - * Copyright (c) 1991-1995 by Xerox Corporation. All rights reserved. - * Copyright (c) 1996-1999 by Silicon Graphics. All rights reserved. - * Copyright (c) 1999 by Hewlett-Packard Company. All rights reserved. - * Copyright (c) 2007 Free Software Foundation, Inc. - * Copyright (c) 2000-2011 by Hewlett-Packard Development Company. - * Copyright (c) 2009-2022 Ivan Maidanski - * - * THIS MATERIAL IS PROVIDED AS IS, WITH ABSOLUTELY NO WARRANTY EXPRESSED - * OR IMPLIED. ANY USE IS AT YOUR OWN RISK. - * - * Permission is hereby granted to use or copy this program - * for any purpose, provided the above notices are retained on all copies. - * Permission to modify the code and to distribute modified code is granted, - * provided the above notices are retained, and a notice that the code was - * modified is included with the above copyright notice. - */ - -/* - * Note that this defines a large number of tuning hooks, which can - * safely be ignored in nearly all cases. For normal use it suffices - * to call only GC_MALLOC and perhaps GC_REALLOC. - * For better performance, also look at GC_MALLOC_ATOMIC, and - * GC_enable_incremental. If you need an action to be performed - * immediately before an object is collected, look at GC_register_finalizer. - * Everything else is best ignored unless you encounter performance - * problems. - */ - -#ifndef GC_H -#define GC_H - -/* Help debug mixed up preprocessor symbols. */ -#if (defined(WIN64) && !defined(_WIN64)) && defined(_MSC_VER) -#pragma message("Warning: Expecting _WIN64 for x64 targets! Notice the leading underscore!") -#endif - -#include "gc_version.h" -/* Define version numbers here to allow test on build machine */ -/* for cross-builds. Note that this defines the header */ -/* version number, which may or may not match that of the */ -/* dynamic library. GC_get_version() can be used to obtain */ -/* the latter. */ - -#include "gc_config_macros.h" - -#ifdef __cplusplus -extern "C" { -#endif - -typedef void *GC_PTR; /* preserved only for backward compatibility */ - -/* Define word and signed_word to be unsigned and signed types of the */ -/* size as char * or void *. There seems to be no way to do this */ -/* even semi-portably. The following is probably no better/worse */ -/* than almost anything else. */ -/* The ANSI standard suggests that size_t and ptrdiff_t might be */ -/* better choices. But those had incorrect definitions on some older */ -/* systems. Notably "typedef int size_t" is WRONG. */ -#ifdef _WIN64 -#if defined(__int64) && !defined(CPPCHECK) -typedef unsigned __int64 GC_word; -typedef __int64 GC_signed_word; -#else -typedef unsigned long long GC_word; -typedef long long GC_signed_word; -#endif -#else -typedef unsigned long GC_word; -typedef long GC_signed_word; -#endif - -/* Get the GC library version. The returned value is a constant in the */ -/* form: ((version_major<<16) | (version_minor<<8) | version_micro). */ -GC_API unsigned GC_CALL GC_get_version(void); - -/* Public read-only variables */ -/* The supplied getter functions are preferred for new code. */ - -GC_API GC_ATTR_DEPRECATED GC_word GC_gc_no; -/* Counter incremented per collection. */ -/* Includes empty GCs at startup. */ -GC_API GC_word GC_CALL GC_get_gc_no(void); -/* GC_get_gc_no() is unsynchronized, so */ -/* it requires GC_call_with_alloc_lock() to */ -/* avoid data races on multiprocessors. */ - -#ifdef GC_THREADS -/* GC is parallelized for performance on multiprocessors. Set to */ -/* a non-zero value when client calls GC_start_mark_threads() */ -/* directly or starts the first non-main thread, provided the */ -/* collector is built with PARALLEL_MARK defined, and either */ -/* GC_MARKERS (or GC_NPROCS) environment variable is set to a value */ -/* bigger than 1, or multiple cores (processors) are available, or */ -/* the client calls GC_set_markers_count() before GC initialization. */ -/* After setting, GC_parallel value is equal to the number of marker */ -/* threads minus one (i.e. the number of existing parallel marker */ -/* threads excluding the initiating one). */ -GC_API GC_ATTR_DEPRECATED int GC_parallel; -#endif - -/* Return value of GC_parallel. Does not acquire the GC lock. */ -GC_API int GC_CALL GC_get_parallel(void); - -/* Set the number of marker threads (including the initiating one) */ -/* to the desired value at start-up. Zero value means the collector */ -/* is to decide. If the correct non-zero value is passed, then later */ -/* GC_parallel will be set to the value minus one. Has no effect if */ -/* called after GC initialization. Does not itself cause creation of */ -/* the marker threads. Does not use any synchronization. */ -GC_API void GC_CALL GC_set_markers_count(unsigned); - -/* Public R/W variables */ -/* The supplied setter and getter functions are preferred for new code. */ - -typedef void *(GC_CALLBACK *GC_oom_func)(size_t /* bytes_requested */); -GC_API GC_ATTR_DEPRECATED GC_oom_func GC_oom_fn; -/* When there is insufficient memory to satisfy */ -/* an allocation request, we return */ -/* (*GC_oom_fn)(size). If it returns, it must */ -/* return either NULL or a valid pointer to */ -/* a previously allocated heap object. */ -/* By default, this just returns NULL. */ -/* GC_oom_fn must not be 0. Both the supplied */ -/* setter and the getter acquire the GC lock */ -/* (to avoid data races). */ -GC_API void GC_CALL GC_set_oom_fn(GC_oom_func) GC_ATTR_NONNULL(1); -GC_API GC_oom_func GC_CALL GC_get_oom_fn(void); - -typedef void(GC_CALLBACK *GC_on_heap_resize_proc)(GC_word /* new_size */); -GC_API GC_ATTR_DEPRECATED GC_on_heap_resize_proc GC_on_heap_resize; -/* Invoked when the heap grows or shrinks. */ -/* Called with the world stopped (and the */ -/* allocation lock held). May be 0. */ -GC_API void GC_CALL GC_set_on_heap_resize(GC_on_heap_resize_proc); -GC_API GC_on_heap_resize_proc GC_CALL GC_get_on_heap_resize(void); -/* Both the supplied setter and the getter */ -/* acquire the GC lock (to avoid data races). */ - -typedef enum { - GC_EVENT_START /* COLLECTION */, - GC_EVENT_MARK_START, - GC_EVENT_MARK_END, - GC_EVENT_RECLAIM_START, - GC_EVENT_RECLAIM_END, - GC_EVENT_END /* COLLECTION */, - GC_EVENT_PRE_STOP_WORLD /* STOPWORLD_BEGIN */, - GC_EVENT_POST_STOP_WORLD /* STOPWORLD_END */, - GC_EVENT_PRE_START_WORLD /* STARTWORLD_BEGIN */, - GC_EVENT_POST_START_WORLD /* STARTWORLD_END */, - GC_EVENT_THREAD_SUSPENDED, - GC_EVENT_THREAD_UNSUSPENDED -} GC_EventType; - -typedef void(GC_CALLBACK *GC_on_collection_event_proc)(GC_EventType); -/* Invoked to indicate progress through the */ -/* collection process. Not used for thread */ -/* suspend/resume notifications. Called with */ -/* the GC lock held (or, even, the world */ -/* stopped). May be 0 (means no notifier). */ -GC_API void GC_CALL GC_set_on_collection_event(GC_on_collection_event_proc); -GC_API GC_on_collection_event_proc GC_CALL GC_get_on_collection_event(void); -/* Both the supplied setter and the getter */ -/* acquire the GC lock (to avoid data races). */ - -#if defined(GC_THREADS) || (defined(GC_BUILD) && defined(NN_PLATFORM_CTR)) -typedef void(GC_CALLBACK *GC_on_thread_event_proc)(GC_EventType, - void * /* thread_id */); -/* Invoked when a thread is suspended or */ -/* resumed during collection. Called with the */ -/* GC lock held (and the world stopped */ -/* partially). May be 0 (means no notifier). */ -GC_API void GC_CALL GC_set_on_thread_event(GC_on_thread_event_proc); -GC_API GC_on_thread_event_proc GC_CALL GC_get_on_thread_event(void); -/* Both the supplied setter and the getter */ -/* acquire the GC lock (to avoid data races). */ -#endif - -GC_API GC_ATTR_DEPRECATED int GC_find_leak; -/* Set to true to turn on the leak-finding mode */ -/* (do not actually garbage collect, but simply */ -/* report inaccessible memory that was not */ -/* deallocated with GC_FREE). Initial value */ -/* is determined by FIND_LEAK macro. */ -/* The value should not typically be modified */ -/* after GC initialization (and, thus, it does */ -/* not use or need synchronization). */ -GC_API void GC_CALL GC_set_find_leak(int); -GC_API int GC_CALL GC_get_find_leak(void); - -GC_API GC_ATTR_DEPRECATED int GC_all_interior_pointers; -/* Arrange for pointers to object interiors to */ -/* be recognized as valid. Typically should */ -/* not be changed after GC initialization (in */ -/* case of calling it after the GC is */ -/* initialized, the setter acquires the GC lock */ -/* (to avoid data races). The initial value */ -/* depends on whether the GC is built with */ -/* ALL_INTERIOR_POINTERS macro defined or not. */ -/* Unless DONT_ADD_BYTE_AT_END is defined, this */ -/* also affects whether sizes are increased by */ -/* at least a byte to allow "off the end" */ -/* pointer recognition (uncollectible objects */ -/* are the exception). Must be only 0 or 1. */ -GC_API void GC_CALL GC_set_all_interior_pointers(int); -GC_API int GC_CALL GC_get_all_interior_pointers(void); - -GC_API GC_ATTR_DEPRECATED int GC_finalize_on_demand; -/* If nonzero, finalizers will only be run in */ -/* response to an explicit GC_invoke_finalizers */ -/* call. The default is determined by whether */ -/* the FINALIZE_ON_DEMAND macro is defined */ -/* when the collector is built. */ -/* The setter and getter are unsynchronized. */ -GC_API void GC_CALL GC_set_finalize_on_demand(int); -GC_API int GC_CALL GC_get_finalize_on_demand(void); - -GC_API GC_ATTR_DEPRECATED int GC_java_finalization; -/* Mark objects reachable from finalizable */ -/* objects in a separate post-pass. This makes */ -/* it a bit safer to use non-topologically- */ -/* ordered finalization. Default value is */ -/* determined by JAVA_FINALIZATION macro. */ -/* Enables GC_register_finalizer_unreachable to */ -/* work correctly. */ -/* The setter and getter are unsynchronized. */ -GC_API void GC_CALL GC_set_java_finalization(int); -GC_API int GC_CALL GC_get_java_finalization(void); - -typedef void(GC_CALLBACK *GC_finalizer_notifier_proc)(void); -GC_API GC_ATTR_DEPRECATED GC_finalizer_notifier_proc GC_finalizer_notifier; -/* Invoked by the collector when there are */ -/* objects to be finalized. Invoked at most */ -/* once per GC cycle. Never invoked unless */ -/* GC_finalize_on_demand is set. */ -/* Typically this will notify a finalization */ -/* thread, which will call GC_invoke_finalizers */ -/* in response. May be 0 (means no notifier). */ -/* Both the supplied setter and the getter */ -/* acquire the GC lock (to avoid data races). */ -GC_API void GC_CALL GC_set_finalizer_notifier(GC_finalizer_notifier_proc); -GC_API GC_finalizer_notifier_proc GC_CALL GC_get_finalizer_notifier(void); - -/* The functions called to report pointer checking errors. Called */ -/* without the GC lock held. The default behavior is to fail with */ -/* the appropriate message which includes the pointers. The functions */ -/* (variables) must not be 0. Both the setters and getters acquire */ -/* the GC lock (to avoid data races). */ -typedef void(GC_CALLBACK *GC_valid_ptr_print_proc_t)(void *); -typedef void(GC_CALLBACK *GC_same_obj_print_proc_t)(void * /* p */, - void * /* q */); -GC_API GC_ATTR_DEPRECATED GC_same_obj_print_proc_t GC_same_obj_print_proc; -GC_API GC_ATTR_DEPRECATED GC_valid_ptr_print_proc_t - GC_is_valid_displacement_print_proc; -GC_API GC_ATTR_DEPRECATED GC_valid_ptr_print_proc_t GC_is_visible_print_proc; -GC_API void GC_CALL GC_set_same_obj_print_proc(GC_same_obj_print_proc_t) - GC_ATTR_NONNULL(1); -GC_API GC_same_obj_print_proc_t GC_CALL GC_get_same_obj_print_proc(void); -GC_API void GC_CALL GC_set_is_valid_displacement_print_proc( - GC_valid_ptr_print_proc_t) GC_ATTR_NONNULL(1); -GC_API GC_valid_ptr_print_proc_t GC_CALL -GC_get_is_valid_displacement_print_proc(void); -GC_API void GC_CALL GC_set_is_visible_print_proc(GC_valid_ptr_print_proc_t) - GC_ATTR_NONNULL(1); -GC_API GC_valid_ptr_print_proc_t GC_CALL GC_get_is_visible_print_proc(void); - -GC_API -#ifndef GC_DONT_GC -GC_ATTR_DEPRECATED -#endif -int GC_dont_gc; /* != 0 ==> Do not collect. This overrides */ - /* explicit GC_gcollect() calls as well. */ - /* Used as a counter, so that nested enabling */ - /* and disabling work correctly. Should */ - /* normally be updated with GC_enable() and */ - /* GC_disable() calls. Direct assignment to */ - /* GC_dont_gc is deprecated. To check whether */ - /* GC is disabled, GC_is_disabled() is */ - /* preferred for new code. */ - -GC_API GC_ATTR_DEPRECATED int GC_dont_expand; -/* Do not expand the heap unless explicitly */ -/* requested or forced to. The setter and */ -/* getter are unsynchronized. */ -GC_API void GC_CALL GC_set_dont_expand(int); -GC_API int GC_CALL GC_get_dont_expand(void); - -GC_API GC_ATTR_DEPRECATED int GC_use_entire_heap; -/* Causes the non-incremental collector to use the */ -/* entire heap before collecting. This sometimes */ -/* results in more large block fragmentation, since */ -/* very large blocks will tend to get broken up */ -/* during each GC cycle. It is likely to result in a */ -/* larger working set, but lower collection */ -/* frequencies, and hence fewer instructions executed */ -/* in the collector. */ - -GC_API GC_ATTR_DEPRECATED int GC_full_freq; -/* Number of partial collections between */ -/* full collections. Matters only if */ -/* GC_is_incremental_mode(). */ -/* Full collections are also triggered if */ -/* the collector detects a substantial */ -/* increase in the number of in-use heap */ -/* blocks. Values in the tens are now */ -/* perfectly reasonable, unlike for */ -/* earlier GC versions. */ -/* The setter and getter are unsynchronized, so */ -/* GC_call_with_alloc_lock() is required to */ -/* avoid data races (if the value is modified */ -/* after the GC is put to multi-threaded mode). */ -GC_API void GC_CALL GC_set_full_freq(int); -GC_API int GC_CALL GC_get_full_freq(void); - -GC_API GC_ATTR_DEPRECATED GC_word GC_non_gc_bytes; -/* Bytes not considered candidates for */ -/* collection. Used only to control scheduling */ -/* of collections. Updated by */ -/* GC_malloc_uncollectable and GC_free. */ -/* Wizards only. */ -/* The setter and getter are unsynchronized, so */ -/* GC_call_with_alloc_lock() is required to */ -/* avoid data races (if the value is modified */ -/* after the GC is put to multi-threaded mode). */ -GC_API void GC_CALL GC_set_non_gc_bytes(GC_word); -GC_API GC_word GC_CALL GC_get_non_gc_bytes(void); - -GC_API GC_ATTR_DEPRECATED int GC_no_dls; -/* Do not register dynamic library data */ -/* segments automatically. Also, if set by the */ -/* collector itself (during GC), this means */ -/* that such a registration is not supported. */ -/* Wizards only. Should be set only if the */ -/* application explicitly registers all roots. */ -/* (In some environments like Microsoft Windows */ -/* and Apple's Darwin, this may also prevent */ -/* registration of the main data segment as a */ -/* part of the root set.) */ -/* The setter and getter are unsynchronized. */ -GC_API void GC_CALL GC_set_no_dls(int); -GC_API int GC_CALL GC_get_no_dls(void); - -GC_API GC_ATTR_DEPRECATED GC_word GC_free_space_divisor; -/* We try to make sure that we allocate at */ -/* least N/GC_free_space_divisor bytes between */ -/* collections, where N is twice the number */ -/* of traced bytes, plus the number of untraced */ -/* bytes (bytes in "atomic" objects), plus */ -/* a rough estimate of the root set size. */ -/* N approximates GC tracing work per GC. */ -/* The initial value is GC_FREE_SPACE_DIVISOR. */ -/* Increasing its value will use less space */ -/* but more collection time. Decreasing it */ -/* will appreciably decrease collection time */ -/* at the expense of space. */ -/* The setter and getter are unsynchronized, so */ -/* GC_call_with_alloc_lock() is required to */ -/* avoid data races (if the value is modified */ -/* after the GC is put to multi-threaded mode). */ -/* In GC v7.1 (and before), the setter returned */ -/* the old value. */ -GC_API void GC_CALL GC_set_free_space_divisor(GC_word); -GC_API GC_word GC_CALL GC_get_free_space_divisor(void); - -GC_API GC_ATTR_DEPRECATED GC_word GC_max_retries; -/* The maximum number of GCs attempted before */ -/* reporting out of memory after heap */ -/* expansion fails. Initially 0. */ -/* The setter and getter are unsynchronized, so */ -/* GC_call_with_alloc_lock() is required to */ -/* avoid data races (if the value is modified */ -/* after the GC is put to multi-threaded mode). */ -GC_API void GC_CALL GC_set_max_retries(GC_word); -GC_API GC_word GC_CALL GC_get_max_retries(void); - -GC_API GC_ATTR_DEPRECATED char *GC_stackbottom; -/* The cold end (bottom) of user stack. */ -/* May be set in the client prior to */ -/* calling any GC_ routines. This */ -/* avoids some overhead, and */ -/* potentially some signals that can */ -/* confuse debuggers. Otherwise the */ -/* collector attempts to set it */ -/* automatically. For multi-threaded */ -/* code, this is the cold end of the */ -/* stack for the primordial thread. */ -/* For multi-threaded code, altering */ -/* GC_stackbottom value directly after */ -/* GC initialization has no effect. */ -/* Portable clients should use */ -/* GC_set_stackbottom(), */ -/* GC_get_stack_base(), */ -/* GC_call_with_gc_active() and */ -/* GC_register_my_thread() instead. */ - -GC_API GC_ATTR_DEPRECATED int GC_dont_precollect; -/* Do not collect as part of GC */ -/* initialization. Should be set only */ -/* if the client wants a chance to */ -/* manually initialize the root set */ -/* before the first collection. */ -/* Interferes with blacklisting. */ -/* Wizards only. The setter and getter */ -/* are unsynchronized (and no external */ -/* locking is needed since the value is */ -/* accessed at GC initialization only). */ -GC_API void GC_CALL GC_set_dont_precollect(int); -GC_API int GC_CALL GC_get_dont_precollect(void); - -GC_API GC_ATTR_DEPRECATED unsigned long GC_time_limit; -/* If incremental collection is enabled, */ -/* we try to terminate collections */ -/* after this many milliseconds (plus */ -/* the amount of nanoseconds as given in */ -/* the latest GC_set_time_limit_tv call, */ -/* if any). Not a hard time bound. */ -/* Setting this variable to */ -/* GC_TIME_UNLIMITED will essentially */ -/* disable incremental collection while */ -/* leaving generational collection */ -/* enabled. */ -#define GC_TIME_UNLIMITED 999999 -/* Setting GC_time_limit to this value */ -/* will disable the "pause time exceeded"*/ -/* tests. */ -/* The setter and getter are unsynchronized, so */ -/* GC_call_with_alloc_lock() is required to */ -/* avoid data races (if the value is modified */ -/* after the GC is put to multi-threaded mode). */ -/* The setter does not update the value of the */ -/* nanosecond part of the time limit (it is */ -/* zero unless ever set by GC_set_time_limit_tv */ -/* call). */ -GC_API void GC_CALL GC_set_time_limit(unsigned long); -GC_API unsigned long GC_CALL GC_get_time_limit(void); - -/* A portable type definition of time with a nanosecond precision. */ -struct GC_timeval_s { - unsigned long tv_ms; /* time in milliseconds */ - unsigned long tv_nsec; /* nanoseconds fraction (<1000000) */ -}; - -/* Public procedures */ - -/* Set/get the time limit of the incremental collections. This is */ -/* similar to GC_set_time_limit and GC_get_time_limit but the time is */ -/* provided with the nanosecond precision. The value of tv_nsec part */ -/* should be less than a million. If the value of tv_ms part is */ -/* GC_TIME_UNLIMITED then tv_nsec is ignored. Initially, the value of */ -/* tv_nsec part of the time limit is zero. The functions do not use */ -/* any synchronization. Defined only if the library has been compiled */ -/* without NO_CLOCK. */ -GC_API void GC_CALL GC_set_time_limit_tv(struct GC_timeval_s); -GC_API struct GC_timeval_s GC_CALL GC_get_time_limit_tv(void); - -/* Set/get the minimum value of the ratio of allocated bytes since GC */ -/* to the amount of finalizers created since that GC (value > */ -/* GC_bytes_allocd / (GC_fo_entries - last_fo_entries)) which triggers */ -/* the collection instead heap expansion. The value has no effect in */ -/* the GC incremental mode. The default value is 10000 unless */ -/* GC_ALLOCD_BYTES_PER_FINALIZER macro with a custom value is defined */ -/* to build libgc. The default value might be not the right choice for */ -/* clients where e.g. most objects have a finalizer. Zero value */ -/* effectively disables taking amount of finalizers in the decision */ -/* whether to collect or not. The functions do not use any */ -/* synchronization. */ -GC_API void GC_CALL GC_set_allocd_bytes_per_finalizer(GC_word); -GC_API GC_word GC_CALL GC_get_allocd_bytes_per_finalizer(void); - -/* Tell the collector to start various performance measurements. */ -/* Only the total time taken by full collections is calculated, as */ -/* of now. And, currently, there is no way to stop the measurements. */ -/* The function does not use any synchronization. Defined only if the */ -/* library has been compiled without NO_CLOCK. */ -GC_API void GC_CALL GC_start_performance_measurement(void); - -/* Get the total time of all full collections since the start of the */ -/* performance measurements. The measurement unit is one millisecond. */ -/* Note that the returned value wraps around on overflow. */ -/* The function does not use any synchronization. Defined only if the */ -/* library has been compiled without NO_CLOCK. */ -GC_API unsigned long GC_CALL GC_get_full_gc_total_time(void); - -/* Set whether the GC will allocate executable memory pages or not. */ -/* A non-zero argument instructs the collector to allocate memory with */ -/* the executable flag on. Must be called before the collector is */ -/* initialized. May have no effect on some platforms. The default */ -/* value is controlled by NO_EXECUTE_PERMISSION macro (if present then */ -/* the flag is off). Portable clients should have */ -/* GC_set_pages_executable(1) call (before GC_INIT) provided they are */ -/* going to execute code on any of the GC-allocated memory objects. */ -GC_API void GC_CALL GC_set_pages_executable(int); - -/* Returns non-zero value if the GC is set to the allocate-executable */ -/* mode. The mode could be changed by GC_set_pages_executable (before */ -/* GC_INIT) unless the former has no effect on the platform. Does not */ -/* use or need synchronization (i.e. acquiring the allocator lock). */ -GC_API int GC_CALL GC_get_pages_executable(void); - -/* The setter and getter of the minimum value returned by the internal */ -/* min_bytes_allocd(). The value should not be zero; the default value */ -/* is one. Not synchronized. */ -GC_API void GC_CALL GC_set_min_bytes_allocd(size_t); -GC_API size_t GC_CALL GC_get_min_bytes_allocd(void); - -/* Set/get the size in pages of units operated by GC_collect_a_little. */ -/* The value should not be zero. Not synchronized. */ -GC_API void GC_CALL GC_set_rate(int); -GC_API int GC_CALL GC_get_rate(void); - -/* Set/get the maximum number of prior attempts at the world-stop */ -/* marking. Not synchronized. */ -GC_API void GC_CALL GC_set_max_prior_attempts(int); -GC_API int GC_CALL GC_get_max_prior_attempts(void); - -/* Control whether to disable algorithm deciding if a collection should */ -/* be started when we allocated enough to amortize GC. Both the setter */ -/* and the getter acquire the GC lock (to avoid data races). */ -GC_API void GC_CALL GC_set_disable_automatic_collection(int); -GC_API int GC_CALL GC_get_disable_automatic_collection(void); - -/* Overrides the default handle-fork mode. Non-zero value means GC */ -/* should install proper pthread_atfork handlers. Has effect only if */ -/* called before GC_INIT. Clients should invoke GC_set_handle_fork */ -/* with non-zero argument if going to use fork with GC functions called */ -/* in the forked child. (Note that such client and atfork handlers */ -/* activities are not fully POSIX-compliant.) GC_set_handle_fork */ -/* instructs GC_init to setup GC fork handlers using pthread_atfork, */ -/* the latter might fail (or, even, absent on some targets) causing */ -/* abort at GC initialization. Issues with missing (or failed) */ -/* pthread_atfork() could be avoided by invocation */ -/* of GC_set_handle_fork(-1) at application start-up and surrounding */ -/* each fork() with the relevant GC_atfork_prepare/parent/child calls. */ -GC_API void GC_CALL GC_set_handle_fork(int); - -/* Routines to handle POSIX fork() manually (no-op if handled */ -/* automatically). GC_atfork_prepare should be called immediately */ -/* before fork(); GC_atfork_parent should be invoked just after fork in */ -/* the branch that corresponds to parent process (i.e., fork result is */ -/* non-zero); GC_atfork_child is to be called immediately in the child */ -/* branch (i.e., fork result is 0). Note that GC_atfork_child() call */ -/* should, of course, precede GC_start_mark_threads call (if any). */ -GC_API void GC_CALL GC_atfork_prepare(void); -GC_API void GC_CALL GC_atfork_parent(void); -GC_API void GC_CALL GC_atfork_child(void); - -/* Initialize the collector. Portable clients should call GC_INIT() */ -/* from the main program instead. */ -GC_API void GC_CALL GC_init(void); - -/* Return 1 (true) if the collector is initialized (or, at least, the */ -/* initialization is in progress), 0 otherwise. */ -GC_API int GC_CALL GC_is_init_called(void); - -/* Perform the collector shutdown. (E.g. dispose critical sections on */ -/* Win32 target.) A duplicate invocation is a no-op. GC_INIT should */ -/* not be called after the shutdown. See also GC_win32_free_heap(). */ -GC_API void GC_CALL GC_deinit(void); - -/* General purpose allocation routines, with roughly malloc calling */ -/* conv. The atomic versions promise that no relevant pointers are */ -/* contained in the object. The non-atomic versions guarantee that the */ -/* new object is cleared. GC_malloc_uncollectable allocates */ -/* an object that is scanned for pointers to collectible */ -/* objects, but is not itself collectible. The object is scanned even */ -/* if it does not appear to be reachable. GC_malloc_uncollectable and */ -/* GC_free called on the resulting object implicitly update */ -/* GC_non_gc_bytes appropriately. */ -GC_API GC_ATTR_MALLOC GC_ATTR_ALLOC_SIZE(1) void *GC_CALL - GC_malloc(size_t /* size_in_bytes */); -GC_API GC_ATTR_MALLOC GC_ATTR_ALLOC_SIZE(1) void *GC_CALL - GC_malloc_atomic(size_t /* size_in_bytes */); -GC_API GC_ATTR_MALLOC char *GC_CALL GC_strdup(const char *); -GC_API GC_ATTR_MALLOC char *GC_CALL -GC_strndup(const char *, size_t) GC_ATTR_NONNULL(1); -GC_API GC_ATTR_MALLOC GC_ATTR_ALLOC_SIZE(1) void *GC_CALL - GC_malloc_uncollectable(size_t /* size_in_bytes */); -GC_API GC_ATTR_DEPRECATED void *GC_CALL GC_malloc_stubborn(size_t); - -/* The routines that guarantee the requested alignment of the allocated */ -/* memory object. The align argument should be a power of two and not */ -/* less than size of a pointer. Note: GC_base() and GC_size() might */ -/* return the value which is not the expected one due to the alignment. */ -GC_API GC_ATTR_MALLOC GC_ATTR_ALLOC_SIZE(2) void *GC_CALL - GC_memalign(size_t /* align */, size_t /* lb */); -GC_API int GC_CALL GC_posix_memalign(void ** /* memptr */, size_t /* align */, - size_t /* lb */) GC_ATTR_NONNULL(1); -#ifndef GC_NO_VALLOC -GC_API GC_ATTR_MALLOC GC_ATTR_ALLOC_SIZE(1) void *GC_CALL - GC_valloc(size_t /* lb */); -GC_API GC_ATTR_MALLOC GC_ATTR_ALLOC_SIZE(1) void *GC_CALL - GC_pvalloc(size_t /* lb */); -#endif /* !GC_NO_VALLOC */ - -/* Explicitly deallocate an object. Dangerous if used incorrectly. */ -/* Requires a pointer to the base of an object. */ -/* An object should not be enabled for finalization (and it should not */ -/* contain registered disappearing links of any kind) when it is */ -/* explicitly deallocated. */ -/* GC_free(0) is a no-op, as required by ANSI C for free. */ -GC_API void GC_CALL GC_free(void *); - -/* The "stubborn" objects allocation is not supported anymore. Exists */ -/* only for the backward compatibility. */ -#define GC_MALLOC_STUBBORN(sz) GC_MALLOC(sz) -#define GC_NEW_STUBBORN(t) GC_NEW(t) -#define GC_CHANGE_STUBBORN(p) GC_change_stubborn(p) -GC_API GC_ATTR_DEPRECATED void GC_CALL GC_change_stubborn(const void *); - -/* Inform the collector that the object has been changed. */ -/* Only non-NULL pointer stores into the object are considered to be */ -/* changes. Matters only if the incremental collection is enabled in */ -/* the manual VDB mode (otherwise the function does nothing). */ -/* Should be followed typically by GC_reachable_here called for each */ -/* of the stored pointers. */ -GC_API void GC_CALL GC_end_stubborn_change(const void *) GC_ATTR_NONNULL(1); - -/* Return a pointer to the base (lowest address) of an object given */ -/* a pointer to a location within the object. */ -/* I.e., map an interior pointer to the corresponding base pointer. */ -/* Note that with debugging allocation, this returns a pointer to the */ -/* actual base of the object, i.e. the debug information, not to */ -/* the base of the user object. */ -/* Return 0 if displaced_pointer doesn't point to within a valid */ -/* object. */ -/* Note that a deallocated object in the garbage collected heap */ -/* may be considered valid, even if it has been deallocated with */ -/* GC_free. */ -GC_API void *GC_CALL GC_base(void * /* displaced_pointer */); - -/* Return 1 (true) if the argument points to somewhere in the GC heap, */ -/* 0 otherwise. Primary use is as a fast alternative to GC_base() to */ -/* check whether the given object is allocated by the collector or not. */ -/* It is assumed that the collector is already initialized. */ -GC_API int GC_CALL GC_is_heap_ptr(const void *); - -/* Given a pointer to the base of an object, return its size in bytes. */ -/* The returned size may be slightly larger than what was originally */ -/* requested. */ -GC_API size_t GC_CALL GC_size(const void * /* obj_addr */) GC_ATTR_NONNULL(1); - -/* For compatibility with C library. This is occasionally faster than */ -/* a malloc followed by a bcopy. But if you rely on that, either here */ -/* or with the standard C library, your code is broken. In my */ -/* opinion, it shouldn't have been invented, but now we're stuck. -HB */ -/* The resulting object has the same kind as the original. */ -/* It is an error to have changes enabled for the original object. */ -/* It does not change the content of the object from its beginning to */ -/* the minimum of old size and new_size_in_bytes; the content above in */ -/* case of object size growth is initialized to zero (not guaranteed */ -/* for atomic object type). The function follows ANSI conventions for */ -/* NULL old_object (i.e., equivalent to GC_malloc regardless of new */ -/* size). If new size is zero (and old_object is non-NULL) then the */ -/* call is equivalent to GC_free (and NULL is returned). If old_object */ -/* is non-NULL, it must have been returned by an earlier call to */ -/* GC_malloc* or GC_realloc. In case of the allocation failure, the */ -/* memory pointed by old_object is untouched (and not freed). */ -/* If the returned pointer is not the same as old_object and both of */ -/* them are non-NULL then old_object is freed. Returns either NULL (in */ -/* case of the allocation failure or zero new size) or pointer to the */ -/* allocated memory. */ -GC_API void *GC_CALL GC_realloc(void * /* old_object */, - size_t /* new_size_in_bytes */) - /* 'realloc' attr */ GC_ATTR_ALLOC_SIZE(2); - -/* Increase the heap size explicitly. Includes a GC_init() call. */ -/* Returns 0 on failure, 1 on success. */ -GC_API int GC_CALL GC_expand_hp(size_t /* number_of_bytes */); - -/* Limit the heap size to n bytes. Useful when you're debugging, */ -/* especially on systems that don't handle running out of memory well. */ -/* n == 0 ==> unbounded. This is the default. This setter function is */ -/* unsynchronized (so it might require GC_call_with_alloc_lock to avoid */ -/* data races). */ -GC_API void GC_CALL GC_set_max_heap_size(GC_word /* n */); - -/* Inform the collector that a certain section of statically allocated */ -/* memory contains no pointers to garbage collected memory. Thus it */ -/* need not be scanned. This is sometimes important if the application */ -/* maps large read/write files into the address space, which could be */ -/* mistaken for dynamic library data segments on some systems. */ -/* Both section start and end are not needed to be pointer-aligned. */ -GC_API void GC_CALL GC_exclude_static_roots(void * /* low_address */, - void * /* high_address_plus_1 */); - -/* Clear the number of entries in the exclusion table. Wizards only. */ -GC_API void GC_CALL GC_clear_exclusion_table(void); - -/* Clear the set of root segments. Wizards only. */ -GC_API void GC_CALL GC_clear_roots(void); - -/* Add a root segment. Wizards only. */ -/* May merge adjacent or overlapping segments if appropriate. */ -/* Both segment start and end are not needed to be pointer-aligned. */ -/* low_address must not be greater than high_address_plus_1. */ -GC_API void GC_CALL GC_add_roots(void * /* low_address */, - void * /* high_address_plus_1 */); - -/* Remove root segments located fully in the region. Wizards only. */ -GC_API void GC_CALL GC_remove_roots(void * /* low_address */, - void * /* high_address_plus_1 */); - -/* Add a displacement to the set of those considered valid by the */ -/* collector. GC_register_displacement(n) means that if p was returned */ -/* by GC_malloc, then (char *)p + n will be considered to be a valid */ -/* pointer to p. N must be small and less than the size of p. */ -/* (All pointers to the interior of objects from the stack are */ -/* considered valid in any case. This applies to heap objects and */ -/* static data.) */ -/* Preferably, this should be called before any other GC procedures. */ -/* Calling it later adds to the probability of excess memory */ -/* retention. */ -/* This is a no-op if the collector has recognition of */ -/* arbitrary interior pointers enabled, which is now the default. */ -GC_API void GC_CALL GC_register_displacement(size_t /* n */); - -/* The following version should be used if any debugging allocation is */ -/* being done. */ -GC_API void GC_CALL GC_debug_register_displacement(size_t /* n */); - -/* Explicitly trigger a full, world-stop collection. */ -GC_API void GC_CALL GC_gcollect(void); - -/* Same as above but ignores the default stop_func setting and tries to */ -/* unmap as much memory as possible (regardless of the corresponding */ -/* switch setting). The recommended usage: on receiving a system */ -/* low-memory event; before retrying a system call failed because of */ -/* the system is running out of resources. */ -GC_API void GC_CALL GC_gcollect_and_unmap(void); - -/* Trigger a full world-stopped collection. Abort the collection if */ -/* and when stop_func returns a nonzero value. Stop_func will be */ -/* called frequently, and should be reasonably fast. (stop_func is */ -/* called with the allocation lock held and the world might be stopped; */ -/* it's not allowed for stop_func to manipulate pointers to the garbage */ -/* collected heap or call most of GC functions.) This works even */ -/* if virtual dirty bits, and hence incremental collection is not */ -/* available for this architecture. Collections can be aborted faster */ -/* than normal pause times for incremental collection. However, */ -/* aborted collections do no useful work; the next collection needs */ -/* to start from the beginning. stop_func must not be 0. */ -/* GC_try_to_collect() returns 0 if the collection was aborted (or the */ -/* collections are disabled), 1 if it succeeded. */ -typedef int(GC_CALLBACK *GC_stop_func)(void); -GC_API int GC_CALL GC_try_to_collect(GC_stop_func /* stop_func */) - GC_ATTR_NONNULL(1); - -/* Set and get the default stop_func. The default stop_func is used by */ -/* GC_gcollect() and by implicitly triggered collections (except for */ -/* the case when handling out of memory). Must not be 0. */ -/* Both the setter and getter acquire the GC lock to avoid data races. */ -GC_API void GC_CALL GC_set_stop_func(GC_stop_func /* stop_func */) - GC_ATTR_NONNULL(1); -GC_API GC_stop_func GC_CALL GC_get_stop_func(void); - -/* Return the number of bytes in the heap. Excludes collector private */ -/* data structures. Excludes the unmapped memory (returned to the OS). */ -/* Includes empty blocks and fragmentation loss. Includes some pages */ -/* that were allocated but never written. */ -/* This is an unsynchronized getter, so it should be called typically */ -/* with the GC lock held to avoid data races on multiprocessors (the */ -/* alternative is to use GC_get_heap_usage_safe or GC_get_prof_stats */ -/* API calls instead). */ -/* This getter remains lock-free (unsynchronized) for compatibility */ -/* reason since some existing clients call it from a GC callback */ -/* holding the allocator lock. (This API function and the following */ -/* four ones below were made thread-safe in GC v7.2alpha1 and */ -/* reverted back in v7.2alpha7 for the reason described.) */ -GC_API size_t GC_CALL GC_get_heap_size(void); - -/* Return a lower bound on the number of free bytes in the heap */ -/* (excluding the unmapped memory space). This is an unsynchronized */ -/* getter (see GC_get_heap_size comment regarding thread-safety). */ -GC_API size_t GC_CALL GC_get_free_bytes(void); - -/* Return the size (in bytes) of the unmapped memory (which is returned */ -/* to the OS but could be remapped back by the collector later unless */ -/* the OS runs out of system/virtual memory). This is an unsynchronized */ -/* getter (see GC_get_heap_size comment regarding thread-safety). */ -GC_API size_t GC_CALL GC_get_unmapped_bytes(void); - -/* Return the number of bytes allocated since the last collection. */ -/* This is an unsynchronized getter (see GC_get_heap_size comment */ -/* regarding thread-safety). */ -GC_API size_t GC_CALL GC_get_bytes_since_gc(void); - -/* Return the number of explicitly deallocated bytes of memory since */ -/* the recent collection. This is an unsynchronized getter. */ -GC_API size_t GC_CALL GC_get_expl_freed_bytes_since_gc(void); - -/* Return the total number of bytes allocated in this process. */ -/* Never decreases, except due to wrapping. This is an unsynchronized */ -/* getter (see GC_get_heap_size comment regarding thread-safety). */ -GC_API size_t GC_CALL GC_get_total_bytes(void); - -/* Return the total number of bytes obtained from OS. Includes the */ -/* unmapped memory. Never decreases. It is an unsynchronized getter. */ -GC_API size_t GC_CALL GC_get_obtained_from_os_bytes(void); - -/* Return the heap usage information. This is a thread-safe (atomic) */ -/* alternative for the five above getters. (This function acquires */ -/* the allocator lock thus preventing data racing and returning the */ -/* consistent result.) Passing NULL pointer is allowed for any */ -/* argument. Returned (filled in) values are of word type. */ -GC_API void GC_CALL GC_get_heap_usage_safe(GC_word * /* pheap_size */, - GC_word * /* pfree_bytes */, - GC_word * /* punmapped_bytes */, - GC_word * /* pbytes_since_gc */, - GC_word * /* ptotal_bytes */); - -/* Structure used to query GC statistics (profiling information). */ -/* More fields could be added in the future. To preserve compatibility */ -/* new fields should be added only to the end, and no deprecated fields */ -/* should be removed from. */ -struct GC_prof_stats_s { - GC_word heapsize_full; - /* Heap size in bytes (including the area unmapped to OS). */ - /* Same as GC_get_heap_size() + GC_get_unmapped_bytes(). */ - GC_word free_bytes_full; - /* Total bytes contained in free and unmapped blocks. */ - /* Same as GC_get_free_bytes() + GC_get_unmapped_bytes(). */ - GC_word unmapped_bytes; - /* Amount of memory unmapped to OS. Same as the value */ - /* returned by GC_get_unmapped_bytes(). */ - GC_word bytes_allocd_since_gc; - /* Number of bytes allocated since the recent collection. */ - /* Same as returned by GC_get_bytes_since_gc(). */ - GC_word allocd_bytes_before_gc; - /* Number of bytes allocated before the recent garbage */ - /* collection. The value may wrap. Same as the result of */ - /* GC_get_total_bytes() - GC_get_bytes_since_gc(). */ - GC_word non_gc_bytes; - /* Number of bytes not considered candidates for garbage */ - /* collection. Same as returned by GC_get_non_gc_bytes(). */ - GC_word gc_no; - /* Garbage collection cycle number. The value may wrap */ - /* (and could be -1). Same as returned by GC_get_gc_no(). */ - GC_word markers_m1; - /* Number of marker threads (excluding the initiating one). */ - /* Same as returned by GC_get_parallel (or 0 if the */ - /* collector is single-threaded). */ - GC_word bytes_reclaimed_since_gc; - /* Approximate number of reclaimed bytes after recent GC. */ - GC_word reclaimed_bytes_before_gc; - /* Approximate number of bytes reclaimed before the recent */ - /* garbage collection. The value may wrap. */ - GC_word expl_freed_bytes_since_gc; - /* Number of bytes freed explicitly since the recent GC. */ - /* Same as returned by GC_get_expl_freed_bytes_since_gc(). */ - GC_word obtained_from_os_bytes; - /* Total amount of memory obtained from OS, in bytes. */ -}; - -/* Atomically get GC statistics (various global counters). Clients */ -/* should pass the size of the buffer (of GC_prof_stats_s type) to fill */ -/* in the values - this is for interoperability between different GC */ -/* versions, an old client could have fewer fields, and vice versa, */ -/* client could use newer gc.h (with more entries declared in the */ -/* structure) than that of the linked libgc binary; in the latter case, */ -/* unsupported (unknown) fields are filled in with -1. Return the size */ -/* (in bytes) of the filled in part of the structure (excluding all */ -/* unknown fields, if any). */ -GC_API size_t GC_CALL GC_get_prof_stats(struct GC_prof_stats_s *, - size_t /* stats_sz */); -#ifdef GC_THREADS -/* Same as above but unsynchronized (i.e., not holding the allocation */ -/* lock). Clients should call it using GC_call_with_alloc_lock to */ -/* avoid data races on multiprocessors. */ -GC_API size_t GC_CALL GC_get_prof_stats_unsafe(struct GC_prof_stats_s *, - size_t /* stats_sz */); -#endif - -/* Get the element value (converted to bytes) at a given index of */ -/* size_map table which provides requested-to-actual allocation size */ -/* mapping. Assumes the collector is initialized. Returns -1 if the */ -/* index is out of size_map table bounds. Does not use synchronization, */ -/* thus clients should call it using GC_call_with_alloc_lock typically */ -/* to avoid data races on multiprocessors. */ -GC_API size_t GC_CALL GC_get_size_map_at(int i); - -/* Count total memory use in bytes by all allocated blocks. Acquires */ -/* the lock. */ -GC_API size_t GC_CALL GC_get_memory_use(void); - -/* Disable garbage collection. Even GC_gcollect calls will be */ -/* ineffective. */ -GC_API void GC_CALL GC_disable(void); - -/* Return 1 (true) if the garbage collection is disabled (i.e., the */ -/* value of GC_dont_gc is non-zero), 0 otherwise. Does not acquire */ -/* the lock. */ -GC_API int GC_CALL GC_is_disabled(void); - -/* Try to re-enable garbage collection. GC_disable() and GC_enable() */ -/* calls nest. Garbage collection is enabled if the number of calls to */ -/* both functions is equal. */ -GC_API void GC_CALL GC_enable(void); - -/* Select whether to use the manual VDB mode for the incremental */ -/* collection. Has no effect if called after enabling the incremental */ -/* collection. The default value is off unless the collector is */ -/* compiled with MANUAL_VDB defined. The manual VDB mode should be */ -/* used only if the client has the appropriate GC_END_STUBBORN_CHANGE */ -/* and GC_reachable_here (or, alternatively, GC_PTR_STORE_AND_DIRTY) */ -/* calls (to ensure proper write barriers). Both the setter and getter */ -/* are not synchronized, and are defined only if the library has been */ -/* compiled without SMALL_CONFIG. */ -GC_API void GC_CALL GC_set_manual_vdb_allowed(int); -GC_API int GC_CALL GC_get_manual_vdb_allowed(void); - -/* Enable incremental/generational collection. Not advisable unless */ -/* dirty bits are available or most heap objects are pointer-free */ -/* (atomic) or immutable. Don't use in leak finding mode. Ignored if */ -/* GC_dont_gc is non-zero. Only the generational piece of this is */ -/* functional if GC_time_limit is set to GC_TIME_UNLIMITED. Causes */ -/* thread-local variant of GC_gcj_malloc() to revert to locked */ -/* allocation. Must be called before any such GC_gcj_malloc() calls. */ -/* For best performance, should be called as early as possible. */ -/* On some platforms, calling it later may have adverse effects. */ -/* Safe to call before GC_INIT(). Includes a GC_init() call. */ -GC_API void GC_CALL GC_enable_incremental(void); - -/* Return 1 (true) if the incremental mode is on, 0 otherwise. */ -/* Does not acquire the lock. */ -GC_API int GC_CALL GC_is_incremental_mode(void); - -#define GC_PROTECTS_POINTER_HEAP 1 /* May protect non-atomic objects. */ -#define GC_PROTECTS_PTRFREE_HEAP 2 -#define GC_PROTECTS_STATIC_DATA 4 /* Currently never. */ -#define GC_PROTECTS_STACK 8 /* Probably impractical. */ - -#define GC_PROTECTS_NONE 0 - -/* Does incremental mode write-protect pages? Returns zero or */ -/* more of the above GC_PROTECTS_*, or'ed together. */ -/* The collector is assumed to be initialized before this call. */ -/* The result is not affected by GC_set_manual_vdb_allowed(). */ -/* Call of GC_enable_incremental() may change the result to */ -/* GC_PROTECTS_NONE if some implementation is chosen at runtime */ -/* not needing to write-protect the pages. */ -GC_API int GC_CALL GC_incremental_protection_needs(void); - -/* Force start of incremental collection. Acquires the GC lock. */ -/* No-op unless GC incremental mode is on. */ -GC_API void GC_CALL GC_start_incremental_collection(void); - -/* Perform some garbage collection work, if appropriate. */ -/* Return 0 if there is no more work to be done (including the */ -/* case when garbage collection is not appropriate). */ -/* Typically performs an amount of work corresponding roughly */ -/* to marking from one page. May do more work if further */ -/* progress requires it, e.g. if incremental collection is */ -/* disabled. It is reasonable to call this in a wait loop */ -/* until it returns 0. */ -GC_API int GC_CALL GC_collect_a_little(void); - -/* Allocate an object of size lb bytes. The client guarantees that as */ -/* long as the object is live, it will be referenced by a pointer that */ -/* points to somewhere within the first GC heap block (hblk) of the */ -/* object. (This should normally be declared volatile to prevent the */ -/* compiler from invalidating this assertion.) This routine is only */ -/* useful if a large array is being allocated. It reduces the chance */ -/* of accidentally retaining such an array as a result of scanning an */ -/* integer that happens to be an address inside the array. (Actually, */ -/* it reduces the chance of the allocator not finding space for such */ -/* an array, since it will try hard to avoid introducing such a false */ -/* reference.) On a SunOS 4.X or Windows system this is recommended */ -/* for arrays likely to be larger than 100 KB or so. For other systems,*/ -/* or if the collector is not configured to recognize all interior */ -/* pointers, the threshold is normally much higher. */ -GC_API GC_ATTR_MALLOC GC_ATTR_ALLOC_SIZE(1) void *GC_CALL - GC_malloc_ignore_off_page(size_t /* lb */); -GC_API GC_ATTR_MALLOC GC_ATTR_ALLOC_SIZE(1) void *GC_CALL - GC_malloc_atomic_ignore_off_page(size_t /* lb */); - -#ifdef GC_ADD_CALLER -#define GC_EXTRAS GC_RETURN_ADDR, __FILE__, __LINE__ -#define GC_EXTRA_PARAMS GC_word ra, const char *s, int i -#else -#define GC_EXTRAS __FILE__, __LINE__ -#define GC_EXTRA_PARAMS const char *s, int i -#endif - -/* The following is only defined if the library has been suitably */ -/* compiled: */ -GC_API GC_ATTR_MALLOC GC_ATTR_ALLOC_SIZE(1) void *GC_CALL - GC_malloc_atomic_uncollectable(size_t /* size_in_bytes */); -GC_API GC_ATTR_MALLOC GC_ATTR_ALLOC_SIZE(1) void *GC_CALL - GC_debug_malloc_atomic_uncollectable(size_t, GC_EXTRA_PARAMS); - -/* Debugging (annotated) allocation. GC_gcollect will check */ -/* objects allocated in this way for overwrites, etc. */ -GC_API GC_ATTR_MALLOC GC_ATTR_ALLOC_SIZE(1) void *GC_CALL - GC_debug_malloc(size_t /* size_in_bytes */, GC_EXTRA_PARAMS); -GC_API GC_ATTR_MALLOC GC_ATTR_ALLOC_SIZE(1) void *GC_CALL - GC_debug_malloc_atomic(size_t /* size_in_bytes */, GC_EXTRA_PARAMS); -GC_API GC_ATTR_MALLOC char *GC_CALL -GC_debug_strdup(const char *, GC_EXTRA_PARAMS); -GC_API GC_ATTR_MALLOC char *GC_CALL -GC_debug_strndup(const char *, size_t, GC_EXTRA_PARAMS) - GC_ATTR_NONNULL(1); -GC_API GC_ATTR_MALLOC GC_ATTR_ALLOC_SIZE(1) void *GC_CALL - GC_debug_malloc_uncollectable(size_t /* size_in_bytes */, - GC_EXTRA_PARAMS); -GC_API GC_ATTR_DEPRECATED void *GC_CALL - GC_debug_malloc_stubborn(size_t /* size_in_bytes */, GC_EXTRA_PARAMS); -GC_API GC_ATTR_MALLOC GC_ATTR_ALLOC_SIZE(1) void *GC_CALL - GC_debug_malloc_ignore_off_page(size_t /* size_in_bytes */, - GC_EXTRA_PARAMS); -GC_API GC_ATTR_MALLOC GC_ATTR_ALLOC_SIZE(1) void *GC_CALL - GC_debug_malloc_atomic_ignore_off_page(size_t /* size_in_bytes */, - GC_EXTRA_PARAMS); -GC_API void GC_CALL GC_debug_free(void *); -GC_API void *GC_CALL GC_debug_realloc(void * /* old_object */, - size_t /* new_size_in_bytes */, GC_EXTRA_PARAMS) - /* 'realloc' attr */ GC_ATTR_ALLOC_SIZE(2); -GC_API GC_ATTR_DEPRECATED void GC_CALL GC_debug_change_stubborn(const void *); -GC_API void GC_CALL GC_debug_end_stubborn_change(const void *) - GC_ATTR_NONNULL(1); - -/* Routines that allocate objects with debug information (like the */ -/* above), but just fill in dummy file and line number information. */ -/* Thus they can serve as drop-in malloc/realloc replacements. This */ -/* can be useful for two reasons: */ -/* 1) It allows the collector to be built with DBG_HDRS_ALL defined */ -/* even if some allocation calls come from 3rd party libraries */ -/* that can't be recompiled. */ -/* 2) On some platforms, the file and line information is redundant, */ -/* since it can be reconstructed from a stack trace. On such */ -/* platforms it may be more convenient not to recompile, e.g. for */ -/* leak detection. This can be accomplished by instructing the */ -/* linker to replace malloc/realloc with these. */ -GC_API GC_ATTR_MALLOC GC_ATTR_ALLOC_SIZE(1) void *GC_CALL - GC_debug_malloc_replacement(size_t /* size_in_bytes */); -GC_API /* 'realloc' attr */ GC_ATTR_ALLOC_SIZE(2) void *GC_CALL - GC_debug_realloc_replacement(void * /* object_addr */, - size_t /* size_in_bytes */); - -/* Convenient macros for disappearing links registration working both */ -/* for debug and non-debug allocated objects, and accepting interior */ -/* pointers to object. */ -#define GC_GENERAL_REGISTER_DISAPPEARING_LINK_SAFE(link, obj) \ - GC_general_register_disappearing_link(link, \ - GC_base((/* no const */ void *)(obj))) -#define GC_REGISTER_LONG_LINK_SAFE(link, obj) \ - GC_register_long_link(link, GC_base((/* no const */ void *)(obj))) - -#ifdef GC_DEBUG_REPLACEMENT -#define GC_MALLOC(sz) GC_debug_malloc_replacement(sz) -#define GC_REALLOC(old, sz) GC_debug_realloc_replacement(old, sz) -#elif defined(GC_DEBUG) -#define GC_MALLOC(sz) GC_debug_malloc(sz, GC_EXTRAS) -#define GC_REALLOC(old, sz) GC_debug_realloc(old, sz, GC_EXTRAS) -#else -#define GC_MALLOC(sz) GC_malloc(sz) -#define GC_REALLOC(old, sz) GC_realloc(old, sz) -#endif /* !GC_DEBUG_REPLACEMENT && !GC_DEBUG */ - -#ifdef GC_DEBUG -#define GC_MALLOC_ATOMIC(sz) GC_debug_malloc_atomic(sz, GC_EXTRAS) -#define GC_STRDUP(s) GC_debug_strdup(s, GC_EXTRAS) -#define GC_STRNDUP(s, sz) GC_debug_strndup(s, sz, GC_EXTRAS) -#define GC_MALLOC_ATOMIC_UNCOLLECTABLE(sz) \ - GC_debug_malloc_atomic_uncollectable(sz, GC_EXTRAS) -#define GC_MALLOC_UNCOLLECTABLE(sz) \ - GC_debug_malloc_uncollectable(sz, GC_EXTRAS) -#define GC_MALLOC_IGNORE_OFF_PAGE(sz) \ - GC_debug_malloc_ignore_off_page(sz, GC_EXTRAS) -#define GC_MALLOC_ATOMIC_IGNORE_OFF_PAGE(sz) \ - GC_debug_malloc_atomic_ignore_off_page(sz, GC_EXTRAS) -#define GC_FREE(p) GC_debug_free(p) -#define GC_REGISTER_FINALIZER(p, f, d, of, od) \ - GC_debug_register_finalizer(p, f, d, of, od) -#define GC_REGISTER_FINALIZER_IGNORE_SELF(p, f, d, of, od) \ - GC_debug_register_finalizer_ignore_self(p, f, d, of, od) -#define GC_REGISTER_FINALIZER_NO_ORDER(p, f, d, of, od) \ - GC_debug_register_finalizer_no_order(p, f, d, of, od) -#define GC_REGISTER_FINALIZER_UNREACHABLE(p, f, d, of, od) \ - GC_debug_register_finalizer_unreachable(p, f, d, of, od) -#define GC_END_STUBBORN_CHANGE(p) GC_debug_end_stubborn_change(p) -#define GC_PTR_STORE_AND_DIRTY(p, q) GC_debug_ptr_store_and_dirty(p, q) -#define GC_GENERAL_REGISTER_DISAPPEARING_LINK(link, obj) \ - GC_GENERAL_REGISTER_DISAPPEARING_LINK_SAFE(link, obj) -#define GC_REGISTER_LONG_LINK(link, obj) \ - GC_REGISTER_LONG_LINK_SAFE(link, obj) -#define GC_REGISTER_DISPLACEMENT(n) GC_debug_register_displacement(n) -#else -#define GC_MALLOC_ATOMIC(sz) GC_malloc_atomic(sz) -#define GC_STRDUP(s) GC_strdup(s) -#define GC_STRNDUP(s, sz) GC_strndup(s, sz) -#define GC_MALLOC_ATOMIC_UNCOLLECTABLE(sz) GC_malloc_atomic_uncollectable(sz) -#define GC_MALLOC_UNCOLLECTABLE(sz) GC_malloc_uncollectable(sz) -#define GC_MALLOC_IGNORE_OFF_PAGE(sz) \ - GC_malloc_ignore_off_page(sz) -#define GC_MALLOC_ATOMIC_IGNORE_OFF_PAGE(sz) \ - GC_malloc_atomic_ignore_off_page(sz) -#define GC_FREE(p) GC_free(p) -#define GC_REGISTER_FINALIZER(p, f, d, of, od) \ - GC_register_finalizer(p, f, d, of, od) -#define GC_REGISTER_FINALIZER_IGNORE_SELF(p, f, d, of, od) \ - GC_register_finalizer_ignore_self(p, f, d, of, od) -#define GC_REGISTER_FINALIZER_NO_ORDER(p, f, d, of, od) \ - GC_register_finalizer_no_order(p, f, d, of, od) -#define GC_REGISTER_FINALIZER_UNREACHABLE(p, f, d, of, od) \ - GC_register_finalizer_unreachable(p, f, d, of, od) -#define GC_END_STUBBORN_CHANGE(p) GC_end_stubborn_change(p) -#define GC_PTR_STORE_AND_DIRTY(p, q) GC_ptr_store_and_dirty(p, q) -#define GC_GENERAL_REGISTER_DISAPPEARING_LINK(link, obj) \ - GC_general_register_disappearing_link(link, obj) -#define GC_REGISTER_LONG_LINK(link, obj) \ - GC_register_long_link(link, obj) -#define GC_REGISTER_DISPLACEMENT(n) GC_register_displacement(n) -#endif /* !GC_DEBUG */ - -/* The following are included because they are often convenient, and */ -/* reduce the chance for a misspecified size argument. But calls may */ -/* expand to something syntactically incorrect if t is a complicated */ -/* type expression. Note that, unlike C++ new operator, these ones */ -/* may return NULL (if out of memory). */ -#define GC_NEW(t) ((t *)GC_MALLOC(sizeof(t))) -#define GC_NEW_ATOMIC(t) ((t *)GC_MALLOC_ATOMIC(sizeof(t))) -#define GC_NEW_UNCOLLECTABLE(t) ((t *)GC_MALLOC_UNCOLLECTABLE(sizeof(t))) - -#ifdef GC_REQUIRE_WCSDUP -/* This might be unavailable on some targets (or not needed). */ -/* wchar_t should be defined in stddef.h */ -GC_API GC_ATTR_MALLOC wchar_t *GC_CALL -GC_wcsdup(const wchar_t *) GC_ATTR_NONNULL(1); -GC_API GC_ATTR_MALLOC wchar_t *GC_CALL -GC_debug_wcsdup(const wchar_t *, GC_EXTRA_PARAMS) GC_ATTR_NONNULL(1); -#ifdef GC_DEBUG -#define GC_WCSDUP(s) GC_debug_wcsdup(s, GC_EXTRAS) -#else -#define GC_WCSDUP(s) GC_wcsdup(s) -#endif -#endif /* GC_REQUIRE_WCSDUP */ - -/* Finalization. Some of these primitives are grossly unsafe. */ -/* The idea is to make them both cheap, and sufficient to build */ -/* a safer layer, closer to Modula-3, Java, or PCedar finalization. */ -/* The interface represents my conclusions from a long discussion */ -/* with Alan Demers, Dan Greene, Carl Hauser, Barry Hayes, */ -/* Christian Jacobi, and Russ Atkinson. It's not perfect, and */ -/* probably nobody else agrees with it. Hans-J. Boehm 3/13/92 */ -typedef void(GC_CALLBACK *GC_finalization_proc)(void * /* obj */, - void * /* client_data */); - -GC_API void GC_CALL GC_register_finalizer(void * /* obj */, - GC_finalization_proc /* fn */, void * /* cd */, - GC_finalization_proc * /* ofn */, void ** /* ocd */) - GC_ATTR_NONNULL(1); -GC_API void GC_CALL GC_debug_register_finalizer(void * /* obj */, - GC_finalization_proc /* fn */, void * /* cd */, - GC_finalization_proc * /* ofn */, void ** /* ocd */) - GC_ATTR_NONNULL(1); -/* When obj is no longer accessible, invoke */ -/* (*fn)(obj, cd). If a and b are inaccessible, and */ -/* a points to b (after disappearing links have been */ -/* made to disappear), then only a will be */ -/* finalized. (If this does not create any new */ -/* pointers to b, then b will be finalized after the */ -/* next collection.) Any finalizable object that */ -/* is reachable from itself by following one or more */ -/* pointers will not be finalized (or collected). */ -/* Thus cycles involving finalizable objects should */ -/* be avoided, or broken by disappearing links. */ -/* All but the last finalizer registered for an object */ -/* is ignored. */ -/* No-op in the leak-finding mode. */ -/* Finalization may be removed by passing 0 as fn. */ -/* Finalizers are implicitly unregistered when they are */ -/* enqueued for finalization (i.e. become ready to be */ -/* finalized). */ -/* The old finalizer and client data are stored in */ -/* *ofn and *ocd. (ofn and/or ocd may be NULL. */ -/* The allocation lock is held while *ofn and *ocd are */ -/* updated. In case of error (no memory to register */ -/* new finalizer), *ofn and *ocd remain unchanged.) */ -/* Fn is never invoked on an accessible object, */ -/* provided hidden pointers are converted to real */ -/* pointers only if the allocation lock is held, and */ -/* such conversions are not performed by finalization */ -/* routines. */ -/* If GC_register_finalizer is aborted as a result of */ -/* a signal, the object may be left with no */ -/* finalization, even if neither the old nor new */ -/* finalizer were NULL. */ -/* Obj should be the starting address of an object */ -/* allocated by GC_malloc or friends. Obj may also be */ -/* NULL or point to something outside GC heap (in this */ -/* case, fn is ignored, *ofn and *ocd are set to NULL). */ -/* Note that any garbage collectible object referenced */ -/* by cd will be considered accessible until the */ -/* finalizer is invoked. */ - -/* Another versions of the above follow. It ignores */ -/* self-cycles, i.e. pointers from a finalizable object to */ -/* itself. There is a stylistic argument that this is wrong, */ -/* but it's unavoidable for C++, since the compiler may */ -/* silently introduce these. It's also benign in that specific */ -/* case. And it helps if finalizable objects are split to */ -/* avoid cycles. */ -/* Note that cd will still be viewed as accessible, even if it */ -/* refers to the object itself. */ -GC_API void GC_CALL GC_register_finalizer_ignore_self(void * /* obj */, - GC_finalization_proc /* fn */, void * /* cd */, - GC_finalization_proc * /* ofn */, void ** /* ocd */) - GC_ATTR_NONNULL(1); -GC_API void GC_CALL GC_debug_register_finalizer_ignore_self(void * /* obj */, - GC_finalization_proc /* fn */, void * /* cd */, - GC_finalization_proc * /* ofn */, void ** /* ocd */) - GC_ATTR_NONNULL(1); - -/* Another version of the above. It ignores all cycles. */ -/* It should probably only be used by Java implementations. */ -/* Note that cd will still be viewed as accessible, even if it */ -/* refers to the object itself. */ -GC_API void GC_CALL GC_register_finalizer_no_order(void * /* obj */, - GC_finalization_proc /* fn */, void * /* cd */, - GC_finalization_proc * /* ofn */, void ** /* ocd */) - GC_ATTR_NONNULL(1); -GC_API void GC_CALL GC_debug_register_finalizer_no_order(void * /* obj */, - GC_finalization_proc /* fn */, void * /* cd */, - GC_finalization_proc * /* ofn */, void ** /* ocd */) - GC_ATTR_NONNULL(1); - -/* This is a special finalizer that is useful when an object's */ -/* finalizer must be run when the object is known to be no */ -/* longer reachable, not even from other finalizable objects. */ -/* It behaves like "normal" finalization, except that the */ -/* finalizer is not run while the object is reachable from */ -/* other objects specifying unordered finalization. */ -/* Effectively it allows an object referenced, possibly */ -/* indirectly, from an unordered finalizable object to override */ -/* the unordered finalization request. */ -/* This can be used in combination with finalizer_no_order so */ -/* as to release resources that must not be released while an */ -/* object can still be brought back to life by other */ -/* finalizers. */ -/* Only works if GC_java_finalization is set. Probably only */ -/* of interest when implementing a language that requires */ -/* unordered finalization (e.g. Java, C#). */ -GC_API void GC_CALL GC_register_finalizer_unreachable(void * /* obj */, - GC_finalization_proc /* fn */, void * /* cd */, - GC_finalization_proc * /* ofn */, void ** /* ocd */) - GC_ATTR_NONNULL(1); -GC_API void GC_CALL GC_debug_register_finalizer_unreachable(void * /* obj */, - GC_finalization_proc /* fn */, void * /* cd */, - GC_finalization_proc * /* ofn */, void ** /* ocd */) - GC_ATTR_NONNULL(1); - -#define GC_NO_MEMORY 2 /* Failure due to lack of memory. */ - -/* The following routine may be used to break cycles between */ -/* finalizable objects, thus causing cyclic finalizable */ -/* objects to be finalized in the correct order. Standard */ -/* use involves calling GC_register_disappearing_link(&p), */ -/* where p is a pointer that is not followed by finalization */ -/* code, and should not be considered in determining */ -/* finalization order. */ -GC_API int GC_CALL GC_register_disappearing_link(void ** /* link */) - GC_ATTR_NONNULL(1); -/* Link should point to a field of a heap allocated */ -/* object obj. *link will be cleared when obj is */ -/* found to be inaccessible. This happens BEFORE any */ -/* finalization code is invoked, and BEFORE any */ -/* decisions about finalization order are made. */ -/* This is useful in telling the finalizer that */ -/* some pointers are not essential for proper */ -/* finalization. This may avoid finalization cycles. */ -/* Note that obj may be resurrected by another */ -/* finalizer, and thus the clearing of *link may */ -/* be visible to non-finalization code. */ -/* There's an argument that an arbitrary action should */ -/* be allowed here, instead of just clearing a pointer. */ -/* But this causes problems if that action alters, or */ -/* examines connectivity. Returns GC_DUPLICATE if link */ -/* was already registered, GC_SUCCESS if registration */ -/* succeeded, GC_NO_MEMORY if it failed for lack of */ -/* memory, and GC_oom_fn did not handle the problem. */ -/* Only exists for backward compatibility. See below: */ - -GC_API int GC_CALL GC_general_register_disappearing_link(void ** /* link */, - const void * /* obj */) - GC_ATTR_NONNULL(1) GC_ATTR_NONNULL(2); -/* A slight generalization of the above. *link is */ -/* cleared when obj first becomes inaccessible. This */ -/* can be used to implement weak pointers easily and */ -/* safely. Typically link will point to a location */ -/* (in a GC-allocated object or not) holding */ -/* a disguised pointer to obj. (A pointer inside */ -/* an "atomic" object is effectively disguised.) */ -/* In this way, weak pointers are broken before any */ -/* object reachable from them gets finalized. */ -/* Each link may be registered only with one obj value, */ -/* i.e. all objects but the last one (link registered */ -/* with) are ignored. This was added after a long */ -/* email discussion with John Ellis. */ -/* link must be non-NULL (and be properly aligned). */ -/* obj must be a pointer to the first word of an object */ -/* allocated by GC_malloc or friends. A link */ -/* disappears when it is unregistered manually, or when */ -/* (*link) is cleared, or when the object containing */ -/* this link is garbage collected. It is unsafe to */ -/* explicitly deallocate the object containing link. */ -/* Explicit deallocation of obj may or may not cause */ -/* link to eventually be cleared. */ -/* No-op in the leak-finding mode. */ -/* This function can be used to implement certain types */ -/* of weak pointers. Note, however, this generally */ -/* requires that the allocation lock is held (see */ -/* GC_call_with_alloc_lock() below) when the disguised */ -/* pointer is accessed. Otherwise a strong pointer */ -/* could be recreated between the time the collector */ -/* decides to reclaim the object and the link is */ -/* cleared. Returns GC_SUCCESS if registration */ -/* succeeded (a new link is registered), GC_DUPLICATE */ -/* if link was already registered (with some object), */ -/* GC_NO_MEMORY if registration failed for lack of */ -/* memory (and GC_oom_fn did not handle the problem), */ -/* GC_UNIMPLEMENTED if GC_find_leak is true. */ - -GC_API int GC_CALL GC_move_disappearing_link(void ** /* link */, - void ** /* new_link */) - GC_ATTR_NONNULL(2); -/* Moves a link previously registered via */ -/* GC_general_register_disappearing_link (or */ -/* GC_register_disappearing_link). Does not change the */ -/* target object of the weak reference. Does not */ -/* change (*new_link) content. May be called with */ -/* new_link equal to link (to check whether link has */ -/* been registered). Returns GC_SUCCESS on success, */ -/* GC_DUPLICATE if there is already another */ -/* disappearing link at the new location (never */ -/* returned if new_link is equal to link), GC_NOT_FOUND */ -/* if no link is registered at the original location. */ - -GC_API int GC_CALL GC_unregister_disappearing_link(void ** /* link */); -/* Undoes a registration by either of the above two */ -/* routines. Returns 0 if link was not actually */ -/* registered (otherwise returns 1). */ - -GC_API int GC_CALL GC_register_long_link(void ** /* link */, - const void * /* obj */) - GC_ATTR_NONNULL(1) GC_ATTR_NONNULL(2); -/* Similar to GC_general_register_disappearing_link but */ -/* *link only gets cleared when obj becomes truly */ -/* inaccessible. An object becomes truly inaccessible */ -/* when it can no longer be resurrected from its */ -/* finalizer (e.g. by assigning itself to a pointer */ -/* traceable from root). This can be used to implement */ -/* long weak pointers easily and safely. */ - -GC_API int GC_CALL GC_move_long_link(void ** /* link */, - void ** /* new_link */) - GC_ATTR_NONNULL(2); -/* Similar to GC_move_disappearing_link but for a link */ -/* previously registered via GC_register_long_link. */ - -GC_API int GC_CALL GC_unregister_long_link(void ** /* link */); -/* Similar to GC_unregister_disappearing_link but for a */ -/* registration by either of the above two routines. */ - -/* Support of toggle-ref style of external memory management */ -/* without hooking up to the host retain/release machinery. */ -/* The idea of toggle-ref is that an external reference to */ -/* an object is kept and it can be either a strong or weak */ -/* reference; a weak reference is used when the external peer */ -/* has no interest in the object, and a strong otherwise. */ -typedef enum { - GC_TOGGLE_REF_DROP, - GC_TOGGLE_REF_STRONG, - GC_TOGGLE_REF_WEAK -} GC_ToggleRefStatus; - -/* The callback is to decide (return) the new state of a given */ -/* object. Invoked by the collector for all objects registered */ -/* for toggle-ref processing. Invoked with the allocation lock */ -/* held (but the "world" is running). */ -typedef GC_ToggleRefStatus(GC_CALLBACK *GC_toggleref_func)(void * /* obj */); - -/* Set (register) a callback that decides the state of a given */ -/* object (by, probably, inspecting its native state). */ -/* The argument may be 0 (means no callback). Both the setter */ -/* and the getter acquire the allocation lock (to avoid data */ -/* races). */ -GC_API void GC_CALL GC_set_toggleref_func(GC_toggleref_func); -GC_API GC_toggleref_func GC_CALL GC_get_toggleref_func(void); - -/* Register a given object for toggle-ref processing. It will */ -/* be stored internally and the toggle-ref callback will be */ -/* invoked on the object until the callback returns */ -/* GC_TOGGLE_REF_DROP or the object is collected. If is_strong */ -/* is true then the object is registered with a strong ref, */ -/* a weak one otherwise. Returns GC_SUCCESS if registration */ -/* succeeded (or no callback registered yet), GC_NO_MEMORY if */ -/* it failed for lack of memory. */ -GC_API int GC_CALL GC_toggleref_add(void * /* obj */, int /* is_strong */) - GC_ATTR_NONNULL(1); - -/* Finalizer callback support. Invoked by the collector (with */ -/* the allocation lock held) for each unreachable object */ -/* enqueued for finalization. */ -typedef void(GC_CALLBACK *GC_await_finalize_proc)(void * /* obj */); -GC_API void GC_CALL GC_set_await_finalize_proc(GC_await_finalize_proc); -GC_API GC_await_finalize_proc GC_CALL GC_get_await_finalize_proc(void); -/* Zero means no callback. The setter */ -/* and getter acquire the lock too. */ - -/* Returns !=0 if GC_invoke_finalizers has something to do. */ -/* Does not use any synchronization. */ -GC_API int GC_CALL GC_should_invoke_finalizers(void); - -/* Set maximum amount of finalizers to run during a single */ -/* invocation of GC_invoke_finalizers. Zero means no limit. */ -/* Both the setter and getter acquire the GC lock. Note that */ -/* invocation of GC_finalize_all resets the maximum amount. */ -GC_API void GC_CALL GC_set_interrupt_finalizers(unsigned); -GC_API unsigned GC_CALL GC_get_interrupt_finalizers(void); - -GC_API int GC_CALL GC_invoke_finalizers(void); -/* Run finalizers for all objects that are ready to */ -/* be finalized. Return the number of finalizers */ -/* that were run. Normally this is also called */ -/* implicitly during some allocations. If */ -/* GC_finalize_on_demand is nonzero, it must be called */ -/* explicitly. */ - -/* Explicitly tell the collector that an object is reachable */ -/* at a particular program point. This prevents the argument */ -/* pointer from being optimized away, even it is otherwise no */ -/* longer needed. It should have no visible effect in the */ -/* absence of finalizers or disappearing links. But it may be */ -/* needed to prevent finalizers from running while the */ -/* associated external resource is still in use. */ -/* The function is sometimes called keep_alive in other */ -/* settings. */ -#if defined(__GNUC__) && !defined(__INTEL_COMPILER) -#if defined(__e2k__) -#define GC_reachable_here(ptr) \ - __asm__ __volatile__(" " \ - : \ - : "r"(ptr) \ - : "memory") -#else -#define GC_reachable_here(ptr) \ - __asm__ __volatile__(" " \ - : \ - : "X"(ptr) \ - : "memory") -#endif -#elif defined(LINT2) -#define GC_reachable_here(ptr) GC_noop1(~(GC_word)(ptr) ^ (~(GC_word)0)) -/* The expression matches the one of COVERT_DATAFLOW(). */ -#else -#define GC_reachable_here(ptr) GC_noop1((GC_word)(ptr)) -#endif - -/* Make the argument appear live to compiler. Should be robust against */ -/* the whole program analysis. */ -GC_API void GC_CALL GC_noop1(GC_word); - -/* GC_set_warn_proc can be used to redirect or filter warning messages. */ -/* p may not be a NULL pointer. msg is printf format string (arg must */ -/* match the format). Both the setter and the getter acquire the GC */ -/* lock (to avoid data races). In GC v7.1 (and before), the setter */ -/* returned the old warn_proc value. */ -typedef void(GC_CALLBACK *GC_warn_proc)(char * /* msg */, - GC_word /* arg */); -GC_API void GC_CALL GC_set_warn_proc(GC_warn_proc /* p */) GC_ATTR_NONNULL(1); -/* GC_get_warn_proc returns the current warn_proc. */ -GC_API GC_warn_proc GC_CALL GC_get_warn_proc(void); - -/* GC_ignore_warn_proc may be used as an argument for GC_set_warn_proc */ -/* to suppress all warnings (unless statistics printing is turned on). */ -GC_API void GC_CALLBACK GC_ignore_warn_proc(char *, GC_word); - -/* Change file descriptor of GC log. Unavailable on some targets. */ -GC_API void GC_CALL GC_set_log_fd(int); - -/* abort_func is invoked on GC fatal aborts (just before OS-dependent */ -/* abort or exit(1) is called). Must be non-NULL. The default one */ -/* outputs msg to stderr provided msg is non-NULL. msg is NULL if */ -/* invoked before exit(1) otherwise msg is non-NULL (i.e., if invoked */ -/* before abort). Both the setter and getter acquire the GC lock. */ -/* Both the setter and getter are defined only if the library has been */ -/* compiled without SMALL_CONFIG. */ -typedef void(GC_CALLBACK *GC_abort_func)(const char * /* msg */); -GC_API void GC_CALL GC_set_abort_func(GC_abort_func) GC_ATTR_NONNULL(1); -GC_API GC_abort_func GC_CALL GC_get_abort_func(void); - -/* A portable way to abort the application because of not enough memory.*/ -GC_API void GC_CALL GC_abort_on_oom(void); - -/* The following is intended to be used by a higher level */ -/* (e.g. Java-like) finalization facility. It is expected */ -/* that finalization code will arrange for hidden pointers to */ -/* disappear. Otherwise objects can be accessed after they */ -/* have been collected. */ -/* Should not be used in the leak-finding mode. */ -/* Note that putting pointers in atomic objects or in */ -/* non-pointer slots of "typed" objects is equivalent to */ -/* disguising them in this way, and may have other advantages. */ -typedef GC_word GC_hidden_pointer; -#define GC_HIDE_POINTER(p) (~(GC_hidden_pointer)(p)) -/* Converting a hidden pointer to a real pointer requires verifying */ -/* that the object still exists. This involves acquiring the */ -/* allocator lock to avoid a race with the collector. */ -#define GC_REVEAL_POINTER(p) ((void *)GC_HIDE_POINTER(p)) - -#if defined(I_HIDE_POINTERS) || defined(GC_I_HIDE_POINTERS) -/* This exists only for compatibility (the GC-prefixed symbols are */ -/* preferred for new code). */ -#define HIDE_POINTER(p) GC_HIDE_POINTER(p) -#define REVEAL_POINTER(p) GC_REVEAL_POINTER(p) -#endif - -/* The routines to acquire/release the allocator lock. */ -/* The lock is not reentrant. GC_alloc_unlock() should not be called */ -/* unless the lock is acquired by the current thread. */ -#ifdef GC_THREADS -GC_API void GC_CALL GC_alloc_lock(void); -GC_API void GC_CALL GC_alloc_unlock(void); -#else -/* No need for real locking if the client is single-threaded. */ -#define GC_alloc_lock() (void)0 -#define GC_alloc_unlock() (void)0 -#endif /* !GC_THREADS */ - -typedef void *(GC_CALLBACK *GC_fn_type)(void * /* client_data */); -GC_API void *GC_CALL GC_call_with_alloc_lock(GC_fn_type /* fn */, - void * /* client_data */) GC_ATTR_NONNULL(1); - -/* These routines are intended to explicitly notify the collector */ -/* of new threads. Often this is unnecessary because thread creation */ -/* is implicitly intercepted by the collector, using header-file */ -/* defines, or linker-based interception. In the long run the intent */ -/* is to always make redundant registration safe. In the short run, */ -/* this is being implemented a platform at a time. */ -/* The interface is complicated by the fact that we probably will not */ -/* ever be able to automatically determine the stack bottom for thread */ -/* stacks on all platforms. */ - -/* Structure representing the bottom (cold end) of a thread stack. */ -/* On most platforms this contains just a single address. */ -struct GC_stack_base { - void *mem_base; /* the bottom of the general-purpose stack */ -#if defined(__e2k__) || defined(__ia64) || defined(__ia64__) || defined(_M_IA64) - void *reg_base; /* the bottom of the register stack */ -#endif -}; - -typedef void *(GC_CALLBACK *GC_stack_base_func)( - struct GC_stack_base * /* sb */, void * /* arg */); - -/* Call a function with a stack base structure corresponding to */ -/* somewhere in the GC_call_with_stack_base frame. This often can */ -/* be used to provide a sufficiently accurate stack bottom. And we */ -/* implement it everywhere. */ -GC_API void *GC_CALL GC_call_with_stack_base(GC_stack_base_func /* fn */, - void * /* arg */) GC_ATTR_NONNULL(1); - -#define GC_SUCCESS 0 -#define GC_DUPLICATE 1 /* Was already registered. */ -#define GC_NO_THREADS 2 /* No thread support in GC. */ - /* GC_NO_THREADS is not returned by any GC function anymore. */ -#define GC_UNIMPLEMENTED 3 /* Not yet implemented on this platform. */ -#define GC_NOT_FOUND 4 /* Requested link not found (returned */ - /* by GC_move_disappearing_link). */ - -/* Start the parallel marker threads, if available. Useful, e.g., */ -/* after POSIX fork in a child process (provided not followed by exec) */ -/* or in single-threaded clients (provided it is OK for the client to */ -/* perform marking in parallel). Acquires the GC lock to avoid a race. */ -GC_API void GC_CALL GC_start_mark_threads(void); - -#if defined(GC_DARWIN_THREADS) || defined(GC_WIN32_THREADS) -/* Use implicit thread registration and processing (via Win32 DllMain */ -/* or Darwin task_threads). Deprecated. Must be called before */ -/* GC_INIT() and other GC routines. Should be avoided if */ -/* GC_pthread_create, GC_beginthreadex (or GC_CreateThread), or */ -/* GC_register_my_thread could be called instead. */ -/* Includes a GC_init() call. Disables parallelized GC on Win32. */ -GC_API void GC_CALL GC_use_threads_discovery(void); -#endif - -#ifdef GC_THREADS -/* Suggest the GC to use the specific signal to suspend threads. */ -/* Has no effect after GC_init and on non-POSIX systems. */ -GC_API void GC_CALL GC_set_suspend_signal(int); - -/* Suggest the GC to use the specific signal to resume threads. */ -/* Has no effect after GC_init and on non-POSIX systems. */ -/* The same signal might be used for threads suspension and restart. */ -GC_API void GC_CALL GC_set_thr_restart_signal(int); - -/* Return the signal number (constant after initialization) used by */ -/* the GC to suspend threads on POSIX systems. Return -1 otherwise. */ -GC_API int GC_CALL GC_get_suspend_signal(void); - -/* Return the signal number (constant after initialization) used by */ -/* the garbage collector to restart (resume) threads on POSIX */ -/* systems. Return -1 otherwise. */ -GC_API int GC_CALL GC_get_thr_restart_signal(void); - -/* Explicitly enable GC_register_my_thread() invocation. */ -/* Done implicitly if a GC thread-creation function is called (or */ -/* implicit thread registration is activated, or the collector is */ -/* compiled with GC_ALWAYS_MULTITHREADED defined). Otherwise, it */ -/* must be called from the main (or any previously registered) thread */ -/* between the collector initialization and the first explicit */ -/* registering of a thread (it should be called as late as possible). */ -/* Includes a GC_start_mark_threads() call. */ -GC_API void GC_CALL GC_allow_register_threads(void); - -/* Register the current thread, with the indicated stack bottom, as */ -/* a new thread whose stack(s) should be traced by the GC. If it */ -/* is not implicitly called by the GC, this must be called before a */ -/* thread can allocate garbage collected memory, or assign pointers */ -/* to the garbage collected heap. Once registered, a thread will be */ -/* stopped during garbage collections. */ -/* This call must be previously enabled (see above). */ -/* This should never be called from the main thread, where it is */ -/* always done implicitly. This is normally done implicitly if GC_ */ -/* functions are called to create the thread, e.g. by including gc.h */ -/* (which redefines some system functions) before calling the system */ -/* thread creation function. Nonetheless, thread cleanup routines */ -/* (e.g., pthread key destructor) typically require manual thread */ -/* registering (and unregistering) if pointers to GC-allocated */ -/* objects are manipulated inside. */ -/* It is also always done implicitly on some platforms if */ -/* GC_use_threads_discovery() is called at start-up. Except for the */ -/* latter case, the explicit call is normally required for threads */ -/* created by third-party libraries. */ -/* A manually registered thread requires manual unregistering. */ -/* Returns GC_SUCCESS on success, GC_DUPLICATE if already registered. */ -GC_API int GC_CALL GC_register_my_thread(const struct GC_stack_base *) - GC_ATTR_NONNULL(1); - -/* Return 1 (true) if the calling (current) thread is registered with */ -/* the garbage collector, 0 otherwise. Acquires the allocator lock. */ -/* If the thread is finished (e.g. running in a destructor and not */ -/* registered manually again), it is considered as not registered. */ -GC_API int GC_CALL GC_thread_is_registered(void); - -/* Notify the collector about the stack and the alt-stack of the */ -/* current thread. normstack and normstack_size are used to */ -/* determine the "normal" stack boundaries when a thread is suspended */ -/* while it is on an alt-stack. */ -GC_API void GC_CALL GC_register_altstack(void * /* normstack */, - GC_word /* normstack_size */, - void * /* altstack */, - GC_word /* altstack_size */); - -/* Unregister the current thread. Only an explicitly registered */ -/* thread (i.e. for which GC_register_my_thread() returns GC_SUCCESS) */ -/* is allowed (and required) to call this function. (As a special */ -/* exception, it is also allowed to once unregister the main thread.) */ -/* The thread may no longer allocate garbage collected memory or */ -/* manipulate pointers to the garbage collected heap after making */ -/* this call. Specifically, if it wants to return or otherwise */ -/* communicate a pointer to the garbage-collected heap to another */ -/* thread, it must do this before calling GC_unregister_my_thread, */ -/* most probably by saving it in a global data structure. Must not */ -/* be called inside a GC callback function (except for */ -/* GC_call_with_stack_base() one). Always returns GC_SUCCESS. */ -GC_API int GC_CALL GC_unregister_my_thread(void); - -/* Stop/start the world explicitly. Not recommended for general use. */ -GC_API void GC_CALL GC_stop_world_external(void); -GC_API void GC_CALL GC_start_world_external(void); -#endif /* GC_THREADS */ - -/* Wrapper for functions that are likely to block (or, at least, do not */ -/* allocate garbage collected memory and/or manipulate pointers to the */ -/* garbage collected heap) for an appreciable length of time. While fn */ -/* is running, the collector is said to be in the "inactive" state for */ -/* the current thread (this means that the thread is not suspended and */ -/* the thread's stack frames "belonging" to the functions in the */ -/* "inactive" state are not scanned during garbage collections). It is */ -/* assumed that the collector is already initialized and the current */ -/* thread is registered. It is allowed for fn to call */ -/* GC_call_with_gc_active() (even recursively), thus temporarily */ -/* toggling the collector's state back to "active". The latter */ -/* technique might be used to make stack scanning more precise (i.e. */ -/* scan only stack frames of functions that allocate garbage collected */ -/* memory and/or manipulate pointers to the garbage collected heap). */ -GC_API void *GC_CALL GC_do_blocking(GC_fn_type /* fn */, - void * /* client_data */) GC_ATTR_NONNULL(1); - -/* Call a function switching to the "active" state of the collector for */ -/* the current thread (i.e. the user function is allowed to call any */ -/* GC function and/or manipulate pointers to the garbage collected */ -/* heap). GC_call_with_gc_active() has the functionality opposite to */ -/* GC_do_blocking() one. It is assumed that the collector is already */ -/* initialized and the current thread is registered. fn may toggle */ -/* the collector thread's state temporarily to "inactive" one by using */ -/* GC_do_blocking. GC_call_with_gc_active() often can be used to */ -/* provide a sufficiently accurate stack bottom. */ -GC_API void *GC_CALL GC_call_with_gc_active(GC_fn_type /* fn */, - void * /* client_data */) GC_ATTR_NONNULL(1); - -/* Attempt to fill in the GC_stack_base structure with the stack bottom */ -/* for this thread. This appears to be required to implement anything */ -/* like the JNI AttachCurrentThread in an environment in which new */ -/* threads are not automatically registered with the collector. */ -/* It is also unfortunately hard to implement well on many platforms. */ -/* Returns GC_SUCCESS or GC_UNIMPLEMENTED. This function acquires the */ -/* GC lock on some platforms. */ -GC_API int GC_CALL GC_get_stack_base(struct GC_stack_base *) - GC_ATTR_NONNULL(1); - -/* Fill in the GC_stack_base structure with the cold end (bottom) of */ -/* the stack of the current thread (or coroutine). */ -/* Unlike GC_get_stack_base, it retrieves the value stored in the */ -/* collector (which is initially set by the collector upon the thread */ -/* is started or registered manually but it could be later updated by */ -/* client using GC_set_stackbottom). Returns the GC-internal non-NULL */ -/* handle of the thread which could be passed to GC_set_stackbottom */ -/* later. It is assumed that the collector is already initialized and */ -/* the thread is registered. Acquires the GC lock to avoid data races. */ -GC_API void *GC_CALL GC_get_my_stackbottom(struct GC_stack_base *) - GC_ATTR_NONNULL(1); - -/* Set the cool end of the user (coroutine) stack of the specified */ -/* thread. The GC thread handle is either the one returned by */ -/* GC_get_my_stackbottom or NULL (the latter designates the current */ -/* thread). The caller should hold the GC lock (e.g. using */ -/* GC_call_with_alloc_lock). Also, the function could be used for */ -/* setting GC_stackbottom value (the bottom of the primordial thread) */ -/* before the collector is initialized (the GC lock is not needed to be */ -/* acquired in this case). */ -GC_API void GC_CALL GC_set_stackbottom(void * /* gc_thread_handle */, - const struct GC_stack_base *) - GC_ATTR_NONNULL(2); - -/* The following routines are primarily intended for use with a */ -/* preprocessor which inserts calls to check C pointer arithmetic. */ -/* They indicate failure by invoking the corresponding _print_proc. */ - -/* Checked pointer pre- and post- increment operations. Note that */ -/* the second argument is in units of bytes, not multiples of the */ -/* object size. This should either be invoked from a macro, or the */ -/* call should be automatically generated. */ -GC_API void *GC_CALL GC_pre_incr(void **, ptrdiff_t /* how_much */) - GC_ATTR_NONNULL(1); -GC_API void *GC_CALL GC_post_incr(void **, ptrdiff_t /* how_much */) - GC_ATTR_NONNULL(1); - -/* Check that p and q point to the same object. GC_same_obj_print_proc */ -/* is called (fail by default) if they do not. Succeeds, as well, if */ -/* neither p nor q points to the heap. (May succeed also if both p and */ -/* q point to between heap objects.) Returns the first argument. */ -/* (The returned value may be hard to use due to typing issues. But if */ -/* we had a suitable preprocessor...) We assume this is somewhat */ -/* performance critical (it should not be called by production code, */ -/* of course, but it can easily make even debugging intolerably slow). */ -GC_API void *GC_CALL GC_same_obj(void * /* p */, void * /* q */); - -/* Check that p is visible to the collector as a possibly pointer */ -/* containing location. If it is not, invoke GC_is_visible_print_proc */ -/* (fail by default). Always returns the argument. May erroneously */ -/* succeed in hard cases. The function is intended for debugging use */ -/* with untyped allocations. (The idea is that it should be possible, */ -/* though slow, to add such a call to all indirect pointer stores.) */ -/* Currently useless for the multi-threaded worlds. */ -GC_API void *GC_CALL GC_is_visible(void * /* p */); - -/* Check that if p is a pointer to a heap page, then it points to */ -/* a valid displacement within a heap object. If it is not, invoke */ -/* GC_is_valid_displacement_print_proc (fail by default). Always */ -/* returns the argument. Uninteresting with all-interior-pointers on. */ -/* Note that we do not lock, since nothing relevant about the header */ -/* should change while we have a valid object pointer to the block. */ -GC_API void *GC_CALL GC_is_valid_displacement(void * /* p */); - -/* Explicitly dump the GC state. This is most often called from the */ -/* debugger, or by setting the GC_DUMP_REGULARLY environment variable, */ -/* but it may be useful to call it from client code during debugging. */ -/* The current collection number is printed in the header of the dump. */ -/* Acquires the GC lock to avoid data races. */ -/* Defined only if the library has been compiled without NO_DEBUGGING. */ -GC_API void GC_CALL GC_dump(void); - -/* The same as GC_dump but allows to specify the name of dump and does */ -/* not acquire the lock. If name is non-NULL, it is printed to help */ -/* identifying individual dumps. Otherwise the current collection */ -/* number is used as the name. */ -/* Defined only if the library has been compiled without NO_DEBUGGING. */ -GC_API void GC_CALL GC_dump_named(const char * /* name */); - -/* Dump information about each block of every GC memory section. */ -/* Defined only if the library has been compiled without NO_DEBUGGING. */ -GC_API void GC_CALL GC_dump_regions(void); - -/* Dump information about every registered disappearing link and */ -/* finalizable object. */ -/* Defined only if the library has been compiled without NO_DEBUGGING. */ -GC_API void GC_CALL GC_dump_finalization(void); - -/* Safer, but slow, pointer addition. Probably useful mainly with */ -/* a preprocessor. Useful only for heap pointers. */ -/* Only the macros without trailing digits are meant to be used */ -/* by clients. These are designed to model the available C pointer */ -/* arithmetic expressions. */ -/* Even then, these are probably more useful as */ -/* documentation than as part of the API. */ -/* Note that GC_PTR_ADD evaluates the first argument more than once. */ -#if defined(GC_DEBUG) && defined(__GNUC__) -#define GC_PTR_ADD3(x, n, type_of_result) \ - ((type_of_result)GC_same_obj((x) + (n), (x))) -#define GC_PRE_INCR3(x, n, type_of_result) \ - ((type_of_result)GC_pre_incr((void **)(&(x)), (n) * sizeof(*x))) -#define GC_POST_INCR3(x, n, type_of_result) \ - ((type_of_result)GC_post_incr((void **)(&(x)), (n) * sizeof(*x))) -#define GC_PTR_ADD(x, n) GC_PTR_ADD3(x, n, __typeof__(x)) -#define GC_PRE_INCR(x, n) GC_PRE_INCR3(x, n, __typeof__(x)) -#define GC_POST_INCR(x) GC_POST_INCR3(x, 1, __typeof__(x)) -#define GC_POST_DECR(x) GC_POST_INCR3(x, -1, __typeof__(x)) -#else /* !GC_DEBUG || !__GNUC__ */ -/* We can't do this right without typeof, which ANSI decided was not */ -/* sufficiently useful. Without it we resort to the non-debug version. */ -/* TODO: This should eventually support C++0x decltype. */ -#define GC_PTR_ADD(x, n) ((x) + (n)) -#define GC_PRE_INCR(x, n) ((x) += (n)) -#define GC_POST_INCR(x) ((x)++) -#define GC_POST_DECR(x) ((x)--) -#endif /* !GC_DEBUG || !__GNUC__ */ - -/* Safer assignment of a pointer to a non-stack location. */ -#ifdef GC_DEBUG -#define GC_PTR_STORE(p, q) \ - (*(void **)GC_is_visible((void *)(p)) = \ - GC_is_valid_displacement((void *)(q))) -#else -#define GC_PTR_STORE(p, q) (*(void **)(p) = (void *)(q)) -#endif - -/* GC_PTR_STORE_AND_DIRTY(p,q) is equivalent to GC_PTR_STORE(p,q) */ -/* followed by GC_END_STUBBORN_CHANGE(p) and GC_reachable_here(q) */ -/* (assuming p and q do not have side effects). */ -GC_API void GC_CALL GC_ptr_store_and_dirty(void * /* p */, - const void * /* q */); -GC_API void GC_CALL GC_debug_ptr_store_and_dirty(void * /* p */, - const void * /* q */); - -#ifdef GC_PTHREADS -/* For pthread support, we generally need to intercept a number of */ -/* thread library calls. We do that here by macro defining them. */ -#ifdef __cplusplus -} /* extern "C" */ -#endif -#include "gc_pthread_redirects.h" -#ifdef __cplusplus -extern "C" { -#endif -#endif - -/* This returns a list of objects, linked through their first word. */ -/* Its use can greatly reduce lock contention problems, since the */ -/* allocation lock can be acquired and released many fewer times. */ -GC_API GC_ATTR_MALLOC void *GC_CALL GC_malloc_many(size_t /* lb */); -#define GC_NEXT(p) (*(void **)(p)) /* Retrieve the next element */ - /* in returned list. */ - -/* A filter function to control the scanning of dynamic libraries. */ -/* If implemented, called by GC before registering a dynamic library */ -/* (discovered by GC) section as a static data root (called only as */ -/* a last reason not to register). The filename of the library, the */ -/* address and the length of the memory region (section) are passed. */ -/* This routine should return nonzero if that region should be scanned. */ -/* Always called with the allocation lock held. Depending on the */ -/* platform, might be called with the "world" stopped. */ -typedef int(GC_CALLBACK *GC_has_static_roots_func)( - const char * /* dlpi_name */, - void * /* section_start */, - size_t /* section_size */); - -/* Register a new callback (a user-supplied filter) to control the */ -/* scanning of dynamic libraries. Replaces any previously registered */ -/* callback. May be 0 (means no filtering). May be unused on some */ -/* platforms (if the filtering is unimplemented or inappropriate). */ -GC_API void GC_CALL GC_register_has_static_roots_callback( - GC_has_static_roots_func); - -#if !defined(CPPCHECK) && !defined(GC_WINDOWS_H_INCLUDED) && defined(WINAPI) -/* windows.h is included before gc.h */ -#define GC_WINDOWS_H_INCLUDED -#endif - -#if defined(GC_WIN32_THREADS) && (!defined(GC_PTHREADS) || defined(GC_BUILD) || defined(GC_WINDOWS_H_INCLUDED)) -/* Note: for Cygwin and pthreads-win32, this is skipped */ -/* unless windows.h is included before gc.h. */ - -#if (!defined(GC_NO_THREAD_DECLS) || defined(GC_BUILD)) && !defined(GC_DONT_INCL_WINDOWS_H) - -#ifdef __cplusplus -} /* Including windows.h in an extern "C" context no longer works. */ -#endif - -#if !defined(_WIN32_WCE) && !defined(__CEGCC__) -#include /* For _beginthreadex, _endthreadex */ -#endif - -#if defined(GC_BUILD) || !defined(GC_DONT_INCLUDE_WINDOWS_H) -#include -#define GC_WINDOWS_H_INCLUDED -#endif - -#ifdef __cplusplus -extern "C" { -#endif - -#ifdef GC_UNDERSCORE_STDCALL -/* Explicitly prefix exported/imported WINAPI (__stdcall) symbols */ -/* with '_' (underscore). Might be useful if MinGW/x86 is used. */ -#define GC_CreateThread _GC_CreateThread -#define GC_ExitThread _GC_ExitThread -#endif - -#ifndef DECLSPEC_NORETURN -/* Typically defined in winnt.h. */ -#ifdef GC_WINDOWS_H_INCLUDED -#define DECLSPEC_NORETURN /* empty */ -#else -#define DECLSPEC_NORETURN __declspec(noreturn) -#endif -#endif - -#if !defined(_UINTPTR_T) && !defined(_UINTPTR_T_DEFINED) && !defined(UINTPTR_MAX) -typedef GC_word GC_uintptr_t; -#else -typedef uintptr_t GC_uintptr_t; -#endif - -#ifdef _WIN64 -#define GC_WIN32_SIZE_T GC_uintptr_t -#elif defined(GC_WINDOWS_H_INCLUDED) -#define GC_WIN32_SIZE_T DWORD -#else -#define GC_WIN32_SIZE_T unsigned long -#endif - -#ifdef GC_INSIDE_DLL -/* Export GC DllMain to be invoked from client DllMain. */ -#ifdef GC_UNDERSCORE_STDCALL -#define GC_DllMain _GC_DllMain -#endif -#ifdef GC_WINDOWS_H_INCLUDED -GC_API BOOL WINAPI GC_DllMain(HINSTANCE /* inst */, - ULONG /* reason */, - LPVOID /* reserved */); -#else -GC_API int __stdcall GC_DllMain(void *, unsigned long, void *); -#endif -#endif /* GC_INSIDE_DLL */ - -/* All threads must be created using GC_CreateThread or */ -/* GC_beginthreadex, or must explicitly call GC_register_my_thread */ -/* (and call GC_unregister_my_thread before thread termination), so */ -/* that they will be recorded in the thread table. For backward */ -/* compatibility, it is possible to build the GC with GC_DLL */ -/* defined, and to call GC_use_threads_discovery. This implicitly */ -/* registers all created threads, but appears to be less robust. */ -/* Currently the collector expects all threads to fall through and */ -/* terminate normally, or call GC_endthreadex() or GC_ExitThread, */ -/* so that the thread is properly unregistered. */ -#ifdef GC_WINDOWS_H_INCLUDED -GC_API HANDLE WINAPI GC_CreateThread( - LPSECURITY_ATTRIBUTES /* lpThreadAttributes */, - GC_WIN32_SIZE_T /* dwStackSize */, - LPTHREAD_START_ROUTINE /* lpStartAddress */, - LPVOID /* lpParameter */, DWORD /* dwCreationFlags */, - LPDWORD /* lpThreadId */); - -GC_API DECLSPEC_NORETURN void WINAPI GC_ExitThread( - DWORD /* dwExitCode */); -#else -struct _SECURITY_ATTRIBUTES; -GC_API void *__stdcall GC_CreateThread(struct _SECURITY_ATTRIBUTES *, - GC_WIN32_SIZE_T, - unsigned long(__stdcall *)(void *), - void *, unsigned long, unsigned long *); -GC_API DECLSPEC_NORETURN void __stdcall GC_ExitThread(unsigned long); -#endif - -#if !defined(_WIN32_WCE) && !defined(__CEGCC__) -GC_API GC_uintptr_t GC_CALL GC_beginthreadex( - void * /* security */, unsigned /* stack_size */, - unsigned(__stdcall *)(void *), - void * /* arglist */, unsigned /* initflag */, - unsigned * /* thrdaddr */); - -/* Note: _endthreadex() is not currently marked as no-return in */ -/* VC++ and MinGW headers, so we don't mark it neither. */ -GC_API void GC_CALL GC_endthreadex(unsigned /* retval */); -#endif /* !_WIN32_WCE */ - -#endif /* !GC_NO_THREAD_DECLS */ - -#ifdef GC_WINMAIN_REDIRECT -/* The collector provides the real WinMain(), which starts a new */ -/* thread to call GC_WinMain() after initializing the GC. */ -#define WinMain GC_WinMain -#endif - -/* For compatibility only. */ -#define GC_use_DllMain GC_use_threads_discovery - -#ifndef GC_NO_THREAD_REDIRECTS -#define CreateThread GC_CreateThread -#define ExitThread GC_ExitThread -#undef _beginthreadex -#define _beginthreadex GC_beginthreadex -#undef _endthreadex -#define _endthreadex GC_endthreadex -/* #define _beginthread { > "Please use _beginthreadex instead of _beginthread" < } */ -#endif /* !GC_NO_THREAD_REDIRECTS */ - -#endif /* GC_WIN32_THREADS */ - -/* Public setter and getter for switching "unmap as much as possible" */ -/* mode on(1) and off(0). Has no effect unless unmapping is turned on. */ -/* Has no effect on implicitly-initiated garbage collections. Initial */ -/* value is controlled by GC_FORCE_UNMAP_ON_GCOLLECT. The setter and */ -/* getter are unsynchronized. */ -GC_API void GC_CALL GC_set_force_unmap_on_gcollect(int); -GC_API int GC_CALL GC_get_force_unmap_on_gcollect(void); - -/* Fully portable code should call GC_INIT() from the main program */ -/* before making any other GC_ calls. On most platforms this is a */ -/* no-op and the collector self-initializes. But a number of */ -/* platforms make that too hard. */ -/* A GC_INIT call is required if the collector is built with */ -/* THREAD_LOCAL_ALLOC defined and the initial allocation call is not */ -/* to GC_malloc() or GC_malloc_atomic(). */ - -#if defined(__CYGWIN32__) || defined(__CYGWIN__) -/* Similarly gnu-win32 DLLs need explicit initialization from the */ -/* main program, as does AIX. */ -#ifdef __x86_64__ -/* Cygwin/x64 does not add leading underscore to symbols anymore. */ -extern int __data_start__[], __data_end__[]; -extern int __bss_start__[], __bss_end__[]; -#define GC_DATASTART ((GC_word)__data_start__ < (GC_word)__bss_start__ \ - ? (void *)__data_start__ \ - : (void *)__bss_start__) -#define GC_DATAEND ((GC_word)__data_end__ > (GC_word)__bss_end__ \ - ? (void *)__data_end__ \ - : (void *)__bss_end__) -#else -extern int _data_start__[], _data_end__[], _bss_start__[], _bss_end__[]; -#define GC_DATASTART ((GC_word)_data_start__ < (GC_word)_bss_start__ \ - ? (void *)_data_start__ \ - : (void *)_bss_start__) -#define GC_DATAEND ((GC_word)_data_end__ > (GC_word)_bss_end__ \ - ? (void *)_data_end__ \ - : (void *)_bss_end__) -#endif /* !__x86_64__ */ -#define GC_INIT_CONF_ROOTS \ - GC_add_roots(GC_DATASTART, GC_DATAEND); \ - GC_gcollect() /* For blacklisting. */ - /* Required at least if GC is in a DLL. And doesn't hurt. */ -#elif defined(_AIX) -extern int _data[], _end[]; -#define GC_DATASTART ((void *)_data) -#define GC_DATAEND ((void *)_end) -#define GC_INIT_CONF_ROOTS GC_add_roots(GC_DATASTART, GC_DATAEND) -#elif (defined(HOST_ANDROID) || defined(__ANDROID__)) && defined(IGNORE_DYNAMIC_LOADING) -/* This is ugly but seems the only way to register data roots of the */ -/* client shared library if the GC dynamic loading support is off. */ -#pragma weak __dso_handle -extern int __dso_handle[]; -GC_API void *GC_CALL GC_find_limit(void * /* start */, int /* up */); -#define GC_INIT_CONF_ROOTS (void)(__dso_handle != 0 \ - ? (GC_add_roots(__dso_handle, \ - GC_find_limit(__dso_handle, \ - 1 /*up*/)), \ - 0) \ - : 0) -#else -#define GC_INIT_CONF_ROOTS /* empty */ -#endif - -#ifdef GC_DONT_EXPAND -/* Set GC_dont_expand to true at start-up. */ -#define GC_INIT_CONF_DONT_EXPAND GC_set_dont_expand(1) -#else -#define GC_INIT_CONF_DONT_EXPAND /* empty */ -#endif - -#ifdef GC_FORCE_UNMAP_ON_GCOLLECT -/* Turn on "unmap as much as possible on explicit GC" mode at start-up */ -#define GC_INIT_CONF_FORCE_UNMAP_ON_GCOLLECT \ - GC_set_force_unmap_on_gcollect(1) -#else -#define GC_INIT_CONF_FORCE_UNMAP_ON_GCOLLECT /* empty */ -#endif - -#ifdef GC_DONT_GC -/* This is for debugging only (useful if environment variables are */ -/* unsupported); cannot call GC_disable as goes before GC_init. */ -#define GC_INIT_CONF_MAX_RETRIES (void)(GC_dont_gc = 1) -#elif defined(GC_MAX_RETRIES) && !defined(CPPCHECK) -/* Set GC_max_retries to the desired value at start-up */ -#define GC_INIT_CONF_MAX_RETRIES GC_set_max_retries(GC_MAX_RETRIES) -#else -#define GC_INIT_CONF_MAX_RETRIES /* empty */ -#endif - -#if defined(GC_ALLOCD_BYTES_PER_FINALIZER) && !defined(CPPCHECK) -/* Set GC_allocd_bytes_per_finalizer to the desired value at start-up. */ -#define GC_INIT_CONF_ALLOCD_BYTES_PER_FINALIZER \ - GC_set_allocd_bytes_per_finalizer(GC_ALLOCD_BYTES_PER_FINALIZER) -#else -#define GC_INIT_CONF_ALLOCD_BYTES_PER_FINALIZER /* empty */ -#endif - -#if defined(GC_FREE_SPACE_DIVISOR) && !defined(CPPCHECK) -/* Set GC_free_space_divisor to the desired value at start-up */ -#define GC_INIT_CONF_FREE_SPACE_DIVISOR \ - GC_set_free_space_divisor(GC_FREE_SPACE_DIVISOR) -#else -#define GC_INIT_CONF_FREE_SPACE_DIVISOR /* empty */ -#endif - -#if defined(GC_FULL_FREQ) && !defined(CPPCHECK) -/* Set GC_full_freq to the desired value at start-up */ -#define GC_INIT_CONF_FULL_FREQ GC_set_full_freq(GC_FULL_FREQ) -#else -#define GC_INIT_CONF_FULL_FREQ /* empty */ -#endif - -#if defined(GC_TIME_LIMIT) && !defined(CPPCHECK) -/* Set GC_time_limit (in ms) to the desired value at start-up. */ -#define GC_INIT_CONF_TIME_LIMIT GC_set_time_limit(GC_TIME_LIMIT) -#else -#define GC_INIT_CONF_TIME_LIMIT /* empty */ -#endif - -#if defined(GC_MARKERS) && defined(GC_THREADS) && !defined(CPPCHECK) -/* Set the number of marker threads (including the initiating */ -/* one) to the desired value at start-up. */ -#define GC_INIT_CONF_MARKERS GC_set_markers_count(GC_MARKERS) -#else -#define GC_INIT_CONF_MARKERS /* empty */ -#endif - -#if defined(GC_SIG_SUSPEND) && defined(GC_THREADS) && !defined(CPPCHECK) -#define GC_INIT_CONF_SUSPEND_SIGNAL GC_set_suspend_signal(GC_SIG_SUSPEND) -#else -#define GC_INIT_CONF_SUSPEND_SIGNAL /* empty */ -#endif - -#if defined(GC_SIG_THR_RESTART) && defined(GC_THREADS) && !defined(CPPCHECK) -#define GC_INIT_CONF_THR_RESTART_SIGNAL \ - GC_set_thr_restart_signal(GC_SIG_THR_RESTART) -#else -#define GC_INIT_CONF_THR_RESTART_SIGNAL /* empty */ -#endif - -#if defined(GC_MAXIMUM_HEAP_SIZE) && !defined(CPPCHECK) -/* Limit the heap size to the desired value (useful for debugging). */ -/* The limit could be overridden either at the program start-up by */ -/* the similar environment variable or anytime later by the */ -/* corresponding API function call. */ -#define GC_INIT_CONF_MAXIMUM_HEAP_SIZE \ - GC_set_max_heap_size(GC_MAXIMUM_HEAP_SIZE) -#else -#define GC_INIT_CONF_MAXIMUM_HEAP_SIZE /* empty */ -#endif - -#ifdef GC_IGNORE_WARN -/* Turn off all warnings at start-up (after GC initialization) */ -#define GC_INIT_CONF_IGNORE_WARN GC_set_warn_proc(GC_ignore_warn_proc) -#else -#define GC_INIT_CONF_IGNORE_WARN /* empty */ -#endif - -#if defined(GC_INITIAL_HEAP_SIZE) && !defined(CPPCHECK) -/* Set heap size to the desired value at start-up */ -#define GC_INIT_CONF_INITIAL_HEAP_SIZE \ - { \ - size_t heap_size = GC_get_heap_size(); \ - if (heap_size < (GC_INITIAL_HEAP_SIZE)) \ - (void)GC_expand_hp((GC_INITIAL_HEAP_SIZE)-heap_size); \ - } -#else -#define GC_INIT_CONF_INITIAL_HEAP_SIZE /* empty */ -#endif - -/* Portable clients should call this at the program start-up. More */ -/* over, some platforms require this call to be done strictly from the */ -/* primordial thread. Multiple invocations are harmless. */ -#define GC_INIT() \ - { \ - GC_INIT_CONF_DONT_EXPAND; /* pre-init */ \ - GC_INIT_CONF_FORCE_UNMAP_ON_GCOLLECT; \ - GC_INIT_CONF_MAX_RETRIES; \ - GC_INIT_CONF_ALLOCD_BYTES_PER_FINALIZER; \ - GC_INIT_CONF_FREE_SPACE_DIVISOR; \ - GC_INIT_CONF_FULL_FREQ; \ - GC_INIT_CONF_TIME_LIMIT; \ - GC_INIT_CONF_MARKERS; \ - GC_INIT_CONF_SUSPEND_SIGNAL; \ - GC_INIT_CONF_THR_RESTART_SIGNAL; \ - GC_INIT_CONF_MAXIMUM_HEAP_SIZE; \ - GC_init(); /* real GC initialization */ \ - GC_INIT_CONF_ROOTS; /* post-init */ \ - GC_INIT_CONF_IGNORE_WARN; \ - GC_INIT_CONF_INITIAL_HEAP_SIZE; \ - } - -/* win32s may not free all resources on process exit. */ -/* This explicitly deallocates the heap. Defined only for Windows. */ -GC_API void GC_CALL GC_win32_free_heap(void); - -#if defined(__SYMBIAN32__) -void GC_init_global_static_roots(void); -#endif - -#if defined(_AMIGA) && !defined(GC_AMIGA_MAKINGLIB) -/* Allocation really goes through GC_amiga_allocwrapper_do. */ -void *GC_amiga_realloc(void *, size_t); -#define GC_realloc(a, b) GC_amiga_realloc(a, b) -void GC_amiga_set_toany(void (*)(void)); -extern int GC_amiga_free_space_divisor_inc; -extern void *(*GC_amiga_allocwrapper_do)(size_t, void *(GC_CALL *)(size_t)); -#define GC_malloc(a) \ - (*GC_amiga_allocwrapper_do)(a, GC_malloc) -#define GC_malloc_atomic(a) \ - (*GC_amiga_allocwrapper_do)(a, GC_malloc_atomic) -#define GC_malloc_uncollectable(a) \ - (*GC_amiga_allocwrapper_do)(a, GC_malloc_uncollectable) -#define GC_malloc_atomic_uncollectable(a) \ - (*GC_amiga_allocwrapper_do)(a, GC_malloc_atomic_uncollectable) -#define GC_malloc_ignore_off_page(a) \ - (*GC_amiga_allocwrapper_do)(a, GC_malloc_ignore_off_page) -#define GC_malloc_atomic_ignore_off_page(a) \ - (*GC_amiga_allocwrapper_do)(a, GC_malloc_atomic_ignore_off_page) -#endif /* _AMIGA && !GC_AMIGA_MAKINGLIB */ - -#ifdef __cplusplus -} /* extern "C" */ -#endif - -#endif /* GC_H */ diff --git a/vendor/bdwgc/private/gc/gc_config_macros.h b/vendor/bdwgc/private/gc/gc_config_macros.h deleted file mode 100644 index 31008d5b..00000000 --- a/vendor/bdwgc/private/gc/gc_config_macros.h +++ /dev/null @@ -1,413 +0,0 @@ -/* - * Copyright (c) 1994 by Xerox Corporation. All rights reserved. - * Copyright (c) 1996 by Silicon Graphics. All rights reserved. - * Copyright (c) 1998 by Fergus Henderson. All rights reserved. - * Copyright (c) 2000-2009 by Hewlett-Packard Development Company. - * All rights reserved. - * Copyright (c) 2008-2020 Ivan Maidanski - * - * THIS MATERIAL IS PROVIDED AS IS, WITH ABSOLUTELY NO WARRANTY EXPRESSED - * OR IMPLIED. ANY USE IS AT YOUR OWN RISK. - * - * Permission is hereby granted to use or copy this program - * for any purpose, provided the above notices are retained on all copies. - * Permission to modify the code and to distribute modified code is granted, - * provided the above notices are retained, and a notice that the code was - * modified is included with the above copyright notice. - */ - -/* This should never be included directly; it is included only from gc.h. */ -/* We separate it only to make gc.h more suitable as documentation. */ -#if defined(GC_H) - -/* Convenient internal macro to test version of gcc. */ -#if defined(__GNUC__) && defined(__GNUC_MINOR__) -#define GC_GNUC_PREREQ(major, minor) \ - ((__GNUC__ << 16) + __GNUC_MINOR__ >= ((major) << 16) + (minor)) -#else -#define GC_GNUC_PREREQ(major, minor) 0 /* FALSE */ -#endif - -/* Some tests for old macros. These violate our namespace rules and */ -/* will disappear shortly. Use the GC_ names. */ -#if defined(SOLARIS_THREADS) || defined(_SOLARIS_THREADS) || defined(_SOLARIS_PTHREADS) || defined(GC_SOLARIS_PTHREADS) -/* We no longer support old style Solaris threads. */ -/* GC_SOLARIS_THREADS now means pthreads. */ -#ifndef GC_SOLARIS_THREADS -#define GC_SOLARIS_THREADS -#endif -#endif -#if defined(IRIX_THREADS) -#define GC_IRIX_THREADS -#endif -#if defined(DGUX_THREADS) && !defined(GC_DGUX386_THREADS) -#define GC_DGUX386_THREADS -#endif -#if defined(AIX_THREADS) -#define GC_AIX_THREADS -#endif -#if defined(HPUX_THREADS) -#define GC_HPUX_THREADS -#endif -#if defined(OSF1_THREADS) -#define GC_OSF1_THREADS -#endif -#if defined(LINUX_THREADS) -#define GC_LINUX_THREADS -#endif -#if defined(WIN32_THREADS) -#define GC_WIN32_THREADS -#endif -#if defined(RTEMS_THREADS) -#define GC_RTEMS_PTHREADS -#endif -#if defined(USE_LD_WRAP) -#define GC_USE_LD_WRAP -#endif - -#if defined(GC_WIN32_PTHREADS) && !defined(GC_WIN32_THREADS) -/* Using pthreads-win32 library (or other Win32 implementation). */ -#define GC_WIN32_THREADS -#endif - -#if defined(GC_AIX_THREADS) || defined(GC_DARWIN_THREADS) || defined(GC_DGUX386_THREADS) || defined(GC_FREEBSD_THREADS) || defined(GC_HPUX_THREADS) || defined(GC_IRIX_THREADS) || defined(GC_LINUX_THREADS) || defined(GC_NETBSD_THREADS) || defined(GC_OPENBSD_THREADS) || defined(GC_OSF1_THREADS) || defined(GC_SOLARIS_THREADS) || defined(GC_WIN32_THREADS) || defined(GC_RTEMS_PTHREADS) -#ifndef GC_THREADS -#define GC_THREADS -#endif -#elif defined(GC_THREADS) -#if defined(__linux__) -#define GC_LINUX_THREADS -#elif defined(__OpenBSD__) -#define GC_OPENBSD_THREADS -#elif defined(_PA_RISC1_1) || defined(_PA_RISC2_0) || defined(hppa) || defined(__HPPA) || (defined(__ia64) && defined(_HPUX_SOURCE)) -#define GC_HPUX_THREADS -#elif defined(__HAIKU__) -#define GC_HAIKU_THREADS -#elif (defined(__DragonFly__) || defined(__FreeBSD_kernel__) || defined(__FreeBSD__)) && !defined(GC_NO_FREEBSD) -#define GC_FREEBSD_THREADS -#elif defined(__NetBSD__) -#define GC_NETBSD_THREADS -#elif defined(__alpha) || defined(__alpha__) /* && !Linux && !xBSD */ -#define GC_OSF1_THREADS -#elif (defined(mips) || defined(__mips) || defined(_mips)) && !(defined(nec_ews) || defined(_nec_ews) || defined(ultrix) || defined(__ultrix)) -#define GC_IRIX_THREADS -#elif defined(__sparc) /* && !Linux */ \ - || ((defined(sun) || defined(__sun)) && (defined(i386) || defined(__i386__) || defined(__amd64) || defined(__amd64__))) -#define GC_SOLARIS_THREADS -#elif defined(__APPLE__) && defined(__MACH__) -#define GC_DARWIN_THREADS -#endif -#if defined(DGUX) && (defined(i386) || defined(__i386__)) -#define GC_DGUX386_THREADS -#endif -#if defined(_AIX) -#define GC_AIX_THREADS -#endif -#if (defined(_WIN32) || defined(_MSC_VER) || defined(__BORLANDC__) || defined(__CYGWIN32__) || defined(__CYGWIN__) || defined(__CEGCC__) || defined(_WIN32_WCE) || defined(__MINGW32__)) && !defined(GC_WIN32_THREADS) -/* Either posix or native Win32 threads. */ -#define GC_WIN32_THREADS -#endif -#if defined(__rtems__) && (defined(i386) || defined(__i386__)) -#define GC_RTEMS_PTHREADS -#endif -#endif /* GC_THREADS */ - -#undef GC_PTHREADS -#if (!defined(GC_WIN32_THREADS) || defined(GC_WIN32_PTHREADS) || defined(__CYGWIN32__) || defined(__CYGWIN__)) && defined(GC_THREADS) && !defined(NN_PLATFORM_CTR) && !defined(NN_BUILD_TARGET_PLATFORM_NX) -/* Posix threads. */ -#define GC_PTHREADS -#endif - -#if !defined(_PTHREADS) && defined(GC_NETBSD_THREADS) -#define _PTHREADS -#endif - -#if defined(GC_DGUX386_THREADS) && !defined(_POSIX4A_DRAFT10_SOURCE) -#define _POSIX4A_DRAFT10_SOURCE 1 -#endif - -#if !defined(_REENTRANT) && defined(GC_PTHREADS) && !defined(GC_WIN32_THREADS) -/* Better late than never. This fails if system headers that depend */ -/* on this were previously included. */ -#define _REENTRANT 1 -#endif - -#define __GC -#if !defined(_WIN32_WCE) || defined(__GNUC__) -#include -#if defined(__MINGW32__) && !defined(_WIN32_WCE) -#include -/* We mention uintptr_t. */ -/* Perhaps this should be included in pure msft environments */ -/* as well? */ -#endif -#else /* _WIN32_WCE */ -/* Yet more kludges for WinCE. */ -#include /* size_t is defined here */ -#ifndef _PTRDIFF_T_DEFINED -/* ptrdiff_t is not defined */ -#define _PTRDIFF_T_DEFINED -typedef long ptrdiff_t; -#endif -#endif /* _WIN32_WCE */ - -#if !defined(GC_NOT_DLL) && !defined(GC_DLL) && ((defined(_DLL) && !defined(__GNUC__)) || (defined(DLL_EXPORT) && defined(GC_BUILD))) -#define GC_DLL -#endif - -#if defined(GC_DLL) && !defined(GC_API) - -#if defined(__CEGCC__) -#if defined(GC_BUILD) -#define GC_API __declspec(dllexport) -#else -#define GC_API __declspec(dllimport) -#endif - -#elif defined(__MINGW32__) -#if defined(__cplusplus) && defined(GC_BUILD) -#define GC_API extern __declspec(dllexport) -#elif defined(GC_BUILD) || defined(__MINGW32_DELAY_LOAD__) -#define GC_API __declspec(dllexport) -#else -#define GC_API extern __declspec(dllimport) -#endif - -#elif defined(_MSC_VER) || defined(__DMC__) || defined(__BORLANDC__) || defined(__CYGWIN__) -#ifdef GC_BUILD -#define GC_API extern __declspec(dllexport) -#else -#define GC_API __declspec(dllimport) -#endif - -#elif defined(__WATCOMC__) -#ifdef GC_BUILD -#define GC_API extern __declspec(dllexport) -#else -#define GC_API extern __declspec(dllimport) -#endif - -#elif defined(__SYMBIAN32__) -#ifdef GC_BUILD -#define GC_API extern EXPORT_C -#else -#define GC_API extern IMPORT_C -#endif - -#elif defined(__GNUC__) -/* Only matters if used in conjunction with -fvisibility=hidden option. */ -#if defined(GC_BUILD) && !defined(GC_NO_VISIBILITY) && (GC_GNUC_PREREQ(4, 0) || defined(GC_VISIBILITY_HIDDEN_SET)) -#define GC_API extern __attribute__((__visibility__("default"))) -#endif -#endif -#endif /* GC_DLL */ - -#ifndef GC_API -#define GC_API extern -#endif - -#ifndef GC_CALL -#define GC_CALL -#endif - -#ifndef GC_CALLBACK -#define GC_CALLBACK GC_CALL -#endif - -#ifndef GC_ATTR_MALLOC -/* 'malloc' attribute should be used for all malloc-like functions */ -/* (to tell the compiler that a function may be treated as if any */ -/* non-NULL pointer it returns cannot alias any other pointer valid */ -/* when the function returns). If the client code violates this rule */ -/* by using custom GC_oom_func then define GC_OOM_FUNC_RETURNS_ALIAS. */ -#ifdef GC_OOM_FUNC_RETURNS_ALIAS -#define GC_ATTR_MALLOC /* empty */ -#elif GC_GNUC_PREREQ(3, 1) -#define GC_ATTR_MALLOC __attribute__((__malloc__)) -#elif defined(_MSC_VER) && (_MSC_VER >= 1900) && !defined(__EDG__) -#define GC_ATTR_MALLOC \ - __declspec(allocator) __declspec(noalias) __declspec(restrict) -#elif defined(_MSC_VER) && _MSC_VER >= 1400 -#define GC_ATTR_MALLOC __declspec(noalias) __declspec(restrict) -#else -#define GC_ATTR_MALLOC -#endif -#endif - -#ifndef GC_ATTR_ALLOC_SIZE -/* 'alloc_size' attribute improves __builtin_object_size correctness. */ -#undef GC_ATTR_CALLOC_SIZE -#ifdef __clang__ -#if __has_attribute(__alloc_size__) -#define GC_ATTR_ALLOC_SIZE(argnum) __attribute__((__alloc_size__(argnum))) -#define GC_ATTR_CALLOC_SIZE(n, s) __attribute__((__alloc_size__(n, s))) -#else -#define GC_ATTR_ALLOC_SIZE(argnum) /* empty */ -#endif -#elif GC_GNUC_PREREQ(4, 3) && !defined(__ICC) -#define GC_ATTR_ALLOC_SIZE(argnum) __attribute__((__alloc_size__(argnum))) -#define GC_ATTR_CALLOC_SIZE(n, s) __attribute__((__alloc_size__(n, s))) -#else -#define GC_ATTR_ALLOC_SIZE(argnum) /* empty */ -#endif -#endif - -#ifndef GC_ATTR_CALLOC_SIZE -#define GC_ATTR_CALLOC_SIZE(n, s) /* empty */ -#endif - -#ifndef GC_ATTR_NONNULL -#if GC_GNUC_PREREQ(4, 0) -#define GC_ATTR_NONNULL(argnum) __attribute__((__nonnull__(argnum))) -#else -#define GC_ATTR_NONNULL(argnum) /* empty */ -#endif -#endif - -#ifndef GC_ATTR_CONST -#if GC_GNUC_PREREQ(4, 0) -#define GC_ATTR_CONST __attribute__((__const__)) -#else -#define GC_ATTR_CONST /* empty */ -#endif -#endif - -#ifndef GC_ATTR_DEPRECATED -#ifdef GC_BUILD -#undef GC_ATTR_DEPRECATED -#define GC_ATTR_DEPRECATED /* empty */ -#elif GC_GNUC_PREREQ(4, 0) -#define GC_ATTR_DEPRECATED __attribute__((__deprecated__)) -#elif defined(_MSC_VER) && _MSC_VER >= 1200 -#define GC_ATTR_DEPRECATED __declspec(deprecated) -#else -#define GC_ATTR_DEPRECATED /* empty */ -#endif -#endif - -#if defined(__sgi) && !defined(__GNUC__) && _COMPILER_VERSION >= 720 -#define GC_ADD_CALLER -#define GC_RETURN_ADDR (GC_word) __return_address -#endif - -#if defined(__linux__) || defined(__GLIBC__) -#if !defined(__native_client__) -#include -#endif -#if (__GLIBC__ == 2 && __GLIBC_MINOR__ >= 1 || __GLIBC__ > 2) && !defined(__ia64__) && !defined(GC_MISSING_EXECINFO_H) && !defined(GC_HAVE_BUILTIN_BACKTRACE) -#define GC_HAVE_BUILTIN_BACKTRACE -#endif -#if defined(__i386__) || defined(__amd64__) || defined(__x86_64__) -#define GC_CAN_SAVE_CALL_STACKS -#endif -#endif /* GLIBC */ - -#if defined(_MSC_VER) && _MSC_VER >= 1200 /* version 12.0+ (MSVC 6.0+) */ \ - && !defined(_M_ARM) && !defined(_M_ARM64) && !defined(_AMD64_) && !defined(_M_X64) && !defined(_WIN32_WCE) && !defined(GC_HAVE_NO_BUILTIN_BACKTRACE) && !defined(GC_HAVE_BUILTIN_BACKTRACE) -#define GC_HAVE_BUILTIN_BACKTRACE -#endif - -#if defined(GC_HAVE_BUILTIN_BACKTRACE) && !defined(GC_CAN_SAVE_CALL_STACKS) -#define GC_CAN_SAVE_CALL_STACKS -#endif - -#if defined(__sparc__) -#define GC_CAN_SAVE_CALL_STACKS -#endif - -/* If we're on a platform on which we can't save call stacks, but */ -/* gcc is normally used, we go ahead and define GC_ADD_CALLER. */ -/* We make this decision independent of whether gcc is actually being */ -/* used, in order to keep the interface consistent, and allow mixing */ -/* of compilers. */ -/* This may also be desirable if it is possible but expensive to */ -/* retrieve the call chain. */ -#if (defined(__linux__) || defined(__DragonFly__) || defined(__FreeBSD__) || defined(__FreeBSD_kernel__) || defined(__HAIKU__) || defined(__NetBSD__) || defined(__OpenBSD__) || defined(HOST_ANDROID) || defined(__ANDROID__)) && !defined(GC_CAN_SAVE_CALL_STACKS) -#define GC_ADD_CALLER -#if GC_GNUC_PREREQ(2, 95) -/* gcc knows how to retrieve return address, but we don't know */ -/* how to generate call stacks. */ -#define GC_RETURN_ADDR (GC_word) __builtin_return_address(0) -#if GC_GNUC_PREREQ(4, 0) && (defined(__i386__) || defined(__amd64__) || defined(__x86_64__) /* and probably others... */) && !defined(GC_NO_RETURN_ADDR_PARENT) -#define GC_HAVE_RETURN_ADDR_PARENT -#define GC_RETURN_ADDR_PARENT \ - (GC_word) __builtin_extract_return_addr(__builtin_return_address(1)) -/* Note: a compiler might complain that calling */ -/* __builtin_return_address with a nonzero argument is unsafe. */ -#endif -#else -/* Just pass 0 for gcc compatibility. */ -#define GC_RETURN_ADDR 0 -#endif -#endif /* !GC_CAN_SAVE_CALL_STACKS */ - -#ifdef GC_PTHREADS - -#if (defined(GC_DARWIN_THREADS) || defined(GC_WIN32_PTHREADS) || defined(__native_client__) || defined(GC_RTEMS_PTHREADS)) && !defined(GC_NO_DLOPEN) -/* Either there is no dlopen() or we do not need to intercept it. */ -#define GC_NO_DLOPEN -#endif - -#if (defined(GC_DARWIN_THREADS) || defined(GC_WIN32_PTHREADS) || defined(GC_OPENBSD_THREADS) || defined(__native_client__)) && !defined(GC_NO_PTHREAD_SIGMASK) -/* Either there is no pthread_sigmask() or no need to intercept it. */ -#define GC_NO_PTHREAD_SIGMASK -#endif - -#if defined(__native_client__) -/* At present, NaCl pthread_create() prototype does not have */ -/* "const" for its "attr" argument; also, NaCl pthread_exit() one */ -/* does not have "noreturn" attribute. */ -#ifndef GC_PTHREAD_CREATE_CONST -#define GC_PTHREAD_CREATE_CONST /* empty */ -#endif -#ifndef GC_HAVE_PTHREAD_EXIT -#define GC_HAVE_PTHREAD_EXIT -#define GC_PTHREAD_EXIT_ATTRIBUTE /* empty */ -#endif -#endif - -#if !defined(GC_HAVE_PTHREAD_EXIT) && !defined(HOST_ANDROID) && !defined(__ANDROID__) && (defined(GC_LINUX_THREADS) || defined(GC_SOLARIS_THREADS)) -#define GC_HAVE_PTHREAD_EXIT -/* Intercept pthread_exit on Linux and Solaris. */ -#if GC_GNUC_PREREQ(2, 7) -#define GC_PTHREAD_EXIT_ATTRIBUTE __attribute__((__noreturn__)) -#elif defined(__NORETURN) /* used in Solaris */ -#define GC_PTHREAD_EXIT_ATTRIBUTE __NORETURN -#else -#define GC_PTHREAD_EXIT_ATTRIBUTE /* empty */ -#endif -#endif - -#if (!defined(GC_HAVE_PTHREAD_EXIT) || defined(__native_client__)) && !defined(GC_NO_PTHREAD_CANCEL) -/* Either there is no pthread_cancel() or no need to intercept it. */ -#define GC_NO_PTHREAD_CANCEL -#endif - -#endif /* GC_PTHREADS */ - -#ifdef __cplusplus - -#ifndef GC_ATTR_EXPLICIT -#if __cplusplus >= 201103L && !defined(__clang__) || _MSVC_LANG >= 201103L || defined(CPPCHECK) -#define GC_ATTR_EXPLICIT explicit -#else -#define GC_ATTR_EXPLICIT /* empty */ -#endif -#endif - -#ifndef GC_NOEXCEPT -#if defined(__DMC__) || (defined(__BORLANDC__) && (defined(_RWSTD_NO_EXCEPTIONS) || defined(_RWSTD_NO_EX_SPEC))) || (defined(_MSC_VER) && defined(_HAS_EXCEPTIONS) && !_HAS_EXCEPTIONS) || (defined(__WATCOMC__) && !defined(_CPPUNWIND)) -#define GC_NOEXCEPT /* empty */ -#ifndef GC_NEW_ABORTS_ON_OOM -#define GC_NEW_ABORTS_ON_OOM -#endif -#elif __cplusplus >= 201103L || _MSVC_LANG >= 201103L -#define GC_NOEXCEPT noexcept -#else -#define GC_NOEXCEPT throw() -#endif -#endif - -#endif /* __cplusplus */ - -#endif diff --git a/vendor/bdwgc/private/gc/gc_inline.h b/vendor/bdwgc/private/gc/gc_inline.h deleted file mode 100644 index 579bfe12..00000000 --- a/vendor/bdwgc/private/gc/gc_inline.h +++ /dev/null @@ -1,208 +0,0 @@ -/* - * Copyright 1988, 1989 Hans-J. Boehm, Alan J. Demers - * Copyright (c) 1991-1995 by Xerox Corporation. All rights reserved. - * Copyright (c) 2005 Hewlett-Packard Development Company, L.P. - * Copyright (c) 2008-2022 Ivan Maidanski - * - * THIS MATERIAL IS PROVIDED AS IS, WITH ABSOLUTELY NO WARRANTY EXPRESSED - * OR IMPLIED. ANY USE IS AT YOUR OWN RISK. - * - * Permission is hereby granted to use or copy this program - * for any purpose, provided the above notices are retained on all copies. - * Permission to modify the code and to distribute modified code is granted, - * provided the above notices are retained, and a notice that the code was - * modified is included with the above copyright notice. - */ - -#ifndef GC_INLINE_H -#define GC_INLINE_H - -/* WARNING: */ -/* Note that for these routines, it is the clients responsibility to */ -/* add the extra byte at the end to deal with one-past-the-end pointers.*/ -/* In the standard collector configuration, the collector assumes that */ -/* such a byte has been added, and hence does not trace the last word */ -/* in the resulting object. */ -/* This is not an issue if the collector is compiled with */ -/* DONT_ADD_BYTE_AT_END, or if GC_all_interior_pointers is not set. */ -/* This interface is most useful for compilers that generate C. */ -/* It is also used internally for thread-local allocation. */ -/* Manual use is hereby discouraged. */ -/* Clients should include atomic_ops.h (or similar) before this header. */ -/* There is no debugging version of this allocation API. */ - -#include "gc.h" -#include "gc_tiny_fl.h" - -#if GC_GNUC_PREREQ(3, 0) -#define GC_EXPECT(expr, outcome) __builtin_expect(expr, outcome) -/* Equivalent to (expr), but predict that usually (expr)==outcome. */ -#else -#define GC_EXPECT(expr, outcome) (expr) -#endif - -#ifndef GC_ASSERT -#ifdef NDEBUG -#define GC_ASSERT(expr) /* empty */ -#else -#include -#define GC_ASSERT(expr) assert(expr) -#endif -#endif - -#ifdef __cplusplus -extern "C" { -#endif - -#ifndef GC_PREFETCH_FOR_WRITE -#if GC_GNUC_PREREQ(3, 0) && !defined(GC_NO_PREFETCH_FOR_WRITE) -#define GC_PREFETCH_FOR_WRITE(x) __builtin_prefetch((x), 1) -#else -#define GC_PREFETCH_FOR_WRITE(x) (void)0 -#endif -#endif - -/* Object kinds (exposed to public). */ -#define GC_I_PTRFREE 0 -#define GC_I_NORMAL 1 - -/* Store a pointer to a list of newly allocated objects of kind k and */ -/* size lb in *result. The caller must make sure that *result is */ -/* traced even if objects are ptrfree. */ -GC_API void GC_CALL GC_generic_malloc_many(size_t /* lb */, int /* k */, - void ** /* result */); - -/* Generalized version of GC_malloc and GC_malloc_atomic. */ -/* Uses appropriately the thread-local (if available) or the global */ -/* free-list of the specified kind. */ -GC_API GC_ATTR_MALLOC GC_ATTR_ALLOC_SIZE(1) void *GC_CALL - GC_malloc_kind(size_t /* lb */, int /* k */); - -#ifdef GC_THREADS -/* Same as above but uses only the global free-list. */ -GC_API GC_ATTR_MALLOC GC_ATTR_ALLOC_SIZE(1) void *GC_CALL - GC_malloc_kind_global(size_t /* lb */, int /* k */); -#else -#define GC_malloc_kind_global GC_malloc_kind -#endif - -/* An internal macro to update the free list pointer atomically (if */ -/* the AO primitives are available) to avoid race with the marker. */ -#if defined(GC_THREADS) && defined(AO_HAVE_store) -#define GC_FAST_M_AO_STORE(my_fl, next) \ - AO_store((volatile AO_t *)(my_fl), (AO_t)(next)) -#else -#define GC_FAST_M_AO_STORE(my_fl, next) (void)(*(my_fl) = (next)) -#endif - -/* The ultimately general inline allocation macro. Allocate an object */ -/* of size granules, putting the resulting pointer in result. Tiny_fl */ -/* is a "tiny" free list array, which will be used first, if the size */ -/* is appropriate. If granules argument is too large, we allocate with */ -/* default_expr instead. If we need to refill the free list, we use */ -/* GC_generic_malloc_many with the indicated kind. */ -/* Tiny_fl should be an array of GC_TINY_FREELISTS void * pointers. */ -/* If num_direct is nonzero, and the individual free list pointers */ -/* are initialized to (void *)1, then we allocate num_direct granules */ -/* directly using generic_malloc before putting multiple objects into */ -/* the tiny_fl entry. If num_direct is zero, then the free lists may */ -/* also be initialized to (void *)0. */ -/* Note that we use the zeroth free list to hold objects 1 granule in */ -/* size that are used to satisfy size 0 allocation requests. */ -/* We rely on much of this hopefully getting optimized away in the */ -/* num_direct = 0 case. */ -/* Particularly, if granules argument is constant, this should generate */ -/* a small amount of code. */ -#define GC_FAST_MALLOC_GRANS(result, granules, tiny_fl, num_direct, \ - kind, default_expr, init) \ - do { \ - if (GC_EXPECT((granules) >= GC_TINY_FREELISTS, 0)) { \ - result = (default_expr); \ - } else { \ - void **my_fl = (tiny_fl) + (granules); \ - void *my_entry = *my_fl; \ - void *next; \ - \ - for (;;) { \ - if (GC_EXPECT((GC_word)my_entry > (num_direct) + GC_TINY_FREELISTS + 1, 1)) { \ - next = *(void **)(my_entry); \ - result = (void *)my_entry; \ - GC_FAST_M_AO_STORE(my_fl, next); \ - init; \ - GC_PREFETCH_FOR_WRITE(next); \ - if ((kind) != GC_I_PTRFREE) { \ - GC_end_stubborn_change(my_fl); \ - GC_reachable_here(next); \ - } \ - GC_ASSERT(GC_size(result) >= (granules)*GC_GRANULE_BYTES); \ - GC_ASSERT((kind) == GC_I_PTRFREE || ((GC_word *)result)[1] == 0); \ - break; \ - } \ - /* Entry contains counter or NULL */ \ - if ((GC_signed_word)my_entry - (GC_signed_word)(num_direct) <= 0 /* (GC_word)my_entry <= (num_direct) */ \ - && my_entry != 0 /* NULL */) { \ - /* Small counter value, not NULL */ \ - GC_FAST_M_AO_STORE(my_fl, (char *)my_entry + (granules) + 1); \ - result = (default_expr); \ - break; \ - } else { \ - /* Large counter or NULL */ \ - GC_generic_malloc_many(((granules) == 0 ? GC_GRANULE_BYTES : GC_RAW_BYTES_FROM_INDEX(granules)), \ - kind, my_fl); \ - my_entry = *my_fl; \ - if (my_entry == 0) { \ - result = (*GC_get_oom_fn())((granules)*GC_GRANULE_BYTES); \ - break; \ - } \ - } \ - } \ - } \ - } while (0) - -#define GC_WORDS_TO_WHOLE_GRANULES(n) \ - GC_WORDS_TO_GRANULES((n) + GC_GRANULE_WORDS - 1) - -/* Allocate n words (NOT BYTES). X is made to point to the result. */ -/* This should really only be used if GC_all_interior_pointers is */ -/* not set, or DONT_ADD_BYTE_AT_END is set. See above. */ -/* Does not acquire lock. The caller is responsible for supplying */ -/* a cleared tiny_fl free list array. For single-threaded */ -/* applications, this may be a global array. */ -#define GC_MALLOC_WORDS_KIND(result, n, tiny_fl, kind, init) \ - do { \ - size_t granules = GC_WORDS_TO_WHOLE_GRANULES(n); \ - GC_FAST_MALLOC_GRANS(result, granules, tiny_fl, 0, kind, \ - GC_malloc_kind(granules *GC_GRANULE_BYTES, kind), \ - init); \ - } while (0) - -#define GC_MALLOC_WORDS(result, n, tiny_fl) \ - GC_MALLOC_WORDS_KIND(result, n, tiny_fl, GC_I_NORMAL, \ - *(void **)(result) = 0) - -#define GC_MALLOC_ATOMIC_WORDS(result, n, tiny_fl) \ - GC_MALLOC_WORDS_KIND(result, n, tiny_fl, GC_I_PTRFREE, (void)0) - -/* And once more for two word initialized objects: */ -#define GC_CONS(result, first, second, tiny_fl) \ - do { \ - void *l = (void *)(first); \ - void *r = (void *)(second); \ - GC_MALLOC_WORDS_KIND(result, 2, tiny_fl, GC_I_NORMAL, (void)0); \ - if ((result) != 0 /* NULL */) { \ - *(void **)(result) = l; \ - GC_ptr_store_and_dirty((void **)(result) + 1, r); \ - GC_reachable_here(l); \ - } \ - } while (0) - -/* Print address of each object in the free list. */ -/* Defined only if the library has been compiled without NO_DEBUGGING. */ -GC_API void GC_CALL GC_print_free_list(int /* kind */, - size_t /* sz_in_granules */); - -#ifdef __cplusplus -} /* extern "C" */ -#endif - -#endif /* !GC_INLINE_H */ diff --git a/vendor/bdwgc/private/gc/gc_mark.h b/vendor/bdwgc/private/gc/gc_mark.h deleted file mode 100644 index c0c028f7..00000000 --- a/vendor/bdwgc/private/gc/gc_mark.h +++ /dev/null @@ -1,388 +0,0 @@ -/* - * Copyright (c) 1991-1994 by Xerox Corporation. All rights reserved. - * Copyright (c) 2001 by Hewlett-Packard Company. All rights reserved. - * Copyright (c) 2009-2022 Ivan Maidanski - * - * THIS MATERIAL IS PROVIDED AS IS, WITH ABSOLUTELY NO WARRANTY EXPRESSED - * OR IMPLIED. ANY USE IS AT YOUR OWN RISK. - * - * Permission is hereby granted to use or copy this program - * for any purpose, provided the above notices are retained on all copies. - * Permission to modify the code and to distribute modified code is granted, - * provided the above notices are retained, and a notice that the code was - * modified is included with the above copyright notice. - * - */ - -/* - * This contains interfaces to the GC marker that are likely to be useful to - * clients that provide detailed heap layout information to the collector. - * This interface should not be used by normal C or C++ clients. - * It will be useful to runtimes for other languages. - * - * This is an experts-only interface! There are many ways to break the - * collector in subtle ways by using this functionality. - */ -#ifndef GC_MARK_H -#define GC_MARK_H - -#ifndef GC_H -#include "gc.h" -#endif - -#ifdef __cplusplus -extern "C" { -#endif - -#define GC_PROC_BYTES 100 - -#if defined(GC_BUILD) || defined(NOT_GCBUILD) -struct GC_ms_entry; -struct GC_hblk_s; -#else -struct GC_ms_entry { - void *opaque; -}; -struct GC_hblk_s { - void *opaque; -}; -#endif - -/* A client supplied mark procedure. Returns new mark stack pointer. */ -/* Primary effect should be to push new entries on the mark stack. */ -/* Mark stack pointer values are passed and returned explicitly. */ -/* Global variables describing mark stack are not necessarily valid. */ -/* (This usually saves a few cycles by keeping things in registers.) */ -/* Assumed to scan about GC_PROC_BYTES on average. If it needs to do */ -/* much more work than that, it should do it in smaller pieces by */ -/* pushing itself back on the mark stack. */ -/* Note that it should always do some work (defined as marking some */ -/* objects) before pushing more than one entry on the mark stack. */ -/* This is required to ensure termination in the event of mark stack */ -/* overflows. */ -/* This procedure is always called with at least one empty entry on the */ -/* mark stack. */ -/* Currently we require that mark procedures look for pointers in a */ -/* subset of the places the conservative marker would. It must be safe */ -/* to invoke the normal mark procedure instead. */ -/* WARNING: Such a mark procedure may be invoked on an unused object */ -/* residing on a free list. Such objects are cleared, except for a */ -/* free list link field in the first word. Thus mark procedures may */ -/* not count on the presence of a type descriptor, and must handle this */ -/* case correctly somehow. Also, a mark procedure should be prepared */ -/* to be executed concurrently from the marker threads (the later ones */ -/* are created only if the client has called GC_start_mark_threads() */ -/* or started a user thread previously). */ -typedef struct GC_ms_entry *(*GC_mark_proc)(GC_word * /* addr */, - struct GC_ms_entry * /* mark_stack_ptr */, - struct GC_ms_entry * /* mark_stack_limit */, - GC_word /* env */); - -#define GC_LOG_MAX_MARK_PROCS 6 -#define GC_MAX_MARK_PROCS (1 << GC_LOG_MAX_MARK_PROCS) - -/* In a few cases it's necessary to assign statically known indices to */ -/* certain mark procs. Thus we reserve a few for well known clients. */ -/* (This is necessary if mark descriptors are compiler generated.) */ -#define GC_RESERVED_MARK_PROCS 8 -#define GC_GCJ_RESERVED_MARK_PROC_INDEX 0 - -/* Object descriptors on mark stack or in objects. Low order two */ -/* bits are tags distinguishing among the following 4 possibilities */ -/* for the high order 30 bits. */ -#define GC_DS_TAG_BITS 2 -#define GC_DS_TAGS ((1 << GC_DS_TAG_BITS) - 1) -#define GC_DS_LENGTH 0 /* The entire word is a length in bytes that */ - /* must be a multiple of 4. */ -#define GC_DS_BITMAP 1 /* 30 (62) bits are a bitmap describing pointer */ - /* fields. The msb is 1 if the first word */ - /* is a pointer. */ - /* (This unconventional ordering sometimes */ - /* makes the marker slightly faster.) */ - /* Zeroes indicate definite nonpointers. Ones */ - /* indicate possible pointers. */ - /* Only usable if pointers are word aligned. */ -#define GC_DS_PROC 2 -/* The objects referenced by this object can be */ -/* pushed on the mark stack by invoking */ -/* PROC(descr). ENV(descr) is passed as the */ -/* last argument. */ -#define GC_MAKE_PROC(proc_index, env) \ - (((((env) << GC_LOG_MAX_MARK_PROCS) | (proc_index)) << GC_DS_TAG_BITS) | GC_DS_PROC) -#define GC_DS_PER_OBJECT 3 /* The real descriptor is at the */ - /* byte displacement from the beginning of the */ - /* object given by descr & ~GC_DS_TAGS. */ - /* If the descriptor is negative, the real */ - /* descriptor is at (*) - */ - /* (descr&~GC_DS_TAGS) - GC_INDIR_PER_OBJ_BIAS */ - /* The latter alternative can be used if each */ - /* object contains a type descriptor in the */ - /* first word. */ - /* Note that in the multi-threaded environments */ - /* per-object descriptors must be located in */ - /* either the first two or last two words of */ - /* the object, since only those are guaranteed */ - /* to be cleared while the allocation lock is */ - /* held. */ -#define GC_INDIR_PER_OBJ_BIAS 0x10 - -GC_API void *GC_least_plausible_heap_addr; -GC_API void *GC_greatest_plausible_heap_addr; -/* Bounds on the heap. Guaranteed valid */ -/* Likely to include future heap expansion. */ -/* Hence usually includes not-yet-mapped */ -/* memory. */ - -/* Handle nested references in a custom mark procedure. */ -/* Check if obj is a valid object. If so, ensure that it is marked. */ -/* If it was not previously marked, push its contents onto the mark */ -/* stack for future scanning. The object will then be scanned using */ -/* its mark descriptor. */ -/* Returns the new mark stack pointer. */ -/* Handles mark stack overflows correctly. */ -/* Since this marks first, it makes progress even if there are mark */ -/* stack overflows. */ -/* Src is the address of the pointer to obj, which is used only */ -/* for back pointer-based heap debugging. */ -/* It is strongly recommended that most objects be handled without mark */ -/* procedures, e.g. with bitmap descriptors, and that mark procedures */ -/* be reserved for exceptional cases. That will ensure that */ -/* performance of this call is not extremely performance critical. */ -/* (Otherwise we would need to inline GC_mark_and_push completely, */ -/* which would tie the client code to a fixed collector version.) */ -/* Note that mark procedures should explicitly call FIXUP_POINTER() */ -/* if required. */ -GC_API struct GC_ms_entry *GC_CALL GC_mark_and_push(void * /* obj */, - struct GC_ms_entry * /* mark_stack_ptr */, - struct GC_ms_entry * /* mark_stack_limit */, - void ** /* src */); - -#define GC_MARK_AND_PUSH(obj, msp, lim, src) \ - ((GC_word)(obj) >= (GC_word)GC_least_plausible_heap_addr && \ - (GC_word)(obj) <= (GC_word)GC_greatest_plausible_heap_addr \ - ? GC_mark_and_push(obj, msp, lim, src) \ - : (msp)) - -/* The size of the header added to objects allocated through the */ -/* GC_debug routines. Defined as a function so that client mark */ -/* procedures do not need to be recompiled for the collector library */ -/* version changes. */ -GC_API GC_ATTR_CONST size_t GC_CALL GC_get_debug_header_size(void); -#define GC_USR_PTR_FROM_BASE(p) \ - ((void *)((char *)(p) + GC_get_debug_header_size())) - -/* The same but defined as a variable. Exists only for the backward */ -/* compatibility. Some compilers do not accept "const" together with */ -/* deprecated or dllimport attributes, so the symbol is exported as */ -/* a non-constant one. */ -GC_API GC_ATTR_DEPRECATED -#ifdef GC_BUILD - const -#endif - size_t GC_debug_header_size; - -/* Return the heap block size. Each heap block is devoted to a single */ -/* size and kind of object. */ -GC_API GC_ATTR_CONST size_t GC_CALL GC_get_hblk_size(void); - -/* Same as GC_walk_hblk_fn but with index of the free list. */ -typedef void(GC_CALLBACK *GC_walk_free_blk_fn)(struct GC_hblk_s *, - int /* index */, - GC_word /* client_data */); - -/* Apply fn to each completely empty heap block. It is the */ -/* responsibility of the caller to avoid data race during the function */ -/* execution (e.g. by holding the allocation lock). */ -GC_API void GC_CALL GC_iterate_free_hblks(GC_walk_free_blk_fn, - GC_word /* client_data */) GC_ATTR_NONNULL(1); - -typedef void(GC_CALLBACK *GC_walk_hblk_fn)(struct GC_hblk_s *, - GC_word /* client_data */); - -/* Apply fn to each allocated heap block. It is the responsibility */ -/* of the caller to avoid data race during the function execution (e.g. */ -/* by holding the allocation lock). */ -GC_API void GC_CALL GC_apply_to_all_blocks(GC_walk_hblk_fn, - GC_word /* client_data */) GC_ATTR_NONNULL(1); - -/* If there are likely to be false references to a block starting at h */ -/* of the indicated length, then return the next plausible starting */ -/* location for h that might avoid these false references. Otherwise */ -/* NULL is returned. Assumes the allocation lock is held but no */ -/* assertion about it by design. */ -GC_API struct GC_hblk_s *GC_CALL GC_is_black_listed(struct GC_hblk_s *, - GC_word /* len */); - -/* Return the number of set mark bits for the heap block where object */ -/* p is located. Defined only if the library has been compiled */ -/* without NO_DEBUGGING. */ -GC_API unsigned GC_CALL GC_count_set_marks_in_hblk(const void * /* p */); - -/* And some routines to support creation of new "kinds", e.g. with */ -/* custom mark procedures, by language runtimes. */ -/* The _inner versions assume the caller holds the allocation lock. */ - -/* Return a new free list array. */ -GC_API void **GC_CALL GC_new_free_list(void); -GC_API void **GC_CALL GC_new_free_list_inner(void); - -/* Return a new kind, as specified. */ -GC_API unsigned GC_CALL GC_new_kind(void ** /* free_list */, - GC_word /* mark_descriptor_template */, - int /* add_size_to_descriptor */, - int /* clear_new_objects */) GC_ATTR_NONNULL(1); -/* The last two parameters must be zero or one. */ -GC_API unsigned GC_CALL GC_new_kind_inner(void ** /* free_list */, - GC_word /* mark_descriptor_template */, - int /* add_size_to_descriptor */, - int /* clear_new_objects */) GC_ATTR_NONNULL(1); - -/* Return a new mark procedure identifier, suitable for use as */ -/* the first argument in GC_MAKE_PROC. */ -GC_API unsigned GC_CALL GC_new_proc(GC_mark_proc); -GC_API unsigned GC_CALL GC_new_proc_inner(GC_mark_proc); - -/* Allocate an object of a given kind. By default, there are only */ -/* a few kinds: composite (pointerful), atomic, uncollectible, etc. */ -/* We claim it is possible for clever client code that understands the */ -/* GC internals to add more, e.g. to communicate object layout */ -/* information to the collector. Note that in the multi-threaded */ -/* contexts, this is usually unsafe for kinds that have the descriptor */ -/* in the object itself, since there is otherwise a window in which */ -/* the descriptor is not correct. Even in the single-threaded case, */ -/* we need to be sure that cleared objects on a free list don't */ -/* cause a GC crash if they are accidentally traced. */ -GC_API GC_ATTR_MALLOC GC_ATTR_ALLOC_SIZE(1) void *GC_CALL GC_generic_malloc( - size_t /* lb */, - int /* knd */); - -GC_API GC_ATTR_MALLOC GC_ATTR_ALLOC_SIZE(1) void *GC_CALL - GC_generic_malloc_ignore_off_page( - size_t /* lb */, int /* knd */); -/* As above, but pointers to past the */ -/* first hblk of the resulting object */ -/* are ignored. */ - -/* Generalized version of GC_malloc_[atomic_]uncollectable. */ -GC_API GC_ATTR_MALLOC GC_ATTR_ALLOC_SIZE(1) void *GC_CALL - GC_generic_malloc_uncollectable( - size_t /* lb */, int /* knd */); - -/* Same as above but primary for allocating an object of the same kind */ -/* as an existing one (kind obtained by GC_get_kind_and_size). */ -/* Not suitable for GCJ and typed-malloc kinds. */ -GC_API GC_ATTR_MALLOC GC_ATTR_ALLOC_SIZE(1) void *GC_CALL - GC_generic_or_special_malloc( - size_t /* size */, int /* knd */); -GC_API GC_ATTR_MALLOC GC_ATTR_ALLOC_SIZE(1) void *GC_CALL - GC_debug_generic_or_special_malloc( - size_t /* size */, int /* knd */, - GC_EXTRA_PARAMS); - -#ifdef GC_DEBUG -#define GC_GENERIC_OR_SPECIAL_MALLOC(sz, knd) \ - GC_debug_generic_or_special_malloc(sz, knd, GC_EXTRAS) -#else -#define GC_GENERIC_OR_SPECIAL_MALLOC(sz, knd) \ - GC_generic_or_special_malloc(sz, knd) -#endif /* !GC_DEBUG */ - -/* Similar to GC_size but returns object kind. Size is returned too */ -/* if psize is not NULL. */ -GC_API int GC_CALL GC_get_kind_and_size(const void *, size_t * /* psize */) - GC_ATTR_NONNULL(1); - -typedef void(GC_CALLBACK *GC_describe_type_fn)(void * /* p */, - char * /* out_buf */); -/* A procedure which */ -/* produces a human-readable */ -/* description of the "type" of object */ -/* p into the buffer out_buf of length */ -/* GC_TYPE_DESCR_LEN. This is used by */ -/* the debug support when printing */ -/* objects. */ -/* These functions should be as robust */ -/* as possible, though we do avoid */ -/* invoking them on objects on the */ -/* global free list. */ -#define GC_TYPE_DESCR_LEN 40 - -GC_API void GC_CALL GC_register_describe_type_fn(int /* kind */, - GC_describe_type_fn); -/* Register a describe_type function */ -/* to be used when printing objects */ -/* of a particular kind. */ - -/* Clear some of the inaccessible part of the stack. Returns its */ -/* argument, so it can be used in a tail call position, hence clearing */ -/* another frame. Argument may be NULL. */ -GC_API void *GC_CALL GC_clear_stack(void *); - -/* Set and get the client notifier on collections. The client function */ -/* is called at the start of every full GC (called with the allocation */ -/* lock held). May be 0. This is a really tricky interface to use */ -/* correctly. Unless you really understand the collector internals, */ -/* the callback should not, directly or indirectly, make any GC_ or */ -/* potentially blocking calls. In particular, it is not safe to */ -/* allocate memory using the garbage collector from within the callback */ -/* function. Both the setter and getter acquire the GC lock. */ -typedef void(GC_CALLBACK *GC_start_callback_proc)(void); -GC_API void GC_CALL GC_set_start_callback(GC_start_callback_proc); -GC_API GC_start_callback_proc GC_CALL GC_get_start_callback(void); - -/* Slow/general mark bit manipulation. The caller should hold the */ -/* allocation lock. GC_is_marked returns 1 (true) or 0. The argument */ -/* should be the real address of an object (i.e. the address of the */ -/* debug header if there is one). */ -GC_API int GC_CALL GC_is_marked(const void *) GC_ATTR_NONNULL(1); -GC_API void GC_CALL GC_clear_mark_bit(const void *) GC_ATTR_NONNULL(1); -GC_API void GC_CALL GC_set_mark_bit(const void *) GC_ATTR_NONNULL(1); - -/* Push everything in the given range onto the mark stack. */ -/* (GC_push_conditional pushes either all or only dirty pages depending */ -/* on the third argument.) GC_push_all_eager also ensures that stack */ -/* is scanned immediately, not just scheduled for scanning. */ -GC_API void GC_CALL GC_push_all(void * /* bottom */, void * /* top */); -GC_API void GC_CALL GC_push_all_eager(void * /* bottom */, void * /* top */); -GC_API void GC_CALL GC_push_conditional(void * /* bottom */, void * /* top */, - int /* bool all */); -GC_API void GC_CALL GC_push_finalizer_structures(void); - -/* Set and get the client push-other-roots procedure. A client */ -/* supplied procedure should also call the original procedure. */ -/* Note that both the setter and getter require some external */ -/* synchronization to avoid data race. */ -typedef void(GC_CALLBACK *GC_push_other_roots_proc)(void); -GC_API void GC_CALL GC_set_push_other_roots(GC_push_other_roots_proc); -GC_API GC_push_other_roots_proc GC_CALL GC_get_push_other_roots(void); - -/* Walk the GC heap visiting all reachable objects. Assume the caller */ -/* holds the allocation lock. Object base pointer, object size and */ -/* client custom data are passed to the callback (holding the lock). */ -typedef void(GC_CALLBACK *GC_reachable_object_proc)(void * /* obj */, - size_t /* bytes */, - void * /* client_data */); -GC_API void GC_CALL GC_enumerate_reachable_objects_inner( - GC_reachable_object_proc, - void * /* client_data */) GC_ATTR_NONNULL(1); - -GC_API int GC_CALL GC_is_tmp_root(void *); - -GC_API void GC_CALL GC_print_trace(GC_word /* gc_no */); -GC_API void GC_CALL GC_print_trace_inner(GC_word /* gc_no */); - -/* Set the client for when mark stack is empty. A client can use */ -/* this callback to process (un)marked objects and push additional */ -/* work onto the stack. Useful for implementing ephemerons. */ -/* Both the setter and getter acquire the GC lock. */ -typedef struct GC_ms_entry *(GC_CALLBACK *GC_on_mark_stack_empty_proc)( - struct GC_ms_entry * /* mark_stack_ptr */, - struct GC_ms_entry * /* mark_stack_limit */); -GC_API void GC_CALL GC_set_on_mark_stack_empty(GC_on_mark_stack_empty_proc); -GC_API GC_on_mark_stack_empty_proc GC_CALL GC_get_on_mark_stack_empty(void); - -#ifdef __cplusplus -} /* extern "C" */ -#endif - -#endif /* GC_MARK_H */ diff --git a/vendor/bdwgc/private/gc/gc_tiny_fl.h b/vendor/bdwgc/private/gc/gc_tiny_fl.h deleted file mode 100644 index 96709e65..00000000 --- a/vendor/bdwgc/private/gc/gc_tiny_fl.h +++ /dev/null @@ -1,86 +0,0 @@ -/* - * Copyright (c) 1999-2005 Hewlett-Packard Development Company, L.P. - * - * THIS MATERIAL IS PROVIDED AS IS, WITH ABSOLUTELY NO WARRANTY EXPRESSED - * OR IMPLIED. ANY USE IS AT YOUR OWN RISK. - * - * Permission is hereby granted to use or copy this program - * for any purpose, provided the above notices are retained on all copies. - * Permission to modify the code and to distribute modified code is granted, - * provided the above notices are retained, and a notice that the code was - * modified is included with the above copyright notice. - */ - -#ifndef GC_TINY_FL_H -#define GC_TINY_FL_H -/* - * Constants and data structures for "tiny" free lists. - * These are used for thread-local allocation or in-lined allocators. - * Each global free list also essentially starts with one of these. - * However, global free lists are known to the GC. "Tiny" free lists - * are basically private to the client. Their contents are viewed as - * "in use" and marked accordingly by the core of the GC. - * - * Note that inlined code might know about the layout of these and the constants - * involved. Thus any change here may invalidate clients, and such changes should - * be avoided. Hence we keep this as simple as possible. - */ - -/* - * We always set GC_GRANULE_BYTES to twice the length of a pointer. - * This means that all allocation requests are rounded up to the next - * multiple of 16 on 64-bit architectures or 8 on 32-bit architectures. - * This appears to be a reasonable compromise between fragmentation overhead - * and space usage for mark bits (usually mark bytes). - * On many 64-bit architectures some memory references require 16-byte - * alignment, making this necessary anyway. - * For a few 32-bit architecture (e.g. x86), we may also need 16-byte alignment - * for certain memory references. But currently that does not seem to be the - * default for all conventional malloc implementations, so we ignore that - * problem. - * It would always be safe, and often useful, to be able to allocate very - * small objects with smaller alignment. But that would cost us mark bit - * space, so we no longer do so. - */ -#ifndef GC_GRANULE_BYTES -/* GC_GRANULE_BYTES should not be overridden in any instances of the GC */ -/* library that may be shared between applications, since it affects */ -/* the binary interface to the library. */ -#if defined(__LP64__) || defined(_LP64) || defined(_WIN64) || defined(__s390x__) || (defined(__x86_64__) && !defined(__ILP32__)) || defined(__alpha__) || defined(__powerpc64__) || defined(__arch64__) -#define GC_GRANULE_BYTES 16 -#define GC_GRANULE_WORDS 2 -#else -#define GC_GRANULE_BYTES 8 -#define GC_GRANULE_WORDS 2 -#endif -#endif /* !GC_GRANULE_BYTES */ - -#if GC_GRANULE_WORDS == 2 -#define GC_WORDS_TO_GRANULES(n) ((n) >> 1) -#else -#define GC_WORDS_TO_GRANULES(n) ((n) * sizeof(void *) / GC_GRANULE_BYTES) -#endif - -/* A "tiny" free list header contains TINY_FREELISTS pointers to */ -/* singly linked lists of objects of different sizes, the ith one */ -/* containing objects i granules in size. Note that there is a list */ -/* of size zero objects. */ -#ifndef GC_TINY_FREELISTS -#if GC_GRANULE_BYTES == 16 -#define GC_TINY_FREELISTS 25 -#else -#define GC_TINY_FREELISTS 33 /* Up to and including 256 bytes */ -#endif -#endif /* !GC_TINY_FREELISTS */ - -/* The ith free list corresponds to size i*GC_GRANULE_BYTES */ -/* Internally to the collector, the index can be computed with */ -/* ALLOC_REQUEST_GRANS(). Externally, we don't know whether */ -/* DONT_ADD_BYTE_AT_END is set, but the client should know. */ - -/* Convert a free list index to the actual size of objects */ -/* on that list, including extra space we added. Not an */ -/* inverse of the above. */ -#define GC_RAW_BYTES_FROM_INDEX(i) ((i)*GC_GRANULE_BYTES) - -#endif /* GC_TINY_FL_H */ diff --git a/vendor/bdwgc/private/gc/gc_version.h b/vendor/bdwgc/private/gc/gc_version.h deleted file mode 100644 index d2be3c95..00000000 --- a/vendor/bdwgc/private/gc/gc_version.h +++ /dev/null @@ -1,45 +0,0 @@ -/* - * Copyright (c) 1994 by Xerox Corporation. All rights reserved. - * Copyright (c) 1996 by Silicon Graphics. All rights reserved. - * Copyright (c) 1998 by Fergus Henderson. All rights reserved. - * Copyright (c) 2000-2009 by Hewlett-Packard Development Company. - * All rights reserved. - * - * THIS MATERIAL IS PROVIDED AS IS, WITH ABSOLUTELY NO WARRANTY EXPRESSED - * OR IMPLIED. ANY USE IS AT YOUR OWN RISK. - * - * Permission is hereby granted to use or copy this program - * for any purpose, provided the above notices are retained on all copies. - * Permission to modify the code and to distribute modified code is granted, - * provided the above notices are retained, and a notice that the code was - * modified is included with the above copyright notice. - */ - -/* This should never be included directly; it is included only from gc.h. */ -#if defined(GC_H) - -/* The policy regarding version numbers: development code has odd */ -/* "minor" number (and "micro" part is 0); when development is finished */ -/* and a release is prepared, "minor" number is incremented (keeping */ -/* "micro" number still zero), whenever a defect is fixed a new release */ -/* is prepared incrementing "micro" part to odd value (the most stable */ -/* release has the biggest "micro" number). */ - -/* The version here should match that in configure/configure.ac */ -/* Eventually this one may become unnecessary. For now we need */ -/* it to keep the old-style build process working. */ -#define GC_TMP_VERSION_MAJOR 8 -#define GC_TMP_VERSION_MINOR 3 -#define GC_TMP_VERSION_MICRO 0 /* 8.3.0 */ - -#ifdef GC_VERSION_MAJOR -#if GC_TMP_VERSION_MAJOR != GC_VERSION_MAJOR || GC_TMP_VERSION_MINOR != GC_VERSION_MINOR || GC_TMP_VERSION_MICRO != GC_VERSION_MICRO -#error Inconsistent version info. Check README.md, include/gc_version.h and configure.ac. -#endif -#else -#define GC_VERSION_MAJOR GC_TMP_VERSION_MAJOR -#define GC_VERSION_MINOR GC_TMP_VERSION_MINOR -#define GC_VERSION_MICRO GC_TMP_VERSION_MICRO -#endif /* !GC_VERSION_MAJOR */ - -#endif diff --git a/vendor/bdwgc/private/gc_alloc_ptrs.h b/vendor/bdwgc/private/gc_alloc_ptrs.h deleted file mode 100644 index 08b9fae5..00000000 --- a/vendor/bdwgc/private/gc_alloc_ptrs.h +++ /dev/null @@ -1,60 +0,0 @@ -/* - * Copyright (c) 1996-1998 by Silicon Graphics. All rights reserved. - * Copyright (c) 2018-2021 Ivan Maidanski - * - * THIS MATERIAL IS PROVIDED AS IS, WITH ABSOLUTELY NO WARRANTY EXPRESSED - * OR IMPLIED. ANY USE IS AT YOUR OWN RISK. - * - * Permission is hereby granted to use or copy this program - * for any purpose, provided the above notices are retained on all copies. - * Permission to modify the code and to distribute modified code is granted, - * provided the above notices are retained, and a notice that the code was - * modified is included with the above copyright notice. - */ - -/* This file is kept for a binary compatibility purpose only. */ - -#ifndef GC_ALLOC_PTRS_H -#define GC_ALLOC_PTRS_H - -#include "gc/gc.h" - -#ifdef __cplusplus -extern "C" { -#endif - -#ifndef GC_API_PRIV -#define GC_API_PRIV GC_API -#endif - -/* Some compilers do not accept "const" together with the dllimport */ -/* attribute, so the symbols below are exported as non-constant ones. */ -#ifndef GC_APIVAR_CONST -#if defined(GC_BUILD) || !defined(GC_DLL) -#define GC_APIVAR_CONST const -#else -#define GC_APIVAR_CONST /* empty */ -#endif -#endif - -GC_API_PRIV void** GC_APIVAR_CONST GC_objfreelist_ptr; -GC_API_PRIV void** GC_APIVAR_CONST GC_aobjfreelist_ptr; -GC_API_PRIV void** GC_APIVAR_CONST GC_uobjfreelist_ptr; - -#ifdef GC_ATOMIC_UNCOLLECTABLE -GC_API_PRIV void** GC_APIVAR_CONST GC_auobjfreelist_ptr; -#endif - -/* Manually update the number of bytes allocated during the current */ -/* collection cycle and the number of explicitly deallocated bytes of */ -/* memory since the last collection, respectively. Both functions are */ -/* unsynchronized, GC_call_with_alloc_lock() should be used to avoid */ -/* data races. */ -GC_API_PRIV void GC_CALL GC_incr_bytes_allocd(size_t /* bytes */); -GC_API_PRIV void GC_CALL GC_incr_bytes_freed(size_t /* bytes */); - -#ifdef __cplusplus -} /* extern "C" */ -#endif - -#endif /* GC_ALLOC_PTRS_H */ diff --git a/vendor/bdwgc/private/gc_atomic_ops.h b/vendor/bdwgc/private/gc_atomic_ops.h deleted file mode 100644 index 8c3eff44..00000000 --- a/vendor/bdwgc/private/gc_atomic_ops.h +++ /dev/null @@ -1,122 +0,0 @@ -/* - * Copyright (c) 2017 Ivan Maidanski - * - * THIS MATERIAL IS PROVIDED AS IS, WITH ABSOLUTELY NO WARRANTY EXPRESSED - * OR IMPLIED. ANY USE IS AT YOUR OWN RISK. - * - * Permission is hereby granted to use or copy this program - * for any purpose, provided the above notices are retained on all copies. - * Permission to modify the code and to distribute modified code is granted, - * provided the above notices are retained, and a notice that the code was - * modified is included with the above copyright notice. - */ - -/* This is a private GC header which provides an implementation of */ -/* libatomic_ops subset primitives sufficient for GC assuming that GCC */ -/* atomic intrinsics are available (and have correct implementation). */ -/* This is enabled by defining GC_BUILTIN_ATOMIC macro. Otherwise, */ -/* libatomic_ops library is used to define the primitives. */ - -#ifndef GC_ATOMIC_OPS_H -#define GC_ATOMIC_OPS_H - -#ifdef GC_BUILTIN_ATOMIC - -#include "gc/gc.h" /* for GC_word */ - -#ifdef __cplusplus -extern "C" { -#endif - -typedef GC_word AO_t; - -#ifdef GC_PRIVATE_H /* have GC_INLINE */ -#define AO_INLINE GC_INLINE -#else -#define AO_INLINE static __inline -#endif - -typedef unsigned char AO_TS_t; -#define AO_TS_CLEAR 0 -#define AO_TS_INITIALIZER (AO_TS_t) AO_TS_CLEAR -#if defined(__GCC_ATOMIC_TEST_AND_SET_TRUEVAL) && !defined(CPPCHECK) -#define AO_TS_SET __GCC_ATOMIC_TEST_AND_SET_TRUEVAL -#else -#define AO_TS_SET (AO_TS_t)1 /* true */ -#endif -#define AO_CLEAR(p) __atomic_clear(p, __ATOMIC_RELEASE) -#define AO_test_and_set_acquire(p) \ - (__atomic_test_and_set(p, __ATOMIC_ACQUIRE) ? AO_TS_SET : AO_TS_CLEAR) -#define AO_HAVE_test_and_set_acquire - -#define AO_compiler_barrier() __atomic_signal_fence(__ATOMIC_SEQ_CST) -#define AO_nop_full() __atomic_thread_fence(__ATOMIC_SEQ_CST) -#define AO_HAVE_nop_full - -#define AO_fetch_and_add(p, v) __atomic_fetch_add(p, v, __ATOMIC_RELAXED) -#define AO_HAVE_fetch_and_add -#define AO_fetch_and_add1(p) AO_fetch_and_add(p, 1) -#define AO_HAVE_fetch_and_add1 - -#define AO_or(p, v) (void)__atomic_or_fetch(p, v, __ATOMIC_RELAXED) -#define AO_HAVE_or - -#define AO_load(p) __atomic_load_n(p, __ATOMIC_RELAXED) -#define AO_HAVE_load -#define AO_load_acquire(p) __atomic_load_n(p, __ATOMIC_ACQUIRE) -#define AO_HAVE_load_acquire -#define AO_load_acquire_read(p) AO_load_acquire(p) -#define AO_HAVE_load_acquire_read - -#define AO_store(p, v) __atomic_store_n(p, v, __ATOMIC_RELAXED) -#define AO_HAVE_store -#define AO_store_release(p, v) __atomic_store_n(p, v, __ATOMIC_RELEASE) -#define AO_HAVE_store_release -#define AO_store_release_write(p, v) AO_store_release(p, v) -#define AO_HAVE_store_release_write - -#define AO_char_load(p) __atomic_load_n(p, __ATOMIC_RELAXED) -#define AO_HAVE_char_load -#define AO_char_store(p, v) __atomic_store_n(p, v, __ATOMIC_RELAXED) -#define AO_HAVE_char_store - -#ifdef AO_REQUIRE_CAS -AO_INLINE int -AO_compare_and_swap(volatile AO_t *p, AO_t ov, AO_t nv) { - return (int)__atomic_compare_exchange_n(p, &ov, nv, 0, - __ATOMIC_RELAXED, __ATOMIC_RELAXED); -} - -AO_INLINE int -AO_compare_and_swap_release(volatile AO_t *p, AO_t ov, AO_t nv) { - return (int)__atomic_compare_exchange_n(p, &ov, nv, 0, - __ATOMIC_RELEASE, __ATOMIC_RELAXED); -} -#define AO_HAVE_compare_and_swap_release -#endif - -#ifdef __cplusplus -} /* extern "C" */ -#endif - -#ifndef NO_LOCKFREE_AO_OR -/* __atomic_or_fetch is assumed to be lock-free. */ -#define HAVE_LOCKFREE_AO_OR 1 -#endif - -#else -/* Fallback to libatomic_ops. */ -#include "atomic_ops.h" - -/* AO_compiler_barrier, AO_load and AO_store should be defined for */ -/* all targets; the rest of the primitives are guaranteed to exist */ -/* only if AO_REQUIRE_CAS is defined (or if the corresponding */ -/* AO_HAVE_x macro is defined). x86/x64 targets have AO_nop_full, */ -/* AO_load_acquire, AO_store_release, at least. */ -#if (!defined(AO_HAVE_load) || !defined(AO_HAVE_store)) && !defined(CPPCHECK) -#error AO_load or AO_store is missing; probably old version of atomic_ops -#endif - -#endif /* !GC_BUILTIN_ATOMIC */ - -#endif /* GC_ATOMIC_OPS_H */ diff --git a/vendor/bdwgc/private/gc_hdrs.h b/vendor/bdwgc/private/gc_hdrs.h deleted file mode 100644 index 30a6bf01..00000000 --- a/vendor/bdwgc/private/gc_hdrs.h +++ /dev/null @@ -1,215 +0,0 @@ -/* - * Copyright 1988, 1989 Hans-J. Boehm, Alan J. Demers - * Copyright (c) 1991-1994 by Xerox Corporation. All rights reserved. - * - * THIS MATERIAL IS PROVIDED AS IS, WITH ABSOLUTELY NO WARRANTY EXPRESSED - * OR IMPLIED. ANY USE IS AT YOUR OWN RISK. - * - * Permission is hereby granted to use or copy this program - * for any purpose, provided the above notices are retained on all copies. - * Permission to modify the code and to distribute modified code is granted, - * provided the above notices are retained, and a notice that the code was - * modified is included with the above copyright notice. - */ - -#ifndef GC_HEADERS_H -#define GC_HEADERS_H - -#if CPP_WORDSZ != 32 && CPP_WORDSZ < 36 && !defined(CPPCHECK) -#error Get a real machine -#endif - -EXTERN_C_BEGIN - -typedef struct hblkhdr hdr; - -/* - * The 2 level tree data structure that is used to find block headers. - * If there are more than 32 bits in a pointer, the top level is a hash - * table. - * - * This defines HDR, GET_HDR, and SET_HDR, the main macros used to - * retrieve and set object headers. - * - * We take advantage of a header lookup - * cache. This is a locally declared direct mapped cache, used inside - * the marker. The HC_GET_HDR macro uses and maintains this - * cache. Assuming we get reasonable hit rates, this shaves a few - * memory references from each pointer validation. - */ - -#if CPP_WORDSZ > 32 -#define HASH_TL -#endif - -/* Define appropriate out-degrees for each of the two tree levels */ -#if defined(LARGE_CONFIG) || !defined(SMALL_CONFIG) -#define LOG_BOTTOM_SZ 10 -#else -#define LOG_BOTTOM_SZ 11 -/* Keep top index size reasonable with smaller blocks. */ -#endif -#define BOTTOM_SZ (1 << LOG_BOTTOM_SZ) - -#ifndef HASH_TL -#define LOG_TOP_SZ (WORDSZ - LOG_BOTTOM_SZ - LOG_HBLKSIZE) -#else -#define LOG_TOP_SZ 11 -#endif -#define TOP_SZ (1 << LOG_TOP_SZ) - -/* #define COUNT_HDR_CACHE_HITS */ - -#ifdef COUNT_HDR_CACHE_HITS -extern word GC_hdr_cache_hits; /* used for debugging/profiling */ -extern word GC_hdr_cache_misses; -#define HC_HIT() (void)(++GC_hdr_cache_hits) -#define HC_MISS() (void)(++GC_hdr_cache_misses) -#else -#define HC_HIT() /* empty */ -#define HC_MISS() /* empty */ -#endif - -typedef struct hce { - word block_addr; /* right shifted by LOG_HBLKSIZE */ - hdr *hce_hdr; -} hdr_cache_entry; - -#define HDR_CACHE_SIZE 8 /* power of 2 */ - -#define DECLARE_HDR_CACHE \ - hdr_cache_entry hdr_cache[HDR_CACHE_SIZE] - -#define INIT_HDR_CACHE BZERO(hdr_cache, sizeof(hdr_cache)) - -#define HCE(h) \ - (hdr_cache + (((word)(h) >> LOG_HBLKSIZE) & (HDR_CACHE_SIZE - 1))) - -#define HCE_VALID_FOR(hce, h) ((hce)->block_addr == \ - ((word)(h) >> LOG_HBLKSIZE)) - -#define HCE_HDR(h) ((hce)->hce_hdr) - -#ifdef PRINT_BLACK_LIST -GC_INNER hdr *GC_header_cache_miss(ptr_t p, hdr_cache_entry *hce, - ptr_t source); -#define HEADER_CACHE_MISS(p, hce, source) \ - GC_header_cache_miss(p, hce, source) -#else -GC_INNER hdr *GC_header_cache_miss(ptr_t p, hdr_cache_entry *hce); -#define HEADER_CACHE_MISS(p, hce, source) GC_header_cache_miss(p, hce) -#endif - -/* Set hhdr to the header for p. Analogous to GET_HDR below, */ -/* except that in the case of large objects, it gets the header for */ -/* the object beginning if GC_all_interior_pointers is set. */ -/* Returns zero if p points to somewhere other than the first page */ -/* of an object, and it is not a valid pointer to the object. */ -#define HC_GET_HDR(p, hhdr, source) \ - { /* cannot use do-while(0) here */ \ - hdr_cache_entry *hce = HCE(p); \ - if (EXPECT(HCE_VALID_FOR(hce, p), TRUE)) { \ - HC_HIT(); \ - hhdr = hce->hce_hdr; \ - } else { \ - hhdr = HEADER_CACHE_MISS(p, hce, source); \ - if (NULL == hhdr) break; /* go to the enclosing loop end */ \ - } \ - } - -typedef struct bi { - hdr *index[BOTTOM_SZ]; - /* - * The bottom level index contains one of three kinds of values: - * 0 means we're not responsible for this block, - * or this is a block other than the first one in a free block. - * 1 < (long)X <= MAX_JUMP means the block starts at least - * X * HBLKSIZE bytes before the current address. - * A valid pointer points to a hdr structure. (The above can't be - * valid pointers due to the GET_MEM return convention.) - */ - struct bi *asc_link; /* All indices are linked in */ - /* ascending order... */ - struct bi *desc_link; /* ... and in descending order. */ - word key; /* high order address bits. */ -#ifdef HASH_TL - struct bi *hash_link; /* Hash chain link. */ -#endif -} bottom_index; - -/* bottom_index GC_all_nils; - really part of GC_arrays */ - -/* extern bottom_index * GC_top_index []; - really part of GC_arrays */ -/* Each entry points to a bottom_index. */ -/* On a 32 bit machine, it points to */ -/* the index for a set of high order */ -/* bits equal to the index. For longer */ -/* addresses, we hash the high order */ -/* bits to compute the index in */ -/* GC_top_index, and each entry points */ -/* to a hash chain. */ -/* The last entry in each chain is */ -/* GC_all_nils. */ - -#define MAX_JUMP (HBLKSIZE - 1) - -#define HDR_FROM_BI(bi, p) \ - (bi)->index[((word)(p) >> LOG_HBLKSIZE) & (BOTTOM_SZ - 1)] -#ifndef HASH_TL -#define BI(p) (GC_top_index \ - [(word)(p) >> (LOG_BOTTOM_SZ + LOG_HBLKSIZE)]) -#define HDR_INNER(p) HDR_FROM_BI(BI(p), p) -#ifdef SMALL_CONFIG -#define HDR(p) GC_find_header((ptr_t)(p)) -#else -#define HDR(p) HDR_INNER(p) -#endif -#define GET_BI(p, bottom_indx) (void)((bottom_indx) = BI(p)) -#define GET_HDR(p, hhdr) (void)((hhdr) = HDR(p)) -#define SET_HDR(p, hhdr) (void)(HDR_INNER(p) = (hhdr)) -#define GET_HDR_ADDR(p, ha) (void)((ha) = &HDR_INNER(p)) -#else /* hash */ -/* Hash function for tree top level */ -#define TL_HASH(hi) ((hi) & (TOP_SZ - 1)) -/* Set bottom_indx to point to the bottom index for address p */ -#define GET_BI(p, bottom_indx) \ - do { \ - REGISTER word hi = (word)(p) >> (LOG_BOTTOM_SZ + LOG_HBLKSIZE); \ - REGISTER bottom_index *_bi = GC_top_index[TL_HASH(hi)]; \ - while (_bi->key != hi && _bi != GC_all_nils) \ - _bi = _bi->hash_link; \ - (bottom_indx) = _bi; \ - } while (0) -#define GET_HDR_ADDR(p, ha) \ - do { \ - REGISTER bottom_index *bi; \ - GET_BI(p, bi); \ - (ha) = &HDR_FROM_BI(bi, p); \ - } while (0) -#define GET_HDR(p, hhdr) \ - do { \ - REGISTER hdr **_ha; \ - GET_HDR_ADDR(p, _ha); \ - (hhdr) = *_ha; \ - } while (0) -#define SET_HDR(p, hhdr) \ - do { \ - REGISTER bottom_index *bi; \ - GET_BI(p, bi); \ - GC_ASSERT(bi != GC_all_nils); \ - HDR_FROM_BI(bi, p) = (hhdr); \ - } while (0) -#define HDR(p) GC_find_header((ptr_t)(p)) -#endif - -/* Is the result a forwarding address to someplace closer to the */ -/* beginning of the block or NULL? */ -#define IS_FORWARDING_ADDR_OR_NIL(hhdr) ((size_t)(hhdr) <= MAX_JUMP) - -/* Get an HBLKSIZE aligned address closer to the beginning of the block */ -/* h. Assumes hhdr == HDR(h) and IS_FORWARDING_ADDR(hhdr). */ -#define FORWARDED_ADDR(h, hhdr) ((struct hblk *)(h) - (size_t)(hhdr)) - -EXTERN_C_END - -#endif /* GC_HEADERS_H */ diff --git a/vendor/bdwgc/private/gc_locks.h b/vendor/bdwgc/private/gc_locks.h deleted file mode 100644 index 1fa52ad0..00000000 --- a/vendor/bdwgc/private/gc_locks.h +++ /dev/null @@ -1,308 +0,0 @@ -/* - * Copyright 1988, 1989 Hans-J. Boehm, Alan J. Demers - * Copyright (c) 1991-1994 by Xerox Corporation. All rights reserved. - * Copyright (c) 1996-1999 by Silicon Graphics. All rights reserved. - * Copyright (c) 1999 by Hewlett-Packard Company. All rights reserved. - * Copyright (c) 2008-2022 Ivan Maidanski - * - * THIS MATERIAL IS PROVIDED AS IS, WITH ABSOLUTELY NO WARRANTY EXPRESSED - * OR IMPLIED. ANY USE IS AT YOUR OWN RISK. - * - * Permission is hereby granted to use or copy this program - * for any purpose, provided the above notices are retained on all copies. - * Permission to modify the code and to distribute modified code is granted, - * provided the above notices are retained, and a notice that the code was - * modified is included with the above copyright notice. - */ - -#ifndef GC_LOCKS_H -#define GC_LOCKS_H - -/* - * Mutual exclusion between allocator/collector routines. - * Needed if there is more than one allocator thread. - * - * Note that I_HOLD_LOCK and I_DONT_HOLD_LOCK are used only positively - * in assertions, and may return TRUE in the "don't know" case. - */ -#ifdef THREADS - -#ifdef PCR -#include -#include -#endif - -EXTERN_C_BEGIN - -#ifdef PCR -GC_EXTERN PCR_Th_ML GC_allocate_ml; -#define UNCOND_LOCK() PCR_Th_ML_Acquire(&GC_allocate_ml) -#define UNCOND_UNLOCK() PCR_Th_ML_Release(&GC_allocate_ml) -#elif defined(NN_PLATFORM_CTR) || defined(NINTENDO_SWITCH) -extern void GC_lock(void); -extern void GC_unlock(void); -#define UNCOND_LOCK() GC_lock() -#define UNCOND_UNLOCK() GC_unlock() -#endif - -#if (!defined(AO_HAVE_test_and_set_acquire) || defined(GC_RTEMS_PTHREADS) || defined(SN_TARGET_PS3) || defined(GC_WIN32_THREADS) || defined(BASE_ATOMIC_OPS_EMULATED) || defined(LINT2)) && defined(GC_PTHREADS) -#define USE_PTHREAD_LOCKS -#undef USE_SPIN_LOCK -#if (defined(LINT2) || defined(GC_WIN32_THREADS)) && !defined(NO_PTHREAD_TRYLOCK) -/* pthread_mutex_trylock may not win in GC_lock on Win32, */ -/* due to builtin support for spinning first? */ -#define NO_PTHREAD_TRYLOCK -#endif -#endif - -#if defined(GC_WIN32_THREADS) && !defined(USE_PTHREAD_LOCKS) || defined(GC_PTHREADS) -#define NO_THREAD ((unsigned long)(-1L)) -/* != NUMERIC_THREAD_ID(pthread_self()) for any thread */ -#ifdef GC_ASSERTIONS -GC_EXTERN unsigned long GC_lock_holder; -#define UNSET_LOCK_HOLDER() (void)(GC_lock_holder = NO_THREAD) -#endif -#endif /* GC_WIN32_THREADS || GC_PTHREADS */ - -#if defined(GC_WIN32_THREADS) && !defined(USE_PTHREAD_LOCKS) -GC_EXTERN CRITICAL_SECTION GC_allocate_ml; -#ifdef GC_ASSERTIONS -#define SET_LOCK_HOLDER() (void)(GC_lock_holder = GetCurrentThreadId()) -#define I_HOLD_LOCK() (!GC_need_to_lock || GC_lock_holder == GetCurrentThreadId()) -#ifdef THREAD_SANITIZER -#define I_DONT_HOLD_LOCK() TRUE /* Conservatively say yes */ -#else -#define I_DONT_HOLD_LOCK() (!GC_need_to_lock || GC_lock_holder != GetCurrentThreadId()) -#endif -#define UNCOND_LOCK() \ - { \ - GC_ASSERT(I_DONT_HOLD_LOCK()); \ - EnterCriticalSection(&GC_allocate_ml); \ - SET_LOCK_HOLDER(); \ - } -#define UNCOND_UNLOCK() \ - { \ - GC_ASSERT(I_HOLD_LOCK()); \ - UNSET_LOCK_HOLDER(); \ - LeaveCriticalSection(&GC_allocate_ml); \ - } -#else -#define UNCOND_LOCK() EnterCriticalSection(&GC_allocate_ml) -#define UNCOND_UNLOCK() LeaveCriticalSection(&GC_allocate_ml) -#endif /* !GC_ASSERTIONS */ -#elif defined(GC_PTHREADS) -EXTERN_C_END -#include -EXTERN_C_BEGIN -/* Posix allows pthread_t to be a struct, though it rarely is. */ -/* Unfortunately, we need to use a pthread_t to index a data */ -/* structure. It also helps if comparisons don't involve a */ -/* function call. Hence we introduce platform-dependent macros */ -/* to compare pthread_t ids and to map them to integers. */ -/* The mapping to integers does not need to result in different */ -/* integers for each thread, though that should be true as much */ -/* as possible. */ -/* Refine to exclude platforms on which pthread_t is struct. */ -#if !defined(GC_WIN32_PTHREADS) -#define NUMERIC_THREAD_ID(id) ((unsigned long)(id)) -#define THREAD_EQUAL(id1, id2) ((id1) == (id2)) -#define NUMERIC_THREAD_ID_UNIQUE -#elif defined(__WINPTHREADS_VERSION_MAJOR) /* winpthreads */ -#define NUMERIC_THREAD_ID(id) ((unsigned long)(id)) -#define THREAD_EQUAL(id1, id2) ((id1) == (id2)) -#ifndef _WIN64 -/* NUMERIC_THREAD_ID is 32-bit and not unique on Win64. */ -#define NUMERIC_THREAD_ID_UNIQUE -#endif -#else /* pthreads-win32 */ -#define NUMERIC_THREAD_ID(id) ((unsigned long)(word)(id.p)) -/* Using documented internal details of pthreads-win32 library. */ -/* Faster than pthread_equal(). Should not change with */ -/* future versions of pthreads-win32 library. */ -#define THREAD_EQUAL(id1, id2) ((id1.p == id2.p) && (id1.x == id2.x)) -#undef NUMERIC_THREAD_ID_UNIQUE -/* Generic definitions based on pthread_equal() always work but */ -/* will result in poor performance (as NUMERIC_THREAD_ID is */ -/* defined to just a constant) and weak assertion checking. */ -#endif - -#ifdef SN_TARGET_PSP2 -EXTERN_C_END -#include "psp2-support.h" -EXTERN_C_BEGIN -GC_EXTERN WapiMutex GC_allocate_ml_PSP2; -#define UNCOND_LOCK() \ - { \ - int res; \ - GC_ASSERT(I_DONT_HOLD_LOCK()); \ - res = PSP2_MutexLock(&GC_allocate_ml_PSP2); \ - GC_ASSERT(0 == res); \ - (void)res; \ - SET_LOCK_HOLDER(); \ - } -#define UNCOND_UNLOCK() \ - { \ - int res; \ - GC_ASSERT(I_HOLD_LOCK()); \ - UNSET_LOCK_HOLDER(); \ - res = PSP2_MutexUnlock(&GC_allocate_ml_PSP2); \ - GC_ASSERT(0 == res); \ - (void)res; \ - } - -#elif (!defined(THREAD_LOCAL_ALLOC) || defined(USE_SPIN_LOCK)) && !defined(USE_PTHREAD_LOCKS) && !defined(THREAD_SANITIZER) -/* In the THREAD_LOCAL_ALLOC case, the allocation lock tends to */ -/* be held for long periods, if it is held at all. Thus spinning */ -/* and sleeping for fixed periods are likely to result in */ -/* significant wasted time. We thus rely mostly on queued locks. */ -#undef USE_SPIN_LOCK -#define USE_SPIN_LOCK -GC_EXTERN volatile AO_TS_t GC_allocate_lock; -GC_INNER void GC_lock(void); -/* Allocation lock holder. Only set if acquired by client through */ -/* GC_call_with_alloc_lock. */ -#ifdef GC_ASSERTIONS -#define UNCOND_LOCK() \ - { \ - GC_ASSERT(I_DONT_HOLD_LOCK()); \ - if (AO_test_and_set_acquire(&GC_allocate_lock) == AO_TS_SET) \ - GC_lock(); \ - SET_LOCK_HOLDER(); \ - } -#define UNCOND_UNLOCK() \ - { \ - GC_ASSERT(I_HOLD_LOCK()); \ - UNSET_LOCK_HOLDER(); \ - AO_CLEAR(&GC_allocate_lock); \ - } -#else -#define UNCOND_LOCK() \ - { \ - if (AO_test_and_set_acquire(&GC_allocate_lock) == AO_TS_SET) \ - GC_lock(); \ - } -#define UNCOND_UNLOCK() AO_CLEAR(&GC_allocate_lock) -#endif /* !GC_ASSERTIONS */ -#else /* THREAD_LOCAL_ALLOC || USE_PTHREAD_LOCKS */ -#ifndef USE_PTHREAD_LOCKS -#define USE_PTHREAD_LOCKS -#endif -#endif /* THREAD_LOCAL_ALLOC || USE_PTHREAD_LOCKS */ -#ifdef USE_PTHREAD_LOCKS -EXTERN_C_END -#include -EXTERN_C_BEGIN -GC_EXTERN pthread_mutex_t GC_allocate_ml; -#ifdef GC_ASSERTIONS -GC_INNER void GC_lock(void); -#define UNCOND_LOCK() \ - { \ - GC_ASSERT(I_DONT_HOLD_LOCK()); \ - GC_lock(); \ - SET_LOCK_HOLDER(); \ - } -#define UNCOND_UNLOCK() \ - { \ - GC_ASSERT(I_HOLD_LOCK()); \ - UNSET_LOCK_HOLDER(); \ - pthread_mutex_unlock(&GC_allocate_ml); \ - } -#else /* !GC_ASSERTIONS */ -#if defined(NO_PTHREAD_TRYLOCK) -#define UNCOND_LOCK() pthread_mutex_lock(&GC_allocate_ml) -#else -GC_INNER void GC_lock(void); -#define UNCOND_LOCK() \ - { \ - if (0 != pthread_mutex_trylock(&GC_allocate_ml)) \ - GC_lock(); \ - } -#endif -#define UNCOND_UNLOCK() pthread_mutex_unlock(&GC_allocate_ml) -#endif /* !GC_ASSERTIONS */ -#endif /* USE_PTHREAD_LOCKS */ -#ifdef GC_ASSERTIONS -#define SET_LOCK_HOLDER() \ - (void)(GC_lock_holder = NUMERIC_THREAD_ID(pthread_self())) -#define I_HOLD_LOCK() \ - (!GC_need_to_lock || GC_lock_holder == NUMERIC_THREAD_ID(pthread_self())) -#if !defined(NUMERIC_THREAD_ID_UNIQUE) || defined(THREAD_SANITIZER) -#define I_DONT_HOLD_LOCK() TRUE /* Conservatively say yes */ -#else -#define I_DONT_HOLD_LOCK() \ - (!GC_need_to_lock || GC_lock_holder != NUMERIC_THREAD_ID(pthread_self())) -#endif -#endif /* GC_ASSERTIONS */ -#ifndef GC_WIN32_THREADS -GC_EXTERN volatile unsigned char GC_collecting; -#ifdef AO_HAVE_char_store -#define ENTER_GC() AO_char_store(&GC_collecting, TRUE) -#define EXIT_GC() AO_char_store(&GC_collecting, FALSE) -#else -#define ENTER_GC() (void)(GC_collecting = TRUE) -#define EXIT_GC() (void)(GC_collecting = FALSE) -#endif -#endif /* !GC_WIN32_THREADS */ -#endif /* GC_PTHREADS */ -#if defined(GC_ALWAYS_MULTITHREADED) && (defined(USE_PTHREAD_LOCKS) || defined(USE_SPIN_LOCK)) -#define GC_need_to_lock TRUE -#define set_need_to_lock() (void)0 -#else -#if defined(GC_ALWAYS_MULTITHREADED) && !defined(CPPCHECK) -#error Runtime initialization of GC lock is needed! -#endif -#undef GC_ALWAYS_MULTITHREADED -GC_EXTERN GC_bool GC_need_to_lock; -#ifdef THREAD_SANITIZER -/* To workaround TSan false positive (e.g., when */ -/* GC_pthread_create is called from multiple threads in */ -/* parallel), do not set GC_need_to_lock if it is already set. */ -#define set_need_to_lock() \ - (void)(*(GC_bool volatile *)&GC_need_to_lock \ - ? FALSE \ - : (GC_need_to_lock = TRUE)) -#else -#define set_need_to_lock() (void)(GC_need_to_lock = TRUE) -/* We are multi-threaded now. */ -#endif -#endif - -EXTERN_C_END - -#else /* !THREADS */ -#define LOCK() (void)0 -#define UNLOCK() (void)0 -#ifdef GC_ASSERTIONS -#define I_HOLD_LOCK() TRUE -#define I_DONT_HOLD_LOCK() TRUE -/* Used only in positive assertions or to test whether */ -/* we still need to acquire the lock. TRUE works in */ -/* either case. */ -#endif -#endif /* !THREADS */ - -#if defined(UNCOND_LOCK) && !defined(LOCK) -#if (defined(LINT2) && defined(USE_PTHREAD_LOCKS)) || defined(GC_ALWAYS_MULTITHREADED) -/* Instruct code analysis tools not to care about GC_need_to_lock */ -/* influence to LOCK/UNLOCK semantic. */ -#define LOCK() UNCOND_LOCK() -#define UNLOCK() UNCOND_UNLOCK() -#else -/* At least two thread running; need to lock. */ -#define LOCK() \ - do { \ - if (GC_need_to_lock) UNCOND_LOCK(); \ - } while (0) -#define UNLOCK() \ - do { \ - if (GC_need_to_lock) UNCOND_UNLOCK(); \ - } while (0) -#endif -#endif - -#ifndef ENTER_GC -#define ENTER_GC() -#define EXIT_GC() -#endif - -#endif /* GC_LOCKS_H */ diff --git a/vendor/bdwgc/private/gc_pmark.h b/vendor/bdwgc/private/gc_pmark.h deleted file mode 100644 index 448637d7..00000000 --- a/vendor/bdwgc/private/gc_pmark.h +++ /dev/null @@ -1,456 +0,0 @@ -/* - * Copyright (c) 1991-1994 by Xerox Corporation. All rights reserved. - * Copyright (c) 2001 by Hewlett-Packard Company. All rights reserved. - * Copyright (c) 2008-2022 Ivan Maidanski - * - * THIS MATERIAL IS PROVIDED AS IS, WITH ABSOLUTELY NO WARRANTY EXPRESSED - * OR IMPLIED. ANY USE IS AT YOUR OWN RISK. - * - * Permission is hereby granted to use or copy this program - * for any purpose, provided the above notices are retained on all copies. - * Permission to modify the code and to distribute modified code is granted, - * provided the above notices are retained, and a notice that the code was - * modified is included with the above copyright notice. - * - */ - -/* Private declarations of GC marker data structures and macros */ - -/* - * Declarations of mark stack. Needed by marker and client supplied mark - * routines. Transitively include gc_priv.h. - */ -#ifndef GC_PMARK_H -#define GC_PMARK_H - -#if defined(HAVE_CONFIG_H) && !defined(GC_PRIVATE_H) -/* When gc_pmark.h is included from gc_priv.h, some of macros might */ -/* be undefined in gcconfig.h, so skip config.h in this case. */ -#include "config.h" -#endif - -#ifndef GC_BUILD -#define GC_BUILD -#endif - -#if (defined(__linux__) || defined(__GLIBC__) || defined(__GNU__)) && !defined(_GNU_SOURCE) && defined(GC_PTHREADS) && !defined(GC_NO_PTHREAD_SIGMASK) -#define _GNU_SOURCE 1 -#endif - -#if defined(KEEP_BACK_PTRS) || defined(PRINT_BLACK_LIST) -#include "dbg_mlc.h" -#endif - -#include "gc/gc_mark.h" -#include "gc_priv.h" - -EXTERN_C_BEGIN - -/* The real declarations of the following is in gc_priv.h, so that */ -/* we can avoid scanning the following table. */ -/* -mark_proc GC_mark_procs[MAX_MARK_PROCS]; -*/ - -#ifndef MARK_DESCR_OFFSET -#define MARK_DESCR_OFFSET sizeof(word) -#endif - -/* - * Mark descriptor stuff that should remain private for now, mostly - * because it's hard to export WORDSZ without including gcconfig.h. - */ -#define BITMAP_BITS (WORDSZ - GC_DS_TAG_BITS) -#define PROC(descr) \ - (GC_mark_procs[((descr) >> GC_DS_TAG_BITS) & (GC_MAX_MARK_PROCS - 1)]) -#define ENV(descr) \ - ((descr) >> (GC_DS_TAG_BITS + GC_LOG_MAX_MARK_PROCS)) -#define MAX_ENV \ - (((word)1 << (WORDSZ - GC_DS_TAG_BITS - GC_LOG_MAX_MARK_PROCS)) - 1) - -GC_EXTERN unsigned GC_n_mark_procs; - -/* Number of mark stack entries to discard on overflow. */ -#define GC_MARK_STACK_DISCARDS (INITIAL_MARK_STACK_SIZE / 8) - -#ifdef PARALLEL_MARK -/* - * Allow multiple threads to participate in the marking process. - * This works roughly as follows: - * The main mark stack never shrinks, but it can grow. - * - * The initiating threads holds the GC lock, and sets GC_help_wanted. - * - * Other threads: - * 1) update helper_count (while holding mark_lock.) - * 2) allocate a local mark stack - * repeatedly: - * 3) Steal a global mark stack entry by atomically replacing - * its descriptor with 0. - * 4) Copy it to the local stack. - * 5) Mark on the local stack until it is empty, or - * it may be profitable to copy it back. - * 6) If necessary, copy local stack to global one, - * holding mark lock. - * 7) Stop when the global mark stack is empty. - * 8) decrement helper_count (holding mark_lock). - * - * This is an experiment to see if we can do something along the lines - * of the University of Tokyo SGC in a less intrusive, though probably - * also less performant, way. - */ - -/* GC_mark_stack_top is protected by mark lock. */ - -/* - * GC_notify_all_marker() is used when GC_help_wanted is first set, - * when the last helper becomes inactive, - * when something is added to the global mark stack, and just after - * GC_mark_no is incremented. - * This could be split into multiple CVs (and probably should be to - * scale to really large numbers of processors.) - */ -#endif /* PARALLEL_MARK */ - -GC_INNER mse *GC_signal_mark_stack_overflow(mse *msp); - -/* Push the object obj with corresponding heap block header hhdr onto */ -/* the mark stack. Returns the updated mark_stack_top value. */ -GC_INLINE mse *GC_push_obj(ptr_t obj, hdr *hhdr, mse *mark_stack_top, - mse *mark_stack_limit) { - word descr = hhdr->hb_descr; - - GC_ASSERT(!HBLK_IS_FREE(hhdr)); - if (descr != 0) { - mark_stack_top++; - if ((word)mark_stack_top >= (word)mark_stack_limit) { - mark_stack_top = GC_signal_mark_stack_overflow(mark_stack_top); - } - mark_stack_top->mse_start = obj; - mark_stack_top->mse_descr.w = descr; - } - return mark_stack_top; -} - -/* Push the contents of current onto the mark stack if it is a valid */ -/* ptr to a currently unmarked object. Mark it. */ -#define PUSH_CONTENTS(current, mark_stack_top, mark_stack_limit, source) \ - do { \ - hdr *my_hhdr; \ - HC_GET_HDR(current, my_hhdr, source); /* contains "break" */ \ - mark_stack_top = GC_push_contents_hdr(current, mark_stack_top, \ - mark_stack_limit, \ - source, my_hhdr, TRUE); \ - } while (0) - -/* Set mark bit, exit (using "break" statement) if it is already set. */ -#ifdef USE_MARK_BYTES -#if defined(PARALLEL_MARK) && defined(AO_HAVE_char_store) && !defined(BASE_ATOMIC_OPS_EMULATED) -/* There is a race here, and we may set the bit twice in the */ -/* concurrent case. This can result in the object being pushed */ -/* twice. But that is only a performance issue. */ -#define SET_MARK_BIT_EXIT_IF_SET(hhdr, bit_no) \ - { /* cannot use do-while(0) here */ \ - volatile unsigned char *mark_byte_addr = \ - (unsigned char *)(hhdr)->hb_marks + (bit_no); \ - /* Unordered atomic load and store are sufficient here. */ \ - if (AO_char_load(mark_byte_addr) != 0) \ - break; /* go to the enclosing loop end */ \ - AO_char_store(mark_byte_addr, 1); \ - } -#else -#define SET_MARK_BIT_EXIT_IF_SET(hhdr, bit_no) \ - { /* cannot use do-while(0) here */ \ - char *mark_byte_addr = (char *)(hhdr)->hb_marks + (bit_no); \ - if (*mark_byte_addr != 0) break; /* go to the enclosing loop end */ \ - *mark_byte_addr = 1; \ - } -#endif /* !PARALLEL_MARK */ -#else -#ifdef PARALLEL_MARK -/* This is used only if we explicitly set USE_MARK_BITS. */ -/* The following may fail to exit even if the bit was already set. */ -/* For our uses, that's benign: */ -#ifdef THREAD_SANITIZER -#define OR_WORD_EXIT_IF_SET(addr, bits) \ - { /* cannot use do-while(0) here */ \ - if (!((word)AO_load((volatile AO_t *)(addr)) & (bits))) { \ - /* Atomic load is just to avoid TSan false positive. */ \ - AO_or((volatile AO_t *)(addr), (AO_t)(bits)); \ - } else { \ - break; /* go to the enclosing loop end */ \ - } \ - } -#else -#define OR_WORD_EXIT_IF_SET(addr, bits) \ - { /* cannot use do-while(0) here */ \ - if (!(*(addr) & (bits))) { \ - AO_or((volatile AO_t *)(addr), (AO_t)(bits)); \ - } else { \ - break; /* go to the enclosing loop end */ \ - } \ - } -#endif /* !THREAD_SANITIZER */ -#else -#define OR_WORD_EXIT_IF_SET(addr, bits) \ - { /* cannot use do-while(0) here */ \ - word old = *(addr); \ - word my_bits = (bits); \ - if ((old & my_bits) != 0) \ - break; /* go to the enclosing loop end */ \ - *(addr) = old | my_bits; \ - } -#endif /* !PARALLEL_MARK */ -#define SET_MARK_BIT_EXIT_IF_SET(hhdr, bit_no) \ - { /* cannot use do-while(0) here */ \ - word *mark_word_addr = (hhdr)->hb_marks + divWORDSZ(bit_no); \ - OR_WORD_EXIT_IF_SET(mark_word_addr, \ - (word)1 << modWORDSZ(bit_no)); /* contains "break" */ \ - } -#endif /* !USE_MARK_BYTES */ - -#ifdef PARALLEL_MARK -#define INCR_MARKS(hhdr) \ - AO_store(&hhdr->hb_n_marks, AO_load(&hhdr->hb_n_marks) + 1) -#else -#define INCR_MARKS(hhdr) (void)(++hhdr->hb_n_marks) -#endif - -#ifdef ENABLE_TRACE -#define TRACE(source, cmd) \ - if (GC_trace_addr != 0 && (ptr_t)(source) == GC_trace_addr) cmd -#define TRACE_TARGET(target, cmd) \ - if (GC_trace_addr != 0 && (target) == *(ptr_t *)GC_trace_addr) cmd -#else -#define TRACE(source, cmd) -#define TRACE_TARGET(source, cmd) -#endif - -#if defined(I386) && defined(__GNUC__) && !defined(NACL) -#define LONG_MULT(hprod, lprod, x, y) \ - do { \ - __asm__ __volatile__("mull %2" \ - : "=a"(lprod), "=d"(hprod) \ - : "g"(y), "0"(x)); \ - } while (0) -#else -#if defined(__int64) && !defined(__GNUC__) && !defined(CPPCHECK) -#define ULONG_MULT_T unsigned __int64 -#else -#define ULONG_MULT_T unsigned long long -#endif -#define LONG_MULT(hprod, lprod, x, y) \ - do { \ - ULONG_MULT_T prod = (ULONG_MULT_T)(x) * (ULONG_MULT_T)(y); \ - GC_STATIC_ASSERT(sizeof(x) + sizeof(y) <= sizeof(prod)); \ - hprod = prod >> 32; \ - lprod = (unsigned32)prod; \ - } while (0) -#endif /* !I386 */ - -/* If the mark bit corresponding to current is not set, set it, and */ -/* push the contents of the object on the mark stack. Current points */ -/* to the beginning of the object. We rely on the fact that the */ -/* preceding header calculation will succeed for a pointer past the */ -/* first page of an object, only if it is in fact a valid pointer */ -/* to the object. Thus we can omit the otherwise necessary tests */ -/* here. Note in particular that the "displ" value is the displacement */ -/* from the beginning of the heap block, which may itself be in the */ -/* interior of a large object. */ -GC_INLINE mse *GC_push_contents_hdr(ptr_t current, mse *mark_stack_top, - mse *mark_stack_limit, ptr_t source, - hdr *hhdr, GC_bool do_offset_check) { - do { - size_t displ = HBLKDISPL(current); /* Displacement in block; in bytes. */ - /* displ is always within range. If current doesn't point to the */ - /* first block, then we are in the all_interior_pointers case, and */ - /* it is safe to use any displacement value. */ - ptr_t base = current; -#ifdef MARK_BIT_PER_GRANULE - size_t gran_displ = BYTES_TO_GRANULES(displ); - size_t gran_offset = hhdr->hb_map[gran_displ]; - size_t byte_offset = displ & (GRANULE_BYTES - 1); - - /* The following always fails for large block references. */ - if (EXPECT((gran_offset | byte_offset) != 0, FALSE)) -#else - unsigned32 gran_displ; /* high_prod */ - unsigned32 inv_sz = hhdr->hb_inv_sz; -#endif /* MARK_BIT_PER_OBJ */ - - { -#ifdef MARK_BIT_PER_GRANULE - if ((hhdr->hb_flags & LARGE_BLOCK) != 0) -#else - if (EXPECT(inv_sz == LARGE_INV_SZ, FALSE)) -#endif /* MARK_BIT_PER_OBJ */ - { - /* gran_offset is bogus. */ - size_t obj_displ; - - base = (ptr_t)hhdr->hb_block; - obj_displ = current - base; - if (obj_displ != displ) { - GC_ASSERT(obj_displ < hhdr->hb_sz); - /* Must be in all_interior_pointer case, not first block */ - /* already did validity check on cache miss. */ - } else if (do_offset_check && !GC_valid_offsets[obj_displ]) { - GC_ADD_TO_BLACK_LIST_NORMAL(current, source); - break; - } - GC_ASSERT(hhdr->hb_sz > HBLKSIZE || hhdr->hb_block == HBLKPTR(current)); - GC_ASSERT((word)hhdr->hb_block <= (word)current); - gran_displ = 0; - } else { -#ifdef MARK_BIT_PER_GRANULE - size_t obj_displ = GRANULES_TO_BYTES(gran_offset) + byte_offset; -#else - unsigned32 low_prod; - - LONG_MULT(gran_displ, low_prod, (unsigned32)displ, inv_sz); - if ((low_prod >> 16) != 0) -#endif /* MARK_BIT_PER_OBJ */ - { -#if defined(MARK_BIT_PER_OBJ) && !defined(MARK_BIT_PER_GRANULE) /* for cppcheck */ - size_t obj_displ; - - /* Accurate enough if HBLKSIZE <= 2**15. */ - GC_STATIC_ASSERT(HBLKSIZE <= (1 << 15)); - obj_displ = (((low_prod >> 16) + 1) * (size_t)hhdr->hb_sz) >> 16; -#endif - if (do_offset_check && !GC_valid_offsets[obj_displ]) { - GC_ADD_TO_BLACK_LIST_NORMAL(current, source); - break; - } -#ifdef MARK_BIT_PER_GRANULE - gran_displ -= gran_offset; -#endif - base -= obj_displ; - } - } - } -#ifdef MARK_BIT_PER_GRANULE - GC_ASSERT(hhdr == GC_find_header(base)); - GC_ASSERT(gran_displ % BYTES_TO_GRANULES(hhdr->hb_sz) == 0); -#else - /* May get here for pointer to start of block not at the */ - /* beginning of object. If so, it is valid, and we are fine. */ - GC_ASSERT(gran_displ <= HBLK_OBJS(hhdr->hb_sz)); -#endif /* MARK_BIT_PER_OBJ */ - TRACE(source, GC_log_printf("GC #%lu: passed validity tests\n", - (unsigned long)GC_gc_no)); - SET_MARK_BIT_EXIT_IF_SET(hhdr, gran_displ); /* contains "break" */ - TRACE(source, GC_log_printf("GC #%lu: previously unmarked\n", - (unsigned long)GC_gc_no)); - TRACE_TARGET(base, GC_log_printf("GC #%lu: marking %p from %p instead\n", - (unsigned long)GC_gc_no, (void *)base, - (void *)source)); - INCR_MARKS(hhdr); - GC_STORE_BACK_PTR(source, base); - mark_stack_top = GC_push_obj(base, hhdr, mark_stack_top, - mark_stack_limit); - } while (0); - return mark_stack_top; -} - -#if defined(PRINT_BLACK_LIST) || defined(KEEP_BACK_PTRS) -#define PUSH_ONE_CHECKED_STACK(p, source) \ - GC_mark_and_push_stack((ptr_t)(p), (ptr_t)(source)) -#else -#define PUSH_ONE_CHECKED_STACK(p, source) \ - GC_mark_and_push_stack((ptr_t)(p)) -#endif - -/* - * Push a single value onto mark stack. Mark from the object pointed to by p. - * Invoke FIXUP_POINTER(p) before any further processing. - * P is considered valid even if it is an interior pointer. - * Previously marked objects are not pushed. Hence we make progress even - * if the mark stack overflows. - */ - -#ifdef NEED_FIXUP_POINTER -/* Try both the raw version and the fixed up one. */ -#define GC_PUSH_ONE_STACK(p, source) \ - do { \ - if ((word)(p) >= (word)GC_least_plausible_heap_addr && (word)(p) < (word)GC_greatest_plausible_heap_addr) { \ - PUSH_ONE_CHECKED_STACK(p, source); \ - } \ - FIXUP_POINTER(p); \ - if ((word)(p) >= (word)GC_least_plausible_heap_addr && (word)(p) < (word)GC_greatest_plausible_heap_addr) { \ - PUSH_ONE_CHECKED_STACK(p, source); \ - } \ - } while (0) -#else /* !NEED_FIXUP_POINTER */ -#define GC_PUSH_ONE_STACK(p, source) \ - do { \ - if ((word)(p) >= (word)GC_least_plausible_heap_addr && (word)(p) < (word)GC_greatest_plausible_heap_addr) { \ - PUSH_ONE_CHECKED_STACK(p, source); \ - } \ - } while (0) -#endif - -/* As above, but interior pointer recognition as for normal heap pointers. */ -#define GC_PUSH_ONE_HEAP(p, source, mark_stack_top) \ - do { \ - FIXUP_POINTER(p); \ - if ((word)(p) >= (word)GC_least_plausible_heap_addr && (word)(p) < (word)GC_greatest_plausible_heap_addr) \ - mark_stack_top = GC_mark_and_push((void *)(p), mark_stack_top, \ - GC_mark_stack_limit, (void **)(source)); \ - } while (0) - -/* Mark starting at mark stack entry top (incl.) down to */ -/* mark stack entry bottom (incl.). Stop after performing */ -/* about one page worth of work. Return the new mark stack */ -/* top entry. */ -GC_INNER mse *GC_mark_from(mse *top, mse *bottom, mse *limit); - -#define MARK_FROM_MARK_STACK() \ - GC_mark_stack_top = GC_mark_from(GC_mark_stack_top, \ - GC_mark_stack, \ - GC_mark_stack + GC_mark_stack_size); - -#define GC_mark_stack_empty() ((word)GC_mark_stack_top < (word)GC_mark_stack) - -/* Current state of marking, as follows.*/ - -/* We say something is dirty if it was */ -/* written since the last time we */ -/* retrieved dirty bits. We say it's */ -/* grungy if it was marked dirty in the */ -/* last set of bits we retrieved. */ - -/* Invariant "I": all roots and marked */ -/* objects p are either dirty, or point */ -/* to objects q that are either marked */ -/* or a pointer to q appears in a range */ -/* on the mark stack. */ - -#define MS_NONE 0 /* No marking in progress. "I" holds. */ - /* Mark stack is empty. */ - -#define MS_PUSH_RESCUERS 1 /* Rescuing objects are currently */ - /* being pushed. "I" holds, except */ - /* that grungy roots may point to */ - /* unmarked objects, as may marked */ - /* grungy objects above GC_scan_ptr. */ - -#define MS_PUSH_UNCOLLECTABLE 2 /* "I" holds, except that marked */ - /* uncollectible objects above */ - /* GC_scan_ptr may point to unmarked */ - /* objects. Roots may point to */ - /* unmarked objects. */ - -#define MS_ROOTS_PUSHED 3 /* "I" holds, mark stack may be nonempty. */ - -#define MS_PARTIALLY_INVALID 4 /* "I" may not hold, e.g. because of */ - /* the mark stack overflow. However, */ - /* marked heap objects below */ - /* GC_scan_ptr point to marked or */ - /* stacked objects. */ - -#define MS_INVALID 5 /* "I" may not hold. */ - -EXTERN_C_END - -#endif /* GC_PMARK_H */ diff --git a/vendor/bdwgc/private/gc_priv.h b/vendor/bdwgc/private/gc_priv.h deleted file mode 100644 index 743ee388..00000000 --- a/vendor/bdwgc/private/gc_priv.h +++ /dev/null @@ -1,3130 +0,0 @@ -/* - * Copyright 1988, 1989 Hans-J. Boehm, Alan J. Demers - * Copyright (c) 1991-1994 by Xerox Corporation. All rights reserved. - * Copyright (c) 1996-1999 by Silicon Graphics. All rights reserved. - * Copyright (c) 1999-2004 Hewlett-Packard Development Company, L.P. - * Copyright (c) 2008-2022 Ivan Maidanski - * - * THIS MATERIAL IS PROVIDED AS IS, WITH ABSOLUTELY NO WARRANTY EXPRESSED - * OR IMPLIED. ANY USE IS AT YOUR OWN RISK. - * - * Permission is hereby granted to use or copy this program - * for any purpose, provided the above notices are retained on all copies. - * Permission to modify the code and to distribute modified code is granted, - * provided the above notices are retained, and a notice that the code was - * modified is included with the above copyright notice. - */ - -#ifndef GC_PRIVATE_H -#define GC_PRIVATE_H - -#ifdef HAVE_CONFIG_H -#include "config.h" -#endif - -#if !defined(GC_BUILD) && !defined(NOT_GCBUILD) -#define GC_BUILD -#endif - -#if (defined(__linux__) || defined(__GLIBC__) || defined(__GNU__) || defined(__CYGWIN__) || defined(HAVE_PTHREAD_SETNAME_NP_WITHOUT_TID) || defined(HAVE_PTHREAD_SETNAME_NP_WITH_TID_AND_ARG) || defined(HAVE_PTHREAD_SETNAME_NP_WITH_TID)) && !defined(_GNU_SOURCE) -/* Can't test LINUX, since this must be defined before other includes. */ -#define _GNU_SOURCE 1 -#endif - -#if defined(__INTERIX) && !defined(_ALL_SOURCE) -#define _ALL_SOURCE 1 -#endif - -#if (defined(DGUX) && defined(GC_THREADS) || defined(DGUX386_THREADS) || defined(GC_DGUX386_THREADS)) && !defined(_USING_POSIX4A_DRAFT10) -#define _USING_POSIX4A_DRAFT10 1 -#endif - -#if defined(__MINGW32__) && !defined(__MINGW_EXCPT_DEFINE_PSDK) && defined(__i386__) && defined(GC_EXTERN) /* defined in gc.c */ - /* See the description in mark.c. */ -#define __MINGW_EXCPT_DEFINE_PSDK 1 -#endif - -#if defined(NO_DEBUGGING) && !defined(GC_ASSERTIONS) && !defined(NDEBUG) -/* To turn off assertion checking (in atomic_ops.h). */ -#define NDEBUG 1 -#endif - -#ifndef GC_H -#include "gc/gc.h" -#endif - -#include -#if !defined(sony_news) -#include -#endif - -#ifdef DGUX -#include -#include -#include -#endif /* DGUX */ - -#ifdef BSD_TIME -#include -#include -#include -#endif /* BSD_TIME */ - -#ifdef PARALLEL_MARK -#define AO_REQUIRE_CAS -#if !defined(__GNUC__) && !defined(AO_ASSUME_WINDOWS98) -#define AO_ASSUME_WINDOWS98 -#endif -#endif - -#include "gc/gc_mark.h" -#include "gc/gc_tiny_fl.h" - -typedef GC_word word; -typedef GC_signed_word signed_word; -typedef unsigned int unsigned32; - -typedef int GC_bool; -#define TRUE 1 -#define FALSE 0 - -#ifndef PTR_T_DEFINED -typedef char *ptr_t; /* A generic pointer to which we can add */ - /* byte displacements and which can be used */ - /* for address comparisons. */ -#define PTR_T_DEFINED -#endif - -#ifndef SIZE_MAX -#include -#endif -#if defined(SIZE_MAX) && !defined(CPPCHECK) -#define GC_SIZE_MAX ((size_t)SIZE_MAX) -/* Extra cast to workaround some buggy SIZE_MAX definitions. */ -#else -#define GC_SIZE_MAX (~(size_t)0) -#endif - -#if GC_GNUC_PREREQ(3, 0) && !defined(LINT2) -#define EXPECT(expr, outcome) __builtin_expect(expr, outcome) -/* Equivalent to (expr), but predict that usually (expr)==outcome. */ -#else -#define EXPECT(expr, outcome) (expr) -#endif /* __GNUC__ */ - -/* Saturated addition of size_t values. Used to avoid value wrap */ -/* around on overflow. The arguments should have no side effects. */ -#define SIZET_SAT_ADD(a, b) \ - (EXPECT((a) < GC_SIZE_MAX - (b), TRUE) ? (a) + (b) : GC_SIZE_MAX) - -#include "gcconfig.h" - -#if !defined(GC_ATOMIC_UNCOLLECTABLE) && defined(ATOMIC_UNCOLLECTABLE) -/* For compatibility with old-style naming. */ -#define GC_ATOMIC_UNCOLLECTABLE -#endif - -#ifndef GC_INNER -/* This tagging macro must be used at the start of every variable */ -/* definition which is declared with GC_EXTERN. Should be also used */ -/* for the GC-scope function definitions and prototypes. Must not be */ -/* used in gcconfig.h. Shouldn't be used for the debugging-only */ -/* functions. Currently, not used for the functions declared in or */ -/* called from the "dated" source files (located in "extra" folder). */ -#if defined(GC_DLL) && defined(__GNUC__) && !defined(MSWIN32) && !defined(MSWINCE) && !defined(CYGWIN32) -#if GC_GNUC_PREREQ(4, 0) && !defined(GC_NO_VISIBILITY) -/* See the corresponding GC_API definition. */ -#define GC_INNER __attribute__((__visibility__("hidden"))) -#else -/* The attribute is unsupported. */ -#define GC_INNER /* empty */ -#endif -#else -#define GC_INNER /* empty */ -#endif - -#define GC_EXTERN extern GC_INNER -/* Used only for the GC-scope variables (prefixed with "GC_") */ -/* declared in the header files. Must not be used for thread-local */ -/* variables. Must not be used in gcconfig.h. Shouldn't be used for */ -/* the debugging-only or profiling-only variables. Currently, not */ -/* used for the variables accessed from the "dated" source files */ -/* (specific.c/h, and in the "extra" folder). */ -/* The corresponding variable definition must start with GC_INNER. */ -#endif /* !GC_INNER */ - -#ifdef __cplusplus -/* Register storage specifier is deprecated in C++11. */ -#define REGISTER /* empty */ -#else -/* Used only for several local variables in the performance-critical */ -/* functions. Should not be used for new code. */ -#define REGISTER register -#endif - -#if defined(CPPCHECK) -#define MACRO_BLKSTMT_BEGIN { -#define MACRO_BLKSTMT_END } -#define LOCAL_VAR_INIT_OK = 0 /* to avoid "uninit var" false positive */ -#else -#define MACRO_BLKSTMT_BEGIN do { -#define MACRO_BLKSTMT_END \ - } \ - while (0) -#define LOCAL_VAR_INIT_OK /* empty */ -#endif - -#if defined(M68K) && defined(__GNUC__) -/* By default, __alignof__(word) is 2 on m68k. Use this attribute to */ -/* have proper word alignment (i.e. 4-byte on a 32-bit arch). */ -#define GC_ATTR_WORD_ALIGNED __attribute__((__aligned__(sizeof(word)))) -#else -#define GC_ATTR_WORD_ALIGNED /* empty */ -#endif - -#ifndef HEADERS_H -#include "gc_hdrs.h" -#endif - -#ifndef GC_ATTR_NO_SANITIZE_ADDR -#ifndef ADDRESS_SANITIZER -#define GC_ATTR_NO_SANITIZE_ADDR /* empty */ -#elif GC_CLANG_PREREQ(3, 8) -#define GC_ATTR_NO_SANITIZE_ADDR __attribute__((no_sanitize("address"))) -#else -#define GC_ATTR_NO_SANITIZE_ADDR __attribute__((no_sanitize_address)) -#endif -#endif /* !GC_ATTR_NO_SANITIZE_ADDR */ - -#ifndef GC_ATTR_NO_SANITIZE_MEMORY -#ifndef MEMORY_SANITIZER -#define GC_ATTR_NO_SANITIZE_MEMORY /* empty */ -#elif GC_CLANG_PREREQ(3, 8) -#define GC_ATTR_NO_SANITIZE_MEMORY __attribute__((no_sanitize("memory"))) -#else -#define GC_ATTR_NO_SANITIZE_MEMORY __attribute__((no_sanitize_memory)) -#endif -#endif /* !GC_ATTR_NO_SANITIZE_MEMORY */ - -#ifndef GC_ATTR_NO_SANITIZE_THREAD -#ifndef THREAD_SANITIZER -#define GC_ATTR_NO_SANITIZE_THREAD /* empty */ -#elif GC_CLANG_PREREQ(3, 8) -#define GC_ATTR_NO_SANITIZE_THREAD __attribute__((no_sanitize("thread"))) -#else -/* It seems that no_sanitize_thread attribute has no effect if the */ -/* function is inlined (as of gcc 11.1.0, at least). */ -#define GC_ATTR_NO_SANITIZE_THREAD \ - GC_ATTR_NOINLINE __attribute__((no_sanitize_thread)) -#endif -#endif /* !GC_ATTR_NO_SANITIZE_THREAD */ - -#ifndef UNUSED_ARG -#define UNUSED_ARG(arg) ((void)(arg)) -#endif - -#ifdef HAVE_CONFIG_H -/* The "inline" keyword is determined by Autoconf AC_C_INLINE. */ -#define GC_INLINE static inline -#elif defined(_MSC_VER) || defined(__INTEL_COMPILER) || defined(__DMC__) || (GC_GNUC_PREREQ(3, 0) && defined(__STRICT_ANSI__)) || defined(__BORLANDC__) || defined(__WATCOMC__) -#define GC_INLINE static __inline -#elif GC_GNUC_PREREQ(3, 0) || defined(__sun) -#define GC_INLINE static inline -#else -#define GC_INLINE static -#endif - -#ifndef GC_ATTR_NOINLINE -#if GC_GNUC_PREREQ(4, 0) -#define GC_ATTR_NOINLINE __attribute__((__noinline__)) -#elif _MSC_VER >= 1400 -#define GC_ATTR_NOINLINE __declspec(noinline) -#else -#define GC_ATTR_NOINLINE /* empty */ -#endif -#endif - -#ifndef GC_API_OSCALL -/* This is used to identify GC routines called by name from OS. */ -#if defined(__GNUC__) -#if GC_GNUC_PREREQ(4, 0) && !defined(GC_NO_VISIBILITY) -/* Same as GC_API if GC_DLL. */ -#define GC_API_OSCALL extern __attribute__((__visibility__("default"))) -#else -/* The attribute is unsupported. */ -#define GC_API_OSCALL extern -#endif -#else -#define GC_API_OSCALL GC_API -#endif -#endif - -#ifndef GC_API_PRIV -#define GC_API_PRIV GC_API -#endif - -#if defined(THREADS) && !defined(NN_PLATFORM_CTR) -#include "gc_atomic_ops.h" -#ifndef AO_HAVE_compiler_barrier -#define AO_HAVE_compiler_barrier 1 -#endif -#endif - -#if defined(MSWIN32) || defined(MSWINCE) || defined(CYGWIN32) -#ifndef WIN32_LEAN_AND_MEAN -#define WIN32_LEAN_AND_MEAN 1 -#endif -#define NOSERVICE -#include -#include -#endif - -#include "gc_locks.h" - -#define GC_WORD_MAX (~(word)0) - -#ifdef STACK_GROWS_DOWN -#define COOLER_THAN > -#define HOTTER_THAN < -#define MAKE_COOLER(x, y) \ - if ((word)((x) + (y)) > (word)(x)) { \ - (x) += (y); \ - } else \ - (x) = (ptr_t)GC_WORD_MAX -#define MAKE_HOTTER(x, y) (x) -= (y) -#else -#define COOLER_THAN < -#define HOTTER_THAN > -#define MAKE_COOLER(x, y) \ - if ((word)((x) - (y)) < (word)(x)) { \ - (x) -= (y); \ - } else \ - (x) = 0 -#define MAKE_HOTTER(x, y) (x) += (y) -#endif - -#if defined(AMIGA) && defined(__SASC) -#define GC_FAR __far -#else -#define GC_FAR -#endif - -#ifdef GC_ASSERTIONS -#define GC_ASSERT(expr) \ - do { \ - if (EXPECT(!(expr), FALSE)) { \ - GC_err_printf("Assertion failure: %s:%d\n", __FILE__, __LINE__); \ - ABORT("assertion failure"); \ - } \ - } while (0) -#else -#define GC_ASSERT(expr) -#endif - -#include "gc/gc_inline.h" - -/*********************************/ -/* */ -/* Definitions for conservative */ -/* collector */ -/* */ -/*********************************/ - -/*********************************/ -/* */ -/* Easily changeable parameters */ -/* */ -/*********************************/ - -/* #define ALL_INTERIOR_POINTERS */ -/* Forces all pointers into the interior of an */ -/* object to be considered valid. Also causes the */ -/* sizes of all objects to be inflated by at least */ -/* one byte. This should suffice to guarantee */ -/* that in the presence of a compiler that does */ -/* not perform garbage-collector-unsafe */ -/* optimizations, all portable, strictly ANSI */ -/* conforming C programs should be safely usable */ -/* with malloc replaced by GC_malloc and free */ -/* calls removed. There are several disadvantages: */ -/* 1. There are probably no interesting, portable, */ -/* strictly ANSI conforming C programs. */ -/* 2. This option makes it hard for the collector */ -/* to allocate space that is not "pointed to" */ -/* by integers, etc. Under SunOS 4.X with a */ -/* statically linked libc, we empirically */ -/* observed that it would be difficult to */ -/* allocate individual objects > 100 KB. */ -/* Even if only smaller objects are allocated, */ -/* more swap space is likely to be needed. */ -/* Fortunately, much of this will never be */ -/* touched. */ -/* If you can easily avoid using this option, do. */ -/* If not, try to keep individual objects small. */ -/* This is now really controlled at startup, */ -/* through GC_all_interior_pointers. */ - -EXTERN_C_BEGIN - -#ifndef GC_NO_FINALIZATION -#define GC_INVOKE_FINALIZERS() GC_notify_or_invoke_finalizers() -GC_INNER void GC_notify_or_invoke_finalizers(void); -/* If GC_finalize_on_demand is not set, invoke */ -/* eligible finalizers. Otherwise: */ -/* Call *GC_finalizer_notifier if there are */ -/* finalizers to be run, and we haven't called */ -/* this procedure yet this GC cycle. */ - -GC_INNER void GC_finalize(void); -/* Perform all indicated finalization actions */ -/* on unmarked objects. */ -/* Unreachable finalizable objects are enqueued */ -/* for processing by GC_invoke_finalizers. */ -/* Invoked with lock. */ - -#ifndef GC_TOGGLE_REFS_NOT_NEEDED -GC_INNER void GC_process_togglerefs(void); -/* Process the toggle-refs before GC starts. */ -#endif -#ifndef SMALL_CONFIG -GC_INNER void GC_print_finalization_stats(void); -#endif -#else -#define GC_INVOKE_FINALIZERS() (void)0 -#endif /* GC_NO_FINALIZATION */ - -#if !defined(DONT_ADD_BYTE_AT_END) -#ifdef LINT2 -/* Explicitly instruct the code analysis tool that */ -/* GC_all_interior_pointers is assumed to have only 0 or 1 value. */ -#define EXTRA_BYTES ((size_t)(GC_all_interior_pointers ? 1 : 0)) -#else -#define EXTRA_BYTES (size_t) GC_all_interior_pointers -#endif -#define MAX_EXTRA_BYTES 1 -#else -#define EXTRA_BYTES 0 -#define MAX_EXTRA_BYTES 0 -#endif - -#ifndef LARGE_CONFIG -#define MINHINCR 16 /* Minimum heap increment, in blocks of HBLKSIZE */ - /* Must be multiple of largest page size. */ -#define MAXHINCR 2048 /* Maximum heap increment, in blocks */ -#else -#define MINHINCR 64 -#define MAXHINCR 4096 -#endif - -#define BL_LIMIT GC_black_list_spacing -/* If we need a block of N bytes, and we have */ -/* a block of N + BL_LIMIT bytes available, */ -/* and N > BL_LIMIT, */ -/* but all possible positions in it are */ -/* blacklisted, we just use it anyway (and */ -/* print a warning, if warnings are enabled). */ -/* This risks subsequently leaking the block */ -/* due to a false reference. But not using */ -/* the block risks unreasonable immediate */ -/* heap growth. */ - -/*********************************/ -/* */ -/* Stack saving for debugging */ -/* */ -/*********************************/ - -#ifdef NEED_CALLINFO -struct callinfo { - word ci_pc; /* Caller, not callee, pc */ -#if NARGS > 0 - word ci_arg[NARGS]; /* bit-wise complement to avoid retention */ -#endif -#if (NFRAMES * (NARGS + 1)) % 2 == 1 - /* Likely alignment problem. */ - word ci_dummy; -#endif -}; -#endif - -#ifdef SAVE_CALL_CHAIN -/* Fill in the pc and argument information for up to NFRAMES of my */ -/* callers. Ignore my frame and my callers frame. */ -GC_INNER void GC_save_callers(struct callinfo info[NFRAMES]); -GC_INNER void GC_print_callers(struct callinfo info[NFRAMES]); -#endif - -EXTERN_C_END - -/*********************************/ -/* */ -/* OS interface routines */ -/* */ -/*********************************/ - -#ifndef NO_CLOCK -#ifdef BSD_TIME -#undef CLOCK_TYPE -#undef GET_TIME -#undef MS_TIME_DIFF -#define CLOCK_TYPE struct timeval -#define CLOCK_TYPE_INITIALIZER \ - { 0, 0 } -#define GET_TIME(x) \ - do { \ - struct rusage rusage; \ - getrusage(RUSAGE_SELF, &rusage); \ - x = rusage.ru_utime; \ - } while (0) -#define MS_TIME_DIFF(a, b) ((unsigned long)((long)(a.tv_sec - b.tv_sec) * 1000 + (long)(a.tv_usec - b.tv_usec) / 1000 - (a.tv_usec < b.tv_usec && (long)(a.tv_usec - b.tv_usec) % 1000 != 0 ? 1 : 0))) -/* "a" time is expected to be not earlier than */ -/* "b" one; the result has unsigned long type. */ -#define NS_FRAC_TIME_DIFF(a, b) ((unsigned long)((a.tv_usec < b.tv_usec && (long)(a.tv_usec - b.tv_usec) % 1000 != 0 ? 1000L : 0) + (long)(a.tv_usec - b.tv_usec) % 1000) * 1000) -/* The total time difference could be computed as */ -/* MS_TIME_DIFF(a,b)*1000000+NS_FRAC_TIME_DIFF(a,b).*/ - -#elif defined(MSWIN32) || defined(MSWINCE) || defined(WINXP_USE_PERF_COUNTER) -#if defined(MSWINRT_FLAVOR) || defined(WINXP_USE_PERF_COUNTER) -#define CLOCK_TYPE ULONGLONG -#define GET_TIME(x) \ - do { \ - LARGE_INTEGER freq, tc; \ - if (!QueryPerformanceFrequency(&freq)) \ - ABORT("QueryPerformanceFrequency requires WinXP+"); \ - /* Note: two standalone if statements are needed to */ \ - /* avoid MS VC false warning about potentially */ \ - /* uninitialized tc variable. */ \ - if (!QueryPerformanceCounter(&tc)) \ - ABORT("QueryPerformanceCounter failed"); \ - x = (CLOCK_TYPE)((double)tc.QuadPart / freq.QuadPart * 1e9); \ - } while (0) -/* TODO: Call QueryPerformanceFrequency once at GC init. */ -#define MS_TIME_DIFF(a, b) ((unsigned long)(((a) - (b)) / 1000000UL)) -#define NS_FRAC_TIME_DIFF(a, b) ((unsigned long)(((a) - (b)) % 1000000UL)) -#else -#define CLOCK_TYPE DWORD -#define GET_TIME(x) (void)(x = GetTickCount()) -#define MS_TIME_DIFF(a, b) ((unsigned long)((a) - (b))) -#define NS_FRAC_TIME_DIFF(a, b) 0UL -#endif /* !WINXP_USE_PERF_COUNTER */ - -#elif defined(NN_PLATFORM_CTR) -#define CLOCK_TYPE long long -EXTERN_C_BEGIN -CLOCK_TYPE n3ds_get_system_tick(void); -CLOCK_TYPE n3ds_convert_tick_to_ms(CLOCK_TYPE tick); -EXTERN_C_END -#define GET_TIME(x) (void)(x = n3ds_get_system_tick()) -#define MS_TIME_DIFF(a, b) ((unsigned long)n3ds_convert_tick_to_ms((a) - (b))) -#define NS_FRAC_TIME_DIFF(a, b) 0UL /* TODO: implement it */ - -#elif defined(HAVE_CLOCK_GETTIME) -#include -#define CLOCK_TYPE struct timespec -#define CLOCK_TYPE_INITIALIZER \ - { 0, 0 } -#if defined(_POSIX_MONOTONIC_CLOCK) && !defined(NINTENDO_SWITCH) -#define GET_TIME(x) \ - do { \ - if (clock_gettime(CLOCK_MONOTONIC, &x) == -1) \ - ABORT("clock_gettime failed"); \ - } while (0) -#else -#define GET_TIME(x) \ - do { \ - if (clock_gettime(CLOCK_REALTIME, &x) == -1) \ - ABORT("clock_gettime failed"); \ - } while (0) -#endif -#define MS_TIME_DIFF(a, b) \ - /* a.tv_nsec - b.tv_nsec is in range -1e9 to 1e9 exclusively */ \ - ((unsigned long)((a).tv_nsec + (1000000L * 1000 - (b).tv_nsec)) / 1000000UL + ((unsigned long)((a).tv_sec - (b).tv_sec) * 1000UL) - 1000UL) -#define NS_FRAC_TIME_DIFF(a, b) \ - ((unsigned long)((a).tv_nsec + (1000000L * 1000 - (b).tv_nsec)) % 1000000UL) - -#else /* !BSD_TIME && !LINUX && !NN_PLATFORM_CTR && !MSWIN32 */ -#include -#if defined(FREEBSD) && !defined(CLOCKS_PER_SEC) -#include -#define CLOCKS_PER_SEC CLK_TCK -#endif -#if !defined(CLOCKS_PER_SEC) -#define CLOCKS_PER_SEC 1000000 -/* This is technically a bug in the implementation. */ -/* ANSI requires that CLOCKS_PER_SEC be defined. But at least */ -/* under SunOS 4.1.1, it isn't. Also note that the combination of */ -/* ANSI C and POSIX is incredibly gross here. The type clock_t */ -/* is used by both clock() and times(). But on some machines */ -/* these use different notions of a clock tick, CLOCKS_PER_SEC */ -/* seems to apply only to clock. Hence we use it here. On many */ -/* machines, including SunOS, clock actually uses units of */ -/* microseconds (which are not really clock ticks). */ -#endif -#define CLOCK_TYPE clock_t -#define GET_TIME(x) (void)(x = clock()) -#define MS_TIME_DIFF(a, b) (CLOCKS_PER_SEC % 1000 == 0 ? (unsigned long)((a) - (b)) / (unsigned long)(CLOCKS_PER_SEC / 1000) \ - : ((unsigned long)((a) - (b)) * 1000) / (unsigned long)CLOCKS_PER_SEC) -/* Avoid using double type since some targets (like ARM) might */ -/* require -lm option for double-to-long conversion. */ -#define NS_FRAC_TIME_DIFF(a, b) (CLOCKS_PER_SEC <= 1000 ? 0UL \ - : (unsigned long)(CLOCKS_PER_SEC <= (clock_t)1000000UL \ - ? (((a) - (b)) * ((clock_t)1000000UL / CLOCKS_PER_SEC) % 1000) * 1000 \ - : (CLOCKS_PER_SEC <= (clock_t)1000000UL * 1000 \ - ? ((a) - (b)) * ((clock_t)1000000UL * 1000 / CLOCKS_PER_SEC) \ - : (((a) - (b)) * (clock_t)1000000UL * 1000) / CLOCKS_PER_SEC) % \ - (clock_t)1000000UL)) -#endif /* !BSD_TIME && !MSWIN32 */ -#ifndef CLOCK_TYPE_INITIALIZER -/* This is used to initialize CLOCK_TYPE variables (to some value) */ -/* to avoid "variable might be uninitialized" compiler warnings. */ -#define CLOCK_TYPE_INITIALIZER 0 -#endif -#endif /* !NO_CLOCK */ - -/* We use bzero and bcopy internally. They may not be available. */ -#if defined(SPARC) && defined(SUNOS4) || (defined(M68K) && defined(NEXT)) || defined(VAX) -#define BCOPY_EXISTS -#elif defined(AMIGA) || defined(DARWIN) -#include -#define BCOPY_EXISTS -#elif defined(MACOS) && defined(POWERPC) -#include -#define bcopy(x, y, n) BlockMoveData(x, y, n) -#define bzero(x, n) BlockZero(x, n) -#define BCOPY_EXISTS -#endif - -#if !defined(BCOPY_EXISTS) || defined(CPPCHECK) -#include -#define BCOPY(x, y, n) memcpy(y, x, (size_t)(n)) -#define BZERO(x, n) memset(x, 0, (size_t)(n)) -#else -#define BCOPY(x, y, n) bcopy((void *)(x), (void *)(y), (size_t)(n)) -#define BZERO(x, n) bzero((void *)(x), (size_t)(n)) -#endif - -#ifdef PCR -#include "th/PCR_ThCtl.h" -#endif - -EXTERN_C_BEGIN - -#if defined(CPPCHECK) && (defined(MSWIN32) || defined(MSWINCE) || defined(CYGWIN32)) -#undef TEXT -#ifdef UNICODE -#define TEXT(s) L##s -#else -#define TEXT(s) s -#endif -#endif /* CPPCHECK */ - -/* - * Stop and restart mutator threads. - */ -#ifdef PCR -#define STOP_WORLD() \ - PCR_ThCtl_SetExclusiveMode(PCR_ThCtl_ExclusiveMode_stopNormal, \ - PCR_allSigsBlocked, \ - PCR_waitForever) -#define START_WORLD() \ - PCR_ThCtl_SetExclusiveMode(PCR_ThCtl_ExclusiveMode_null, \ - PCR_allSigsBlocked, \ - PCR_waitForever) -#else -#if defined(NN_PLATFORM_CTR) || defined(NINTENDO_SWITCH) || defined(GC_WIN32_THREADS) || defined(GC_PTHREADS) -GC_INNER void GC_stop_world(void); -GC_INNER void GC_start_world(void); -#define STOP_WORLD() GC_stop_world() -#define START_WORLD() GC_start_world() -#else -/* Just do a sanity check: we are not inside GC_do_blocking(). */ -#define STOP_WORLD() GC_ASSERT(GC_blocked_sp == NULL) -#define START_WORLD() -#endif -#endif - -#ifdef THREADS -GC_EXTERN GC_on_thread_event_proc GC_on_thread_event; -#endif - -/* Abandon ship */ -#if defined(SMALL_CONFIG) || defined(PCR) -#define GC_on_abort(msg) (void)0 /* be silent on abort */ -#else -GC_API_PRIV GC_abort_func GC_on_abort; -#endif -#if defined(CPPCHECK) -#define ABORT(msg) \ - { \ - GC_on_abort(msg); \ - abort(); \ - } -#elif defined(PCR) -#define ABORT(s) PCR_Base_Panic(s) -#else -#if defined(MSWIN_XBOX1) && !defined(DebugBreak) -#define DebugBreak() __debugbreak() -#elif defined(MSWINCE) && !defined(DebugBreak) && (!defined(UNDER_CE) || (defined(__MINGW32CE__) && !defined(ARM32))) -/* This simplifies linking for WinCE (and, probably, doesn't */ -/* hurt debugging much); use -DDebugBreak=DebugBreak to override */ -/* this behavior if really needed. This is also a workaround for */ -/* x86mingw32ce toolchain (if it is still declaring DebugBreak() */ -/* instead of defining it as a macro). */ -#define DebugBreak() _exit(-1) /* there is no abort() in WinCE */ -#endif -#if defined(MSWIN32) && (defined(NO_DEBUGGING) || defined(LINT2)) -/* A more user-friendly abort after showing fatal message. */ -#define ABORT(msg) (GC_on_abort(msg), _exit(-1)) -/* Exit on error without running "at-exit" callbacks. */ -#elif defined(MSWINCE) && defined(NO_DEBUGGING) -#define ABORT(msg) (GC_on_abort(msg), ExitProcess(-1)) -#elif defined(MSWIN32) || defined(MSWINCE) -#if defined(_CrtDbgBreak) && defined(_DEBUG) && defined(_MSC_VER) -#define ABORT(msg) \ - { \ - GC_on_abort(msg); \ - _CrtDbgBreak() /* __debugbreak() */; \ - } -#else -#define ABORT(msg) \ - { \ - GC_on_abort(msg); \ - DebugBreak(); \ - } -/* Note that: on a WinCE box, this could be silently */ -/* ignored (i.e., the program is not aborted); */ -/* DebugBreak is a statement in some toolchains. */ -#endif -#else -#define ABORT(msg) (GC_on_abort(msg), abort()) -#endif /* !MSWIN32 */ -#endif /* !PCR */ - -/* For abort message with 1-3 arguments. C_msg and C_fmt should be */ -/* literals. C_msg should not contain format specifiers. Arguments */ -/* should match their format specifiers. */ -#define ABORT_ARG1(C_msg, C_fmt, arg1) \ - MACRO_BLKSTMT_BEGIN \ - GC_ERRINFO_PRINTF(C_msg /* + */ C_fmt "\n", arg1); \ - ABORT(C_msg); \ - MACRO_BLKSTMT_END -#define ABORT_ARG2(C_msg, C_fmt, arg1, arg2) \ - MACRO_BLKSTMT_BEGIN \ - GC_ERRINFO_PRINTF(C_msg /* + */ C_fmt "\n", arg1, arg2); \ - ABORT(C_msg); \ - MACRO_BLKSTMT_END -#define ABORT_ARG3(C_msg, C_fmt, arg1, arg2, arg3) \ - MACRO_BLKSTMT_BEGIN \ - GC_ERRINFO_PRINTF(C_msg /* + */ C_fmt "\n", \ - arg1, arg2, arg3); \ - ABORT(C_msg); \ - MACRO_BLKSTMT_END - -/* Same as ABORT but does not have 'no-return' attribute. */ -/* ABORT on a dummy condition (which is always true). */ -#define ABORT_RET(msg) \ - if ((signed_word)GC_current_warn_proc == -1) { \ - } else \ - ABORT(msg) - -/* Exit abnormally, but without making a mess (e.g. out of memory) */ -#ifdef PCR -#define EXIT() PCR_Base_Exit(1, PCR_waitForever) -#else -#define EXIT() (GC_on_abort(NULL), exit(1 /* EXIT_FAILURE */)) -#endif - -/* Print warning message, e.g. almost out of memory. */ -/* The argument (if any) format specifier should be: */ -/* "%s", "%p", "%"WARN_PRIdPTR or "%"WARN_PRIuPTR. */ -#define WARN(msg, arg) \ - ((*GC_current_warn_proc)((/* no const */ char *)("GC Warning: " msg), \ - (word)(arg)), __builtin_trap()) -GC_EXTERN GC_warn_proc GC_current_warn_proc; - -/* Print format type macro for decimal signed_word value passed WARN(). */ -/* This could be redefined for Win64 or LLP64, but typically should */ -/* not be done as the WARN format string is, possibly, processed on the */ -/* client side, so non-standard print type modifiers (like MS "I64d") */ -/* should be avoided here if possible. */ -#ifndef WARN_PRIdPTR -/* Assume sizeof(void *) == sizeof(long) or a little-endian machine. */ -#define WARN_PRIdPTR "ld" -#define WARN_PRIuPTR "lu" -#endif - -/* A tagging macro (for a code static analyzer) to indicate that the */ -/* string obtained from an untrusted source (e.g., argv[], getenv) is */ -/* safe to use in a vulnerable operation (e.g., open, exec). */ -#define TRUSTED_STRING(s) (char *)COVERT_DATAFLOW(s) - -/* Get environment entry */ -#ifdef GC_READ_ENV_FILE -GC_INNER char *GC_envfile_getenv(const char *name); -#define GETENV(name) GC_envfile_getenv(name) -#elif defined(NO_GETENV) && !defined(CPPCHECK) -#define GETENV(name) NULL -#elif defined(EMPTY_GETENV_RESULTS) -/* Workaround for a reputed Wine bug. */ -GC_INLINE char *fixed_getenv(const char *name) { - char *value = getenv(name); - return value != NULL && *value != '\0' ? value : NULL; -} -#define GETENV(name) fixed_getenv(name) -#else -#define GETENV(name) getenv(name) -#endif - -EXTERN_C_END - -#if defined(DARWIN) -#include -#ifndef MAC_OS_X_VERSION_MAX_ALLOWED -#include -/* Include this header just to import the above macro. */ -#endif -#if defined(POWERPC) -#if CPP_WORDSZ == 32 -#define GC_THREAD_STATE_T ppc_thread_state_t -#else -#define GC_THREAD_STATE_T ppc_thread_state64_t -#define GC_MACH_THREAD_STATE PPC_THREAD_STATE64 -#define GC_MACH_THREAD_STATE_COUNT PPC_THREAD_STATE64_COUNT -#endif -#elif defined(I386) || defined(X86_64) -#if CPP_WORDSZ == 32 -#if defined(i386_THREAD_STATE_COUNT) && !defined(x86_THREAD_STATE32_COUNT) -/* Use old naming convention for 32-bit x86. */ -#define GC_THREAD_STATE_T i386_thread_state_t -#define GC_MACH_THREAD_STATE i386_THREAD_STATE -#define GC_MACH_THREAD_STATE_COUNT i386_THREAD_STATE_COUNT -#else -#define GC_THREAD_STATE_T x86_thread_state32_t -#define GC_MACH_THREAD_STATE x86_THREAD_STATE32 -#define GC_MACH_THREAD_STATE_COUNT x86_THREAD_STATE32_COUNT -#endif -#else -#define GC_THREAD_STATE_T x86_thread_state64_t -#define GC_MACH_THREAD_STATE x86_THREAD_STATE64 -#define GC_MACH_THREAD_STATE_COUNT x86_THREAD_STATE64_COUNT -#endif -#elif defined(ARM32) && defined(ARM_UNIFIED_THREAD_STATE) && !defined(CPPCHECK) -#define GC_THREAD_STATE_T arm_unified_thread_state_t -#define GC_MACH_THREAD_STATE ARM_UNIFIED_THREAD_STATE -#define GC_MACH_THREAD_STATE_COUNT ARM_UNIFIED_THREAD_STATE_COUNT -#elif defined(ARM32) -#define GC_THREAD_STATE_T arm_thread_state_t -#ifdef ARM_MACHINE_THREAD_STATE_COUNT -#define GC_MACH_THREAD_STATE ARM_MACHINE_THREAD_STATE -#define GC_MACH_THREAD_STATE_COUNT ARM_MACHINE_THREAD_STATE_COUNT -#endif -#elif defined(AARCH64) -#define GC_THREAD_STATE_T arm_thread_state64_t -#define GC_MACH_THREAD_STATE ARM_THREAD_STATE64 -#define GC_MACH_THREAD_STATE_COUNT ARM_THREAD_STATE64_COUNT -#elif !defined(CPPCHECK) -#error define GC_THREAD_STATE_T -#endif -#ifndef GC_MACH_THREAD_STATE -#define GC_MACH_THREAD_STATE MACHINE_THREAD_STATE -#define GC_MACH_THREAD_STATE_COUNT MACHINE_THREAD_STATE_COUNT -#endif - -#if CPP_WORDSZ == 32 -#define GC_MACH_HEADER mach_header -#define GC_MACH_SECTION section -#define GC_GETSECTBYNAME getsectbynamefromheader -#else -#define GC_MACH_HEADER mach_header_64 -#define GC_MACH_SECTION section_64 -#define GC_GETSECTBYNAME getsectbynamefromheader_64 -#endif - -/* Try to work out the right way to access thread state structure */ -/* members. The structure has changed its definition in different */ -/* Darwin versions. This now defaults to the (older) names */ -/* without __, thus hopefully, not breaking any existing */ -/* Makefile.direct builds. */ -#if __DARWIN_UNIX03 -#define THREAD_FLD_NAME(x) __##x -#else -#define THREAD_FLD_NAME(x) x -#endif -#if defined(ARM32) && defined(ARM_UNIFIED_THREAD_STATE) -#define THREAD_FLD(x) ts_32.THREAD_FLD_NAME(x) -#else -#define THREAD_FLD(x) THREAD_FLD_NAME(x) -#endif -#endif /* DARWIN */ - -#ifndef WASI -#include -#endif - -#include - -#if __STDC_VERSION__ >= 201112L -#include /* for static_assert */ -#endif - -EXTERN_C_BEGIN - -/*********************************/ -/* */ -/* Word-size-dependent defines */ -/* */ -/*********************************/ - -#if CPP_WORDSZ == 32 -#define WORDS_TO_BYTES(x) ((x) << 2) -#define BYTES_TO_WORDS(x) ((x) >> 2) -#define LOGWL ((word)5) /* log[2] of CPP_WORDSZ */ -#define modWORDSZ(n) ((n)&0x1f) /* n mod size of word */ -#if ALIGNMENT != 4 -#define UNALIGNED_PTRS -#endif -#endif - -#if CPP_WORDSZ == 64 -#define WORDS_TO_BYTES(x) ((x) << 3) -#define BYTES_TO_WORDS(x) ((x) >> 3) -#define LOGWL ((word)6) /* log[2] of CPP_WORDSZ */ -#define modWORDSZ(n) ((n)&0x3f) /* n mod size of word */ -#if ALIGNMENT != 8 -#define UNALIGNED_PTRS -#endif -#endif - -/* The first TINY_FREELISTS free lists correspond to the first */ -/* TINY_FREELISTS multiples of GRANULE_BYTES, i.e. we keep */ -/* separate free lists for each multiple of GRANULE_BYTES */ -/* up to (TINY_FREELISTS-1) * GRANULE_BYTES. After that they */ -/* may be spread out further. */ - -#define GRANULE_BYTES GC_GRANULE_BYTES -#define TINY_FREELISTS GC_TINY_FREELISTS - -#define WORDSZ ((word)CPP_WORDSZ) -#define SIGNB ((word)1 << (WORDSZ - 1)) -#define BYTES_PER_WORD ((word)(sizeof(word))) -#define divWORDSZ(n) ((n) >> LOGWL) /* divide n by size of word */ - -#if GRANULE_BYTES == 8 -#define BYTES_TO_GRANULES(n) ((n) >> 3) -#define GRANULES_TO_BYTES(n) ((n) << 3) -#if CPP_WORDSZ == 64 -#define GRANULES_TO_WORDS(n) (n) -#elif CPP_WORDSZ == 32 -#define GRANULES_TO_WORDS(n) ((n) << 1) -#else -#define GRANULES_TO_WORDS(n) BYTES_TO_WORDS(GRANULES_TO_BYTES(n)) -#endif -#elif GRANULE_BYTES == 16 -#define BYTES_TO_GRANULES(n) ((n) >> 4) -#define GRANULES_TO_BYTES(n) ((n) << 4) -#if CPP_WORDSZ == 64 -#define GRANULES_TO_WORDS(n) ((n) << 1) -#elif CPP_WORDSZ == 32 -#define GRANULES_TO_WORDS(n) ((n) << 2) -#else -#define GRANULES_TO_WORDS(n) BYTES_TO_WORDS(GRANULES_TO_BYTES(n)) -#endif -#else -#error Bad GRANULE_BYTES value -#endif - -/*********************/ -/* */ -/* Size Parameters */ -/* */ -/*********************/ - -/* Heap block size, bytes. Should be power of 2. */ -/* Incremental GC with MPROTECT_VDB currently requires the */ -/* page size to be a multiple of HBLKSIZE. Since most modern */ -/* architectures support variable page sizes down to 4 KB, and */ -/* x86 is generally 4 KB, we now default to 4 KB, except for */ -/* Alpha: Seems to be used with 8 KB pages. */ -/* SMALL_CONFIG: Want less block-level fragmentation. */ -#ifndef HBLKSIZE -#if defined(LARGE_CONFIG) || !defined(SMALL_CONFIG) -#ifdef ALPHA -#define CPP_LOG_HBLKSIZE 13 -#else -#define CPP_LOG_HBLKSIZE 12 -#endif -#else -#define CPP_LOG_HBLKSIZE 10 -#endif -#else -#if HBLKSIZE == 512 -#define CPP_LOG_HBLKSIZE 9 -#elif HBLKSIZE == 1024 -#define CPP_LOG_HBLKSIZE 10 -#elif HBLKSIZE == 2048 -#define CPP_LOG_HBLKSIZE 11 -#elif HBLKSIZE == 4096 -#define CPP_LOG_HBLKSIZE 12 -#elif HBLKSIZE == 8192 -#define CPP_LOG_HBLKSIZE 13 -#elif HBLKSIZE == 16384 -#define CPP_LOG_HBLKSIZE 14 -#elif HBLKSIZE == 32768 -#define CPP_LOG_HBLKSIZE 15 -#elif HBLKSIZE == 65536 -#define CPP_LOG_HBLKSIZE 16 -#elif !defined(CPPCHECK) -#error Bad HBLKSIZE value -#endif -#undef HBLKSIZE -#endif - -#define CPP_HBLKSIZE (1 << CPP_LOG_HBLKSIZE) -#define LOG_HBLKSIZE ((size_t)CPP_LOG_HBLKSIZE) -#define HBLKSIZE ((size_t)CPP_HBLKSIZE) - -#define GC_SQRT_SIZE_MAX ((((size_t)1) << (WORDSZ / 2)) - 1) - -/* Max size objects supported by freelist (larger objects are */ -/* allocated directly with allchblk(), by rounding to the next */ -/* multiple of HBLKSIZE). */ -#define CPP_MAXOBJBYTES (CPP_HBLKSIZE / 2) -#define MAXOBJBYTES ((size_t)CPP_MAXOBJBYTES) -#define CPP_MAXOBJWORDS BYTES_TO_WORDS(CPP_MAXOBJBYTES) -#define MAXOBJWORDS ((size_t)CPP_MAXOBJWORDS) -#define CPP_MAXOBJGRANULES BYTES_TO_GRANULES(CPP_MAXOBJBYTES) -#define MAXOBJGRANULES ((size_t)CPP_MAXOBJGRANULES) - -#define divHBLKSZ(n) ((n) >> LOG_HBLKSIZE) - -#define HBLK_PTR_DIFF(p, q) divHBLKSZ((ptr_t)p - (ptr_t)q) -/* Equivalent to subtracting 2 hblk pointers. */ -/* We do it this way because a compiler should */ -/* find it hard to use an integer division */ -/* instead of a shift. The bundled SunOS 4.1 */ -/* o.w. sometimes pessimizes the subtraction to */ -/* involve a call to .div. */ - -#define modHBLKSZ(n) ((n) & (HBLKSIZE - 1)) - -#define HBLKPTR(objptr) ((struct hblk *)(((word)(objptr)) & ~(word)(HBLKSIZE - 1))) -#define HBLKDISPL(objptr) modHBLKSZ((size_t)(objptr)) - -/* Round up allocation size (in bytes) to a multiple of a granule. */ -#define ROUNDUP_GRANULE_SIZE(lb) /* lb should have no side-effect */ \ - (SIZET_SAT_ADD(lb, GRANULE_BYTES - 1) & ~(GRANULE_BYTES - 1)) - -#define ADD_EXTRA_BYTES(lb) /* lb should have no side-effect */ \ - SIZET_SAT_ADD(lb, EXTRA_BYTES) - -/* Round up byte allocation request (after adding EXTRA_BYTES) to */ -/* a multiple of a granule, then convert it to granules. */ -#define ALLOC_REQUEST_GRANS(lb) /* lb should have no side-effect */ \ - BYTES_TO_GRANULES(SIZET_SAT_ADD(lb, GRANULE_BYTES - 1 + EXTRA_BYTES)) - -#if MAX_EXTRA_BYTES == 0 -#define SMALL_OBJ(bytes) EXPECT((bytes) <= MAXOBJBYTES, TRUE) -#else -#define SMALL_OBJ(bytes) /* bytes argument should have no side-effect */ \ - (EXPECT((bytes) <= MAXOBJBYTES - MAX_EXTRA_BYTES, TRUE) || (bytes) <= MAXOBJBYTES - EXTRA_BYTES) -/* This really just tests bytes <= MAXOBJBYTES - EXTRA_BYTES. */ -/* But we try to avoid looking up EXTRA_BYTES. */ -#endif - -/* - * Hash table representation of sets of pages. - * Implements a map from aligned HBLKSIZE chunks of the address space to one - * bit each. - * This assumes it is OK to spuriously set bits, e.g. because multiple - * addresses are represented by a single location. - * Used by black-listing code, and perhaps by dirty bit maintenance code. - */ - -#ifndef LOG_PHT_ENTRIES -#ifdef LARGE_CONFIG -#if CPP_WORDSZ == 32 -#define LOG_PHT_ENTRIES 20 /* Collisions likely at 1M blocks, */ - /* which is >= 4 GB. Each table takes */ - /* 128 KB, some of which may never be */ - /* touched. */ -#else -#define LOG_PHT_ENTRIES 21 /* Collisions likely at 2M blocks, */ -/* which is >= 8 GB. Each table takes */ -/* 256 KB, some of which may never be */ -/* touched. */ -#endif -#elif !defined(SMALL_CONFIG) -#define LOG_PHT_ENTRIES 18 /* Collisions are likely if heap grows */ -/* to more than 256K hblks >= 1 GB. */ -/* Each hash table occupies 32 KB. */ -/* Even for somewhat smaller heaps, */ -/* say half that, collisions may be an */ -/* issue because we blacklist */ -/* addresses outside the heap. */ -#else -#define LOG_PHT_ENTRIES 15 /* Collisions are likely if heap grows */ -/* to more than 32K hblks (128 MB). */ -/* Each hash table occupies 4 KB. */ -#endif -#endif /* !LOG_PHT_ENTRIES */ - -#define PHT_ENTRIES ((word)1 << LOG_PHT_ENTRIES) -#define PHT_SIZE (PHT_ENTRIES >> LOGWL) -typedef word page_hash_table[PHT_SIZE]; - -#define PHT_HASH(addr) ((((word)(addr)) >> LOG_HBLKSIZE) & (PHT_ENTRIES - 1)) - -#define get_pht_entry_from_index(bl, index) \ - (((bl)[divWORDSZ(index)] >> modWORDSZ(index)) & 1) -#define set_pht_entry_from_index(bl, index) \ - (void)((bl)[divWORDSZ(index)] |= (word)1 << modWORDSZ(index)) - -#if defined(THREADS) && defined(AO_HAVE_or) -/* And, one more version for GC_add_to_black_list_normal/stack */ -/* (invoked indirectly by GC_do_local_mark) and */ -/* async_set_pht_entry_from_index (invoked by GC_dirty or the write */ -/* fault handler). */ -#define set_pht_entry_from_index_concurrent(bl, index) \ - AO_or((volatile AO_t *)&(bl)[divWORDSZ(index)], \ - (AO_t)((word)1 << modWORDSZ(index))) -#else -#define set_pht_entry_from_index_concurrent(bl, index) \ - set_pht_entry_from_index(bl, index) -#endif - -/********************************************/ -/* */ -/* H e a p B l o c k s */ -/* */ -/********************************************/ - -#define MARK_BITS_PER_HBLK (HBLKSIZE / GRANULE_BYTES) -/* upper bound */ -/* We allocate 1 bit per allocation granule. */ -/* If MARK_BIT_PER_GRANULE is defined, we use */ -/* every nth bit, where n is the number of */ -/* allocation granules per object. If */ -/* MARK_BIT_PER_OBJ is defined, we only use the */ -/* initial group of mark bits, and it is safe */ -/* to allocate smaller header for large objects. */ - -union word_ptr_ao_u { - word w; - signed_word sw; - void *vp; -#ifdef PARALLEL_MARK - volatile AO_t ao; -#endif -}; - -/* We maintain layout maps for heap blocks containing objects of a given */ -/* size. Each entry in this map describes a byte offset and has the */ -/* following type. */ -struct hblkhdr { - struct hblk *hb_next; /* Link field for hblk free list */ - /* and for lists of chunks waiting to be */ - /* reclaimed. */ - struct hblk *hb_prev; /* Backwards link for free list. */ - struct hblk *hb_block; /* The corresponding block. */ - unsigned char hb_obj_kind; - /* Kind of objects in the block. Each kind */ - /* identifies a mark procedure and a set of */ - /* list headers. Sometimes called regions. */ - unsigned char hb_flags; -#define IGNORE_OFF_PAGE 1 /* Ignore pointers that do not */ - /* point to the first hblk of */ - /* this object. */ -#define WAS_UNMAPPED 2 /* This is a free block, which has */ - /* been unmapped from the address */ - /* space. */ - /* GC_remap must be invoked on it */ - /* before it can be reallocated. */ - /* Only set with USE_MUNMAP. */ -#define FREE_BLK 4 /* Block is free, i.e. not in use. */ -#ifdef ENABLE_DISCLAIM -#define HAS_DISCLAIM 8 - /* This kind has a callback on reclaim. */ -#define MARK_UNCONDITIONALLY 0x10 - /* Mark from all objects, marked or */ - /* not. Used to mark objects needed by */ - /* reclaim notifier. */ -#endif -#ifdef MARK_BIT_PER_GRANULE -#define LARGE_BLOCK 0x20 -#endif - unsigned short hb_last_reclaimed; - /* Value of GC_gc_no when block was */ - /* last allocated or swept. May wrap. */ - /* For a free block, this is maintained */ - /* only for USE_MUNMAP, and indicates */ - /* when the header was allocated, or */ - /* when the size of the block last */ - /* changed. */ -#ifdef MARK_BIT_PER_OBJ - unsigned32 hb_inv_sz; /* A good upper bound for 2**32/hb_sz. */ - /* For large objects, we use */ - /* LARGE_INV_SZ. */ -#define LARGE_INV_SZ (1 << 16) -#endif - word hb_sz; /* If in use, size in bytes, of objects in the block. */ - /* if free, the size in bytes of the whole block. */ - /* We assume that this is convertible to signed_word */ - /* without generating a negative result. We avoid */ - /* generating free blocks larger than that. */ - word hb_descr; /* object descriptor for marking. See */ - /* gc_mark.h. */ -#ifdef MARK_BIT_PER_GRANULE - unsigned short *hb_map; /* Essentially a table of remainders */ - /* mod BYTES_TO_GRANULES(hb_sz), except */ - /* for large blocks. See GC_obj_map. */ -#endif -#ifdef PARALLEL_MARK - volatile AO_t hb_n_marks; /* Number of set mark bits, excluding */ - /* the one always set at the end. */ - /* Currently it is concurrently */ - /* updated and hence only approximate. */ - /* But a zero value does guarantee that */ - /* the block contains no marked */ - /* objects. */ - /* Ensuring this property means that we */ - /* never decrement it to zero during a */ - /* collection, and hence the count may */ - /* be one too high. Due to concurrent */ - /* updates, an arbitrary number of */ - /* increments, but not all of them (!) */ - /* may be lost, hence it may in theory */ - /* be much too low. */ - /* The count may also be too high if */ - /* multiple mark threads mark the */ - /* same object due to a race. */ -#else - size_t hb_n_marks; /* Without parallel marking, the count */ - /* is accurate. */ -#endif -#ifdef USE_MARK_BYTES -#define MARK_BITS_SZ (MARK_BITS_PER_HBLK + 1) - /* Unlike the other case, this is in units of bytes. */ - /* Since we force double-word alignment, we need at most one */ - /* mark bit per 2 words. But we do allocate and set one */ - /* extra mark bit to avoid an explicit check for the */ - /* partial object at the end of each block. */ - union { - char _hb_marks[MARK_BITS_SZ]; - /* The i'th byte is 1 if the object */ - /* starting at granule i or object i is */ - /* marked, 0 otherwise. */ - /* The mark bit for the "one past the end" */ - /* object is always set to avoid a special */ - /* case test in the marker. */ - word dummy; /* Force word alignment of mark bytes. */ - } _mark_byte_union; -#define hb_marks _mark_byte_union._hb_marks -#else -#define MARK_BITS_SZ (MARK_BITS_PER_HBLK / CPP_WORDSZ + 1) - word hb_marks[MARK_BITS_SZ]; -#endif /* !USE_MARK_BYTES */ -}; - -#define ANY_INDEX 23 /* "Random" mark bit index for assertions */ - -/* heap block body */ - -#define HBLK_WORDS (HBLKSIZE / sizeof(word)) -#define HBLK_GRANULES (HBLKSIZE / GRANULE_BYTES) - -/* The number of objects in a block dedicated to a certain size. */ -/* may erroneously yield zero (instead of one) for large objects. */ -#define HBLK_OBJS(sz_in_bytes) (HBLKSIZE / (sz_in_bytes)) - -struct hblk { - char hb_body[HBLKSIZE]; -}; - -#define HBLK_IS_FREE(hdr) (((hdr)->hb_flags & FREE_BLK) != 0) - -#define OBJ_SZ_TO_BLOCKS(lb) divHBLKSZ((lb) + HBLKSIZE - 1) -#define OBJ_SZ_TO_BLOCKS_CHECKED(lb) /* lb should have no side-effect */ \ - divHBLKSZ(SIZET_SAT_ADD(lb, HBLKSIZE - 1)) -/* Size of block (in units of HBLKSIZE) needed to hold objects of */ -/* given lb (in bytes). The checked variant prevents wrap around. */ - -/* Object free list link */ -#define obj_link(p) (*(void **)(p)) - -#define LOG_MAX_MARK_PROCS 6 -#define MAX_MARK_PROCS (1 << LOG_MAX_MARK_PROCS) - -/* Root sets. Logically private to mark_rts.c. But we don't want the */ -/* tables scanned, so we put them here. */ -/* MAX_ROOT_SETS is the maximum number of ranges that can be */ -/* registered as static roots. */ -#define MAX_ROOT_SETS 8192 - -#define MAX_EXCLUSIONS (MAX_ROOT_SETS / 4) -/* Maximum number of segments that can be excluded from root sets. */ - -/* - * Data structure for excluded static roots. - */ -struct exclusion { - ptr_t e_start; - ptr_t e_end; -}; - -/* Data structure for list of root sets. */ -/* We keep a hash table, so that we can filter out duplicate additions. */ -/* Under Win32, we need to do a better job of filtering overlaps, so */ -/* we resort to sequential search, and pay the price. */ -struct roots { - ptr_t r_start; /* multiple of word size */ - ptr_t r_end; /* multiple of word size and greater than r_start */ -#if !defined(MSWIN32) && !defined(MSWINCE) && !defined(CYGWIN32) - struct roots *r_next; -#endif - GC_bool r_tmp; - /* Delete before registering new dynamic libraries */ -}; - -#if !defined(MSWIN32) && !defined(MSWINCE) && !defined(CYGWIN32) -/* Size of hash table index to roots. */ -#define LOG_RT_SIZE 6 -#define RT_SIZE (1 << LOG_RT_SIZE) /* Power of 2, may be != MAX_ROOT_SETS */ -#endif - -#if (!defined(MAX_HEAP_SECTS) || defined(CPPCHECK)) && (defined(CYGWIN32) || defined(MSWIN32) || defined(MSWINCE) || defined(USE_PROC_FOR_LIBRARIES)) -#ifdef LARGE_CONFIG -#if CPP_WORDSZ > 32 -#define MAX_HEAP_SECTS 81920 -#else -#define MAX_HEAP_SECTS 7680 -#endif -#elif defined(SMALL_CONFIG) && !defined(USE_PROC_FOR_LIBRARIES) -#if defined(PARALLEL_MARK) && (defined(MSWIN32) || defined(CYGWIN32)) -#define MAX_HEAP_SECTS 384 -#else -#define MAX_HEAP_SECTS 128 /* Roughly 256 MB (128*2048*1024) */ -#endif -#elif CPP_WORDSZ > 32 -#define MAX_HEAP_SECTS 1024 /* Roughly 8 GB */ -#else -#define MAX_HEAP_SECTS 512 /* Roughly 4 GB */ -#endif -#endif /* !MAX_HEAP_SECTS */ - -typedef struct GC_ms_entry { - ptr_t mse_start; /* First word of object, word aligned. */ - union word_ptr_ao_u mse_descr; - /* Descriptor; low order two bits are tags, */ - /* as described in gc_mark.h. */ -} mse; - -typedef int mark_state_t; /* Current state of marking. */ - /* Used to remember where we are during */ - /* concurrent marking. */ - -struct disappearing_link; -struct finalizable_object; - -struct dl_hashtbl_s { - struct disappearing_link **head; - word entries; - unsigned log_size; -}; - -struct fnlz_roots_s { - struct finalizable_object **fo_head; - /* List of objects that should be finalized now: */ - struct finalizable_object *finalize_now; -}; - -union toggle_ref_u { - /* The lowest bit is used to distinguish between choices. */ - void *strong_ref; - GC_hidden_pointer weak_ref; -}; - -/* Extended descriptors. GC_typed_mark_proc understands these. */ -/* These are used for simple objects that are larger than what */ -/* can be described by a BITMAP_BITS sized bitmap. */ -typedef struct { - word ed_bitmap; /* lsb corresponds to first word. */ - GC_bool ed_continued; /* next entry is continuation. */ -} typed_ext_descr_t; - -struct HeapSect { - ptr_t hs_start; - size_t hs_bytes; -}; - -/* Lists of all heap blocks and free lists */ -/* as well as other random data structures */ -/* that should not be scanned by the */ -/* collector. */ -/* These are grouped together in a struct */ -/* so that they can be easily skipped by the */ -/* GC_mark routine. */ -/* The ordering is weird to make GC_malloc */ -/* faster by keeping the important fields */ -/* sufficiently close together that a */ -/* single load of a base register will do. */ -/* Scalars that could easily appear to */ -/* be pointers are also put here. */ -/* The main fields should precede any */ -/* conditionally included fields, so that */ -/* gc_inline.h will work even if a different */ -/* set of macros is defined when the client is */ -/* compiled. */ - -struct _GC_arrays { - word _heapsize; /* Heap size in bytes (value never goes down). */ - word _requested_heapsize; /* Heap size due to explicit expansion. */ -#define GC_heapsize_on_gc_disable GC_arrays._heapsize_on_gc_disable - word _heapsize_on_gc_disable; - ptr_t _last_heap_addr; - word _large_free_bytes; - /* Total bytes contained in blocks on large object free */ - /* list. */ - word _large_allocd_bytes; - /* Total number of bytes in allocated large objects blocks. */ - /* For the purposes of this counter and the next one only, a */ - /* large object is one that occupies a block of at least */ - /* 2*HBLKSIZE. */ - word _max_large_allocd_bytes; - /* Maximum number of bytes that were ever allocated in */ - /* large object blocks. This is used to help decide when it */ - /* is safe to split up a large block. */ - word _bytes_allocd_before_gc; - /* Number of bytes allocated before this */ - /* collection cycle. */ -#define GC_our_mem_bytes GC_arrays._our_mem_bytes - word _our_mem_bytes; -#ifndef SEPARATE_GLOBALS -#define GC_bytes_allocd GC_arrays._bytes_allocd - word _bytes_allocd; - /* Number of bytes allocated during this collection cycle. */ -#endif - word _bytes_dropped; - /* Number of black-listed bytes dropped during GC cycle */ - /* as a result of repeated scanning during allocation */ - /* attempts. These are treated largely as allocated, */ - /* even though they are not useful to the client. */ - word _bytes_finalized; - /* Approximate number of bytes in objects (and headers) */ - /* that became ready for finalization in the last */ - /* collection. */ - word _bytes_freed; - /* Number of explicitly deallocated bytes of memory */ - /* since last collection. */ - word _finalizer_bytes_freed; - /* Bytes of memory explicitly deallocated while */ - /* finalizers were running. Used to approximate memory */ - /* explicitly deallocated by finalizers. */ - bottom_index *_all_bottom_indices; - /* Pointer to the first (lowest address) bottom_index; */ - /* assumes the lock is held. */ - bottom_index *_all_bottom_indices_end; - /* Pointer to the last (highest address) bottom_index; */ - /* assumes the lock is held. */ - ptr_t _scratch_free_ptr; - hdr *_hdr_free_list; - ptr_t _scratch_end_ptr; - /* GC_scratch_end_ptr is end point of the current scratch area. */ -#if defined(IRIX5) || (defined(USE_PROC_FOR_LIBRARIES) && !defined(LINUX)) -#define USE_SCRATCH_LAST_END_PTR -#define GC_scratch_last_end_ptr GC_arrays._scratch_last_end_ptr - ptr_t _scratch_last_end_ptr; - /* GC_scratch_last_end_ptr is the end point of the last */ - /* obtained scratch area. */ - /* Used by GC_register_dynamic_libraries(). */ -#endif - mse *_mark_stack; - /* Limits of stack for GC_mark routine. All ranges */ - /* between GC_mark_stack (incl.) and GC_mark_stack_top */ - /* (incl.) still need to be marked from. */ - mse *_mark_stack_limit; -#ifdef PARALLEL_MARK - mse *volatile _mark_stack_top; - /* Updated only with mark lock held, but read asynchronously. */ - /* TODO: Use union to avoid casts to AO_t */ -#else - mse *_mark_stack_top; -#endif - word _composite_in_use; /* Number of bytes in the accessible */ - /* composite objects. */ - word _atomic_in_use; /* Number of bytes in the accessible */ - /* atomic objects. */ -#define GC_last_heap_growth_gc_no GC_arrays._last_heap_growth_gc_no - word _last_heap_growth_gc_no; - /* GC number of latest successful GC_expand_hp_inner call */ -#ifdef USE_MUNMAP -#define GC_unmapped_bytes GC_arrays._unmapped_bytes - word _unmapped_bytes; -#ifdef COUNT_UNMAPPED_REGIONS -#define GC_num_unmapped_regions GC_arrays._num_unmapped_regions - signed_word _num_unmapped_regions; -#endif -#else -#define GC_unmapped_bytes 0 -#endif - bottom_index *_all_nils; -#define GC_scan_ptr GC_arrays._scan_ptr - struct hblk *_scan_ptr; -#ifdef PARALLEL_MARK -#define GC_main_local_mark_stack GC_arrays._main_local_mark_stack - mse *_main_local_mark_stack; -#define GC_first_nonempty GC_arrays._first_nonempty - volatile AO_t _first_nonempty; - /* Lowest entry on mark stack that may be */ - /* nonempty. Updated only by initiating thread. */ -#endif -#define GC_mark_stack_size GC_arrays._mark_stack_size - size_t _mark_stack_size; -#define GC_mark_state GC_arrays._mark_state - mark_state_t _mark_state; /* Initialized to MS_NONE (0). */ -#define GC_mark_stack_too_small GC_arrays._mark_stack_too_small - GC_bool _mark_stack_too_small; - /* We need a larger mark stack. May be set by */ - /* client supplied mark routines. */ -#define GC_objects_are_marked GC_arrays._objects_are_marked - GC_bool _objects_are_marked; - /* Are there collectible marked objects in the heap? */ -#ifdef ENABLE_TRACE -#define GC_trace_addr GC_arrays._trace_addr - ptr_t _trace_addr; -#endif -#define GC_capacity_heap_sects GC_arrays._capacity_heap_sects - size_t _capacity_heap_sects; -#define GC_n_heap_sects GC_arrays._n_heap_sects - word _n_heap_sects; /* Number of separately added heap sections. */ -#if defined(MSWIN32) || defined(MSWINCE) || defined(CYGWIN32) -#define GC_n_heap_bases GC_arrays._n_heap_bases - word _n_heap_bases; /* See GC_heap_bases. */ -#endif -#ifdef USE_PROC_FOR_LIBRARIES -#define GC_n_memory GC_arrays._n_memory - word _n_memory; /* Number of GET_MEM allocated memory sections. */ -#endif -#ifdef GC_GCJ_SUPPORT -#define GC_gcjobjfreelist GC_arrays._gcjobjfreelist - ptr_t *_gcjobjfreelist; -#endif -#define GC_fo_entries GC_arrays._fo_entries - word _fo_entries; -#ifndef GC_NO_FINALIZATION -#define GC_dl_hashtbl GC_arrays._dl_hashtbl -#define GC_fnlz_roots GC_arrays._fnlz_roots -#define GC_log_fo_table_size GC_arrays._log_fo_table_size -#ifndef GC_LONG_REFS_NOT_NEEDED -#define GC_ll_hashtbl GC_arrays._ll_hashtbl - struct dl_hashtbl_s _ll_hashtbl; -#endif - struct dl_hashtbl_s _dl_hashtbl; - struct fnlz_roots_s _fnlz_roots; - unsigned _log_fo_table_size; -#ifndef GC_TOGGLE_REFS_NOT_NEEDED -#define GC_toggleref_arr GC_arrays._toggleref_arr -#define GC_toggleref_array_size GC_arrays._toggleref_array_size -#define GC_toggleref_array_capacity GC_arrays._toggleref_array_capacity - union toggle_ref_u *_toggleref_arr; - size_t _toggleref_array_size; - size_t _toggleref_array_capacity; -#endif -#endif -#ifdef TRACE_BUF -#define GC_trace_buf_ptr GC_arrays._trace_buf_ptr - int _trace_buf_ptr; -#endif -#ifdef ENABLE_DISCLAIM -#define GC_finalized_kind GC_arrays._finalized_kind - int _finalized_kind; -#endif -#define n_root_sets GC_arrays._n_root_sets -#define GC_excl_table_entries GC_arrays._excl_table_entries - int _n_root_sets; /* GC_static_roots[0..n_root_sets) contains the */ - /* valid root sets. */ - size_t _excl_table_entries; /* Number of entries in use. */ -#ifdef THREADS -#define GC_roots_were_cleared GC_arrays._roots_were_cleared - GC_bool _roots_were_cleared; -#endif -#define GC_explicit_typing_initialized GC_arrays._explicit_typing_initialized -#define GC_ed_size GC_arrays._ed_size -#define GC_avail_descr GC_arrays._avail_descr -#define GC_ext_descriptors GC_arrays._ext_descriptors -#ifdef AO_HAVE_load_acquire - volatile AO_t _explicit_typing_initialized; -#else - GC_bool _explicit_typing_initialized; -#endif - size_t _ed_size; /* Current size of above arrays. */ - size_t _avail_descr; /* Next available slot. */ - typed_ext_descr_t *_ext_descriptors; /* Points to array of extended */ - /* descriptors. */ - GC_mark_proc _mark_procs[MAX_MARK_PROCS]; - /* Table of user-defined mark procedures. There is */ - /* a small number of these, which can be referenced */ - /* by DS_PROC mark descriptors. See gc_mark.h. */ - char _modws_valid_offsets[sizeof(word)]; - /* GC_valid_offsets[i] ==> */ - /* GC_modws_valid_offsets[i%sizeof(word)] */ -#if !defined(MSWIN32) && !defined(MSWINCE) && !defined(CYGWIN32) -#define GC_root_index GC_arrays._root_index - struct roots *_root_index[RT_SIZE]; -#endif -#ifdef SAVE_CALL_CHAIN -#define GC_last_stack GC_arrays._last_stack - struct callinfo _last_stack[NFRAMES]; - /* Stack at last garbage collection. Useful for */ - /* debugging mysterious object disappearances. In the */ - /* multi-threaded case, we currently only save the */ - /* calling stack. */ -#endif -#ifndef SEPARATE_GLOBALS -#define GC_objfreelist GC_arrays._objfreelist - void *_objfreelist[MAXOBJGRANULES + 1]; - /* free list for objects */ -#define GC_aobjfreelist GC_arrays._aobjfreelist - void *_aobjfreelist[MAXOBJGRANULES + 1]; - /* free list for atomic objects */ -#endif - void *_uobjfreelist[MAXOBJGRANULES + 1]; - /* Uncollectible but traced objects. */ - /* Objects on this and _auobjfreelist */ - /* are always marked, except during */ - /* garbage collections. */ -#ifdef GC_ATOMIC_UNCOLLECTABLE -#define GC_auobjfreelist GC_arrays._auobjfreelist - void *_auobjfreelist[MAXOBJGRANULES + 1]; - /* Atomic uncollectible but traced objects. */ -#endif - size_t _size_map[MAXOBJBYTES + 1]; - /* Number of granules to allocate when asked for a certain */ - /* number of bytes (plus EXTRA_BYTES). Should be accessed with */ - /* the allocation lock held. */ -#ifdef MARK_BIT_PER_GRANULE -#define GC_obj_map GC_arrays._obj_map - unsigned short *_obj_map[MAXOBJGRANULES + 1]; - /* If not NULL, then a pointer to a map of valid */ - /* object addresses. */ - /* GC_obj_map[sz_in_granules][i] is */ - /* i % sz_in_granules. */ - /* This is now used purely to replace a */ - /* division in the marker by a table lookup. */ - /* _obj_map[0] is used for large objects and */ - /* contains all nonzero entries. This gets us */ - /* out of the marker fast path without an extra */ - /* test. */ -#define OBJ_MAP_LEN BYTES_TO_GRANULES(HBLKSIZE) -#endif -#define VALID_OFFSET_SZ HBLKSIZE - char _valid_offsets[VALID_OFFSET_SZ]; - /* GC_valid_offsets[i] == TRUE ==> i */ - /* is registered as a displacement. */ -#ifndef GC_DISABLE_INCREMENTAL -#define GC_grungy_pages GC_arrays._grungy_pages - page_hash_table _grungy_pages; /* Pages that were dirty at last */ - /* GC_read_dirty. */ -#define GC_dirty_pages GC_arrays._dirty_pages - volatile page_hash_table _dirty_pages; - /* Pages dirtied since last GC_read_dirty. */ -#endif -#if (defined(CHECKSUMS) && (defined(GWW_VDB) || defined(SOFT_VDB))) || defined(PROC_VDB) -#define GC_written_pages GC_arrays._written_pages - page_hash_table _written_pages; /* Pages ever dirtied */ -#endif -#define GC_heap_sects GC_arrays._heap_sects - struct HeapSect *_heap_sects; /* Heap segments potentially */ - /* client objects. */ -#if defined(USE_PROC_FOR_LIBRARIES) -#define GC_our_memory GC_arrays._our_memory - struct HeapSect _our_memory[MAX_HEAP_SECTS]; - /* All GET_MEM allocated */ - /* memory. Includes block */ - /* headers and the like. */ -#endif -#if defined(MSWIN32) || defined(MSWINCE) || defined(CYGWIN32) -#define GC_heap_bases GC_arrays._heap_bases - ptr_t _heap_bases[MAX_HEAP_SECTS]; - /* Start address of memory regions obtained from kernel. */ -#endif -#ifdef MSWINCE -#define GC_heap_lengths GC_arrays._heap_lengths - word _heap_lengths[MAX_HEAP_SECTS]; - /* Committed lengths of memory regions obtained from kernel. */ -#endif - struct roots _static_roots[MAX_ROOT_SETS]; - struct exclusion _excl_table[MAX_EXCLUSIONS]; - /* Block header index; see gc_headers.h */ - bottom_index *_top_index[TOP_SZ]; -}; - -GC_API_PRIV GC_FAR struct _GC_arrays GC_arrays; - -#define GC_all_nils GC_arrays._all_nils -#define GC_atomic_in_use GC_arrays._atomic_in_use -#define GC_bytes_allocd_before_gc GC_arrays._bytes_allocd_before_gc -#define GC_bytes_dropped GC_arrays._bytes_dropped -#define GC_bytes_finalized GC_arrays._bytes_finalized -#define GC_bytes_freed GC_arrays._bytes_freed -#define GC_composite_in_use GC_arrays._composite_in_use -#define GC_excl_table GC_arrays._excl_table -#define GC_finalizer_bytes_freed GC_arrays._finalizer_bytes_freed -#define GC_heapsize GC_arrays._heapsize -#define GC_large_allocd_bytes GC_arrays._large_allocd_bytes -#define GC_large_free_bytes GC_arrays._large_free_bytes -#define GC_last_heap_addr GC_arrays._last_heap_addr -#define GC_mark_stack GC_arrays._mark_stack -#define GC_mark_stack_limit GC_arrays._mark_stack_limit -#define GC_mark_stack_top GC_arrays._mark_stack_top -#define GC_mark_procs GC_arrays._mark_procs -#define GC_max_large_allocd_bytes GC_arrays._max_large_allocd_bytes -#define GC_modws_valid_offsets GC_arrays._modws_valid_offsets -#define GC_requested_heapsize GC_arrays._requested_heapsize -#define GC_all_bottom_indices GC_arrays._all_bottom_indices -#define GC_all_bottom_indices_end GC_arrays._all_bottom_indices_end -#define GC_scratch_free_ptr GC_arrays._scratch_free_ptr -#define GC_hdr_free_list GC_arrays._hdr_free_list -#define GC_scratch_end_ptr GC_arrays._scratch_end_ptr -#define GC_size_map GC_arrays._size_map -#define GC_static_roots GC_arrays._static_roots -#define GC_top_index GC_arrays._top_index -#define GC_uobjfreelist GC_arrays._uobjfreelist -#define GC_valid_offsets GC_arrays._valid_offsets - -#define beginGC_arrays ((ptr_t)(&GC_arrays)) -#define endGC_arrays (((ptr_t)(&GC_arrays)) + (sizeof GC_arrays)) - -/* Object kinds: */ -#ifndef MAXOBJKINDS -#define MAXOBJKINDS 16 -#endif -GC_EXTERN struct obj_kind { - void **ok_freelist; /* Array of free list headers for this kind of */ - /* object. Point either to GC_arrays or to */ - /* storage allocated with GC_scratch_alloc. */ - struct hblk **ok_reclaim_list; - /* List headers for lists of blocks waiting to */ - /* be swept. Indexed by object size in */ - /* granules. */ - word ok_descriptor; /* Descriptor template for objects in this */ - /* block. */ - GC_bool ok_relocate_descr; - /* Add object size in bytes to descriptor */ - /* template to obtain descriptor. Otherwise */ - /* template is used as is. */ - GC_bool ok_init; - /* Clear objects before putting them on the free list. */ -#ifdef ENABLE_DISCLAIM - GC_bool ok_mark_unconditionally; - /* Mark from all, including unmarked, objects */ - /* in block. Used to protect objects reachable */ - /* from reclaim notifiers. */ - int(GC_CALLBACK *ok_disclaim_proc)(void * /*obj*/); - /* The disclaim procedure is called before obj */ - /* is reclaimed, but must also tolerate being */ - /* called with object from freelist. Non-zero */ - /* exit prevents object from being reclaimed. */ -#define OK_DISCLAIM_INITZ /* comma */ , FALSE, 0 -#else -#define OK_DISCLAIM_INITZ /* empty */ -#endif /* !ENABLE_DISCLAIM */ -} GC_obj_kinds[MAXOBJKINDS]; - -#define beginGC_obj_kinds ((ptr_t)(&GC_obj_kinds)) -#define endGC_obj_kinds (beginGC_obj_kinds + (sizeof GC_obj_kinds)) - -/* Variables that used to be in GC_arrays, but need to be accessed by */ -/* inline allocation code. If they were in GC_arrays, the inlined */ -/* allocation code would include GC_arrays offsets (as it did), which */ -/* introduce maintenance problems. */ - -#ifdef SEPARATE_GLOBALS -extern word GC_bytes_allocd; -/* Number of bytes allocated during this collection cycle. */ -extern ptr_t GC_objfreelist[MAXOBJGRANULES + 1]; -/* free list for NORMAL objects */ -#define beginGC_objfreelist ((ptr_t)(&GC_objfreelist)) -#define endGC_objfreelist (beginGC_objfreelist + sizeof(GC_objfreelist)) - -extern ptr_t GC_aobjfreelist[MAXOBJGRANULES + 1]; -/* free list for atomic (PTRFREE) objects */ -#define beginGC_aobjfreelist ((ptr_t)(&GC_aobjfreelist)) -#define endGC_aobjfreelist (beginGC_aobjfreelist + sizeof(GC_aobjfreelist)) -#endif /* SEPARATE_GLOBALS */ - -/* Predefined kinds: */ -#define PTRFREE GC_I_PTRFREE -#define NORMAL GC_I_NORMAL -#define UNCOLLECTABLE 2 -#ifdef GC_ATOMIC_UNCOLLECTABLE -#define AUNCOLLECTABLE 3 -#define IS_UNCOLLECTABLE(k) (((k) & ~1) == UNCOLLECTABLE) -#define GC_N_KINDS_INITIAL_VALUE 4 -#else -#define IS_UNCOLLECTABLE(k) ((k) == UNCOLLECTABLE) -#define GC_N_KINDS_INITIAL_VALUE 3 -#endif - -GC_EXTERN unsigned GC_n_kinds; - -GC_EXTERN size_t GC_page_size; -/* May mean the allocation granularity size, not page size. */ - -#ifdef REAL_PAGESIZE_NEEDED -GC_EXTERN size_t GC_real_page_size; -#else -#define GC_real_page_size GC_page_size -#endif - -/* Round up allocation size to a multiple of a page size. */ -/* GC_setpagesize() is assumed to be already invoked. */ -#define ROUNDUP_PAGESIZE(lb) /* lb should have no side-effect */ \ - (SIZET_SAT_ADD(lb, GC_page_size - 1) & ~(GC_page_size - 1)) - -/* Same as above but used to make GET_MEM() argument safe. */ -#ifdef MMAP_SUPPORTED -#define ROUNDUP_PAGESIZE_IF_MMAP(lb) ROUNDUP_PAGESIZE(lb) -#else -#define ROUNDUP_PAGESIZE_IF_MMAP(lb) (lb) -#endif - -#if defined(MSWIN32) || defined(MSWINCE) || defined(CYGWIN32) -GC_EXTERN SYSTEM_INFO GC_sysinfo; -GC_INNER GC_bool GC_is_heap_base(const void *p); -#endif - -GC_EXTERN word GC_black_list_spacing; -/* Average number of bytes between blacklisted */ -/* blocks. Approximate. */ -/* Counts only blocks that are */ -/* "stack-blacklisted", i.e. that are */ -/* problematic in the interior of an object. */ - -#ifdef GC_GCJ_SUPPORT -extern struct hblk *GC_hblkfreelist[]; -extern word GC_free_bytes[]; /* Both remain visible to GNU GCJ. */ -#endif - -GC_EXTERN word GC_root_size; /* Total size of registered root sections. */ - -GC_EXTERN GC_bool GC_debugging_started; -/* GC_debug_malloc has been called. */ - -/* This is used by GC_do_blocking[_inner](). */ -struct blocking_data { - GC_fn_type fn; - void *client_data; /* and result */ -}; - -/* This is used by GC_call_with_gc_active(), GC_push_all_stack_sections(). */ -struct GC_traced_stack_sect_s { - ptr_t saved_stack_ptr; -#ifdef IA64 - ptr_t saved_backing_store_ptr; - ptr_t backing_store_end; -#endif - struct GC_traced_stack_sect_s *prev; -}; - -#ifdef THREADS -/* Process all "traced stack sections" - scan entire stack except for */ -/* frames belonging to the user functions invoked by GC_do_blocking. */ -GC_INNER void GC_push_all_stack_sections(ptr_t lo, ptr_t hi, - struct GC_traced_stack_sect_s *traced_stack_sect); -GC_EXTERN word GC_total_stacksize; /* updated on every push_all_stacks */ -#else -GC_EXTERN ptr_t GC_blocked_sp; -GC_EXTERN struct GC_traced_stack_sect_s *GC_traced_stack_sect; -/* Points to the "frame" data held in stack by */ -/* the innermost GC_call_with_gc_active(). */ -/* NULL if no such "frame" active. */ -#endif /* !THREADS */ - -#ifdef IA64 -GC_EXTERN ptr_t GC_register_stackbottom; -#endif - -#if defined(E2K) || defined(IA64) -/* Similar to GC_push_all_stack_sections() but for IA-64 registers store. */ -GC_INNER void GC_push_all_register_sections(ptr_t bs_lo, ptr_t bs_hi, - int eager, struct GC_traced_stack_sect_s *traced_stack_sect); -#endif - -/* Marks are in a reserved area in */ -/* each heap block. Each word has one mark bit associated */ -/* with it. Only those corresponding to the beginning of an */ -/* object are used. */ - -/* Mark bit operations */ - -/* - * Retrieve, set, clear the nth mark bit in a given heap block. - * - * (Recall that bit n corresponds to nth object or allocation granule - * relative to the beginning of the block, including unused words) - */ - -#ifdef USE_MARK_BYTES -#define mark_bit_from_hdr(hhdr, n) ((hhdr)->hb_marks[n]) -#define set_mark_bit_from_hdr(hhdr, n) ((hhdr)->hb_marks[n] = 1) -#define clear_mark_bit_from_hdr(hhdr, n) ((hhdr)->hb_marks[n] = 0) -#else -/* Set mark bit correctly, even if mark bits may be concurrently */ -/* accessed. */ -#if defined(PARALLEL_MARK) || (defined(THREAD_SANITIZER) && defined(THREADS)) -/* Workaround TSan false positive: there is no race between */ -/* mark_bit_from_hdr and set_mark_bit_from_hdr when n is different */ -/* (alternatively, USE_MARK_BYTES could be used). If TSan is off, */ -/* AO_or() is used only if we set USE_MARK_BITS explicitly. */ -#define OR_WORD(addr, bits) AO_or((volatile AO_t *)(addr), (AO_t)(bits)) -#else -#define OR_WORD(addr, bits) (void)(*(addr) |= (bits)) -#endif -#define mark_bit_from_hdr(hhdr, n) \ - (((hhdr)->hb_marks[divWORDSZ(n)] >> modWORDSZ(n)) & (word)1) -#define set_mark_bit_from_hdr(hhdr, n) \ - OR_WORD((hhdr)->hb_marks + divWORDSZ(n), (word)1 << modWORDSZ(n)) -#define clear_mark_bit_from_hdr(hhdr, n) \ - ((hhdr)->hb_marks[divWORDSZ(n)] &= ~((word)1 << modWORDSZ(n))) -#endif /* !USE_MARK_BYTES */ - -#ifdef MARK_BIT_PER_OBJ -#define MARK_BIT_NO(offset, sz) (((word)(offset)) / (sz)) -/* Get the mark bit index corresponding to the given byte */ -/* offset and size (in bytes). */ -#define MARK_BIT_OFFSET(sz) 1 -/* Spacing between useful mark bits. */ -#define IF_PER_OBJ(x) x -#define FINAL_MARK_BIT(sz) ((sz) > MAXOBJBYTES ? 1 : HBLK_OBJS(sz)) -/* Position of final, always set, mark bit. */ -#else /* MARK_BIT_PER_GRANULE */ -#define MARK_BIT_NO(offset, sz) BYTES_TO_GRANULES((word)(offset)) -#define MARK_BIT_OFFSET(sz) BYTES_TO_GRANULES(sz) -#define IF_PER_OBJ(x) -#define FINAL_MARK_BIT(sz) \ - ((sz) > MAXOBJBYTES ? MARK_BITS_PER_HBLK \ - : BYTES_TO_GRANULES((sz)*HBLK_OBJS(sz))) -#endif - -/* Important internal collector routines */ - -GC_INNER ptr_t GC_approx_sp(void); - -GC_INNER GC_bool GC_should_collect(void); - -GC_INNER struct hblk *GC_next_block(struct hblk *h, GC_bool allow_free); -/* Get the next block whose address is at least */ -/* h. Returned block is managed by GC. The */ -/* block must be in use unless allow_free is */ -/* true. Return 0 if there is no such block. */ -GC_INNER struct hblk *GC_prev_block(struct hblk *h); -/* Get the last (highest address) block whose */ -/* address is at most h. Returned block is */ -/* managed by GC, but may or may not be in use. */ -/* Return 0 if there is no such block. */ -GC_INNER void GC_mark_init(void); -GC_INNER void GC_clear_marks(void); -/* Clear mark bits for all heap objects. */ -GC_INNER void GC_invalidate_mark_state(void); -/* Tell the marker that marked */ -/* objects may point to unmarked */ -/* ones, and roots may point to */ -/* unmarked objects. Reset mark stack. */ -GC_INNER GC_bool GC_mark_some(ptr_t cold_gc_frame); -/* Perform about one pages worth of marking */ -/* work of whatever kind is needed. Returns */ -/* quickly if no collection is in progress. */ -/* Return TRUE if mark phase finished. */ -GC_INNER void GC_initiate_gc(void); -/* initiate collection. */ -/* If the mark state is invalid, this */ -/* becomes full collection. Otherwise */ -/* it's partial. */ - -GC_INNER GC_bool GC_collection_in_progress(void); -/* Collection is in progress, or was abandoned. */ - -#define GC_PUSH_ALL_SYM(sym) \ - GC_push_all((/* no volatile */ void *)&(sym), \ - (/* no volatile */ void *)(&(sym) + 1)) - -GC_INNER void GC_push_all_stack(ptr_t b, ptr_t t); -/* As GC_push_all but consider */ -/* interior pointers as valid. */ - -#ifdef NO_VDB_FOR_STATIC_ROOTS -#define GC_push_conditional_static(b, t, all) \ - ((void)(all), GC_push_all(b, t)) -#else -/* Same as GC_push_conditional (does either of GC_push_all or */ -/* GC_push_selected depending on the third argument) but the caller */ -/* guarantees the region belongs to the registered static roots. */ -GC_INNER void GC_push_conditional_static(void *b, void *t, GC_bool all); -#endif - -#if defined(WRAP_MARK_SOME) && defined(PARALLEL_MARK) -/* GC_mark_local does not handle memory protection faults yet. So, */ -/* the static data regions are scanned immediately by GC_push_roots. */ -GC_INNER void GC_push_conditional_eager(void *bottom, void *top, - GC_bool all); -#endif - -/* In the threads case, we push part of the current thread stack */ -/* with GC_push_all_eager when we push the registers. This gets the */ -/* callee-save registers that may disappear. The remainder of the */ -/* stacks are scheduled for scanning in *GC_push_other_roots, which */ -/* is thread-package-specific. */ - -GC_INNER void GC_push_roots(GC_bool all, ptr_t cold_gc_frame); -/* Push all or dirty roots. */ - -GC_API_PRIV GC_push_other_roots_proc GC_push_other_roots; -/* Push system or application specific roots */ -/* onto the mark stack. In some environments */ -/* (e.g. threads environments) this is */ -/* predefined to be non-zero. A client */ -/* supplied replacement should also call the */ -/* original function. Remains externally */ -/* visible as used by some well-known 3rd-party */ -/* software (e.g., ECL) currently. */ - -#ifdef THREADS -void GC_push_thread_structures(void); -#endif -GC_EXTERN void (*GC_push_typed_structures)(void); -/* A pointer such that we can avoid linking in */ -/* the typed allocation support if unused. */ - -GC_INNER void GC_with_callee_saves_pushed(void (*fn)(ptr_t, void *), - volatile ptr_t arg); - -#if defined(E2K) || defined(IA64) || defined(SPARC) -/* Cause all stacked registers to be saved in memory. Return a */ -/* pointer to the top of the corresponding memory stack. */ -ptr_t GC_save_regs_in_stack(void); -#endif - -#ifdef E2K -/* Copy the full procedure stack to the provided buffer (with the */ -/* given capacity). Returns either the required buffer size if it */ -/* is bigger than capacity, otherwise the amount of copied bytes. */ -/* May be called from a signal handler. */ -GC_INNER size_t GC_get_procedure_stack(ptr_t, size_t); - -#if defined(CPPCHECK) -#define PS_ALLOCA_BUF(sz) NULL -#define ALLOCA_SAFE_LIMIT 0 -#else -#define PS_ALLOCA_BUF(sz) alloca(sz) /* cannot return NULL */ -#ifndef ALLOCA_SAFE_LIMIT -#define ALLOCA_SAFE_LIMIT (HBLKSIZE * 256) -#endif -#endif /* !CPPCHECK */ - -/* Copy procedure (register) stack to a stack-allocated or */ -/* memory-mapped buffer. Usable from a signal handler. */ -/* FREE_PROCEDURE_STACK_LOCAL() must be called with the same */ -/* *pbuf and *psz values before the caller function returns */ -/* (thus, the buffer is valid only within the function). */ -#define GET_PROCEDURE_STACK_LOCAL(pbuf, psz) \ - do { \ - size_t capacity = 0; \ - GC_ASSERT(GC_page_size != 0); \ - for (*(pbuf) = NULL;; capacity = *(psz)) { \ - *(psz) = GC_get_procedure_stack(*(pbuf), capacity); \ - if (*(psz) <= capacity) break; \ - if (*(psz) > ALLOCA_SAFE_LIMIT || EXPECT(capacity != 0, FALSE)) { \ - /* Deallocate old buffer if any. */ \ - if (EXPECT(capacity > ALLOCA_SAFE_LIMIT, FALSE)) \ - GC_unmap_procedure_stack_buf(*(pbuf), capacity); \ - *(psz) = ROUNDUP_PAGESIZE(*(psz)); \ - *(pbuf) = GC_mmap_procedure_stack_buf(*(psz)); \ - } else { \ - /* Allocate buffer on the stack if not large. */ \ - *(pbuf) = PS_ALLOCA_BUF(*(psz)); \ - } \ - } \ - if (capacity > ALLOCA_SAFE_LIMIT && EXPECT(((capacity - *(psz)) & ~(GC_page_size - 1)) != 0, FALSE)) { \ - /* Ensure sz value passed to munmap() later */ \ - /* matches that passed to mmap() above. */ \ - *(psz) = capacity - (GC_page_size - 1); \ - } \ - } while (0) - -/* Indicate that the buffer with copied procedure stack is not needed. */ -#define FREE_PROCEDURE_STACK_LOCAL(buf, sz) \ - (void)((sz) > ALLOCA_SAFE_LIMIT \ - ? (GC_unmap_procedure_stack_buf(buf, sz), 0) \ - : 0) - -GC_INNER ptr_t GC_mmap_procedure_stack_buf(size_t); -GC_INNER void GC_unmap_procedure_stack_buf(ptr_t, size_t); - -#ifdef THREADS -/* Allocate a buffer in the GC heap (as an atomic object) and copy */ -/* procedure stack there. Assumes the GC allocation lock is held. */ -/* May trigger a collection (thus, cannot be used in GC_push_roots */ -/* or in a signal handler). The buffer should be freed with */ -/* GC_INTERNAL_FREE later when not needed (or, alternatively, it */ -/* could be just garbage-collected). */ -/* Similar to GET_PROCEDURE_STACK_LOCAL in other aspects. */ -GC_INNER size_t GC_alloc_and_get_procedure_stack(ptr_t *pbuf); -#endif -#endif /* E2K */ - -#if defined(E2K) && defined(USE_PTR_HWTAG) -/* Load value and get tag of the target memory. */ -#if defined(__ptr64__) -#define LOAD_TAGGED_VALUE(v, tag, p) \ - do { \ - word val; \ - __asm__ __volatile__( \ - "ldd, sm %[adr], 0x0, %[val]\n\t" \ - "gettagd %[val], %[tag]\n" \ - : [val] "=r"(val), \ - [tag] "=r"(tag) \ - : [adr] "r"(p)); \ - v = val; \ - } while (0) -#elif !defined(CPPCHECK) -#error Unsupported -march for e2k target -#endif - -#define LOAD_WORD_OR_CONTINUE(v, p) \ - { \ - int tag LOCAL_VAR_INIT_OK; \ - LOAD_TAGGED_VALUE(v, tag, p); \ - if (tag != 0) continue; \ - } -#else -#define LOAD_WORD_OR_CONTINUE(v, p) (void)(v = *(word *)(p)) -#endif /* !E2K */ - -#if defined(AMIGA) || defined(MACOS) || defined(GC_DARWIN_THREADS) -void GC_push_one(word p); -/* If p points to an object, mark it */ -/* and push contents on the mark stack */ -/* Pointer recognition test always */ -/* accepts interior pointers, i.e. this */ -/* is appropriate for pointers found on */ -/* stack. */ -#endif - -#ifdef GC_WIN32_THREADS -/* Same as GC_push_one but for a sequence of registers. */ -GC_INNER void GC_push_many_regs(const word *regs, unsigned count); -#endif - -#if defined(PRINT_BLACK_LIST) || defined(KEEP_BACK_PTRS) -GC_INNER void GC_mark_and_push_stack(ptr_t p, ptr_t source); -/* Ditto, omits plausibility test */ -#else -GC_INNER void GC_mark_and_push_stack(ptr_t p); -#endif - -GC_INNER void GC_clear_hdr_marks(hdr *hhdr); -/* Clear the mark bits in a header */ -GC_INNER void GC_set_hdr_marks(hdr *hhdr); -/* Set the mark bits in a header */ -GC_INNER void GC_set_fl_marks(ptr_t p); -/* Set all mark bits associated with */ -/* a free list. */ -#if defined(GC_ASSERTIONS) && defined(THREAD_LOCAL_ALLOC) -void GC_check_fl_marks(void **); -/* Check that all mark bits */ -/* associated with a free list are */ -/* set. Abort if not. */ -#endif -void GC_add_roots_inner(ptr_t b, ptr_t e, GC_bool tmp); -#ifdef USE_PROC_FOR_LIBRARIES -GC_INNER void GC_remove_roots_subregion(ptr_t b, ptr_t e); -#endif -GC_INNER void GC_exclude_static_roots_inner(void *start, void *finish); -#if defined(DYNAMIC_LOADING) || defined(MSWIN32) || defined(MSWINCE) || defined(CYGWIN32) || defined(PCR) -GC_INNER void GC_register_dynamic_libraries(void); -/* Add dynamic library data sections to the root set. */ -#endif -GC_INNER void GC_cond_register_dynamic_libraries(void); -/* Remove and reregister dynamic libraries if we're */ -/* configured to do that at each GC. */ - -/* Machine dependent startup routines */ -ptr_t GC_get_main_stack_base(void); /* Cold end of stack. */ -#ifdef IA64 -GC_INNER ptr_t GC_get_register_stack_base(void); -/* Cold end of register stack. */ -#endif - -void GC_register_data_segments(void); - -#ifdef THREADS -/* Both are invoked from GC_init only. */ -GC_INNER void GC_thr_init(void); -GC_INNER void GC_init_parallel(void); -#else -GC_INNER GC_bool GC_is_static_root(void *p); -/* Is the address p in one of the registered static */ -/* root sections? */ -#ifdef TRACE_BUF -void GC_add_trace_entry(char *kind, word arg1, word arg2); -#endif -#endif /* !THREADS */ - -/* Black listing: */ -#ifdef PRINT_BLACK_LIST -GC_INNER void GC_add_to_black_list_normal(word p, ptr_t source); -/* Register bits as a possible future false */ -/* reference from the heap or static data */ -#define GC_ADD_TO_BLACK_LIST_NORMAL(bits, source) \ - if (GC_all_interior_pointers) { \ - GC_add_to_black_list_stack((word)(bits), (source)); \ - } else \ - GC_add_to_black_list_normal((word)(bits), (source)) -GC_INNER void GC_add_to_black_list_stack(word p, ptr_t source); -#define GC_ADD_TO_BLACK_LIST_STACK(bits, source) \ - GC_add_to_black_list_stack((word)(bits), (source)) -#else -GC_INNER void GC_add_to_black_list_normal(word p); -#define GC_ADD_TO_BLACK_LIST_NORMAL(bits, source) \ - if (GC_all_interior_pointers) { \ - GC_add_to_black_list_stack((word)(bits)); \ - } else \ - GC_add_to_black_list_normal((word)(bits)) -GC_INNER void GC_add_to_black_list_stack(word p); -#define GC_ADD_TO_BLACK_LIST_STACK(bits, source) \ - GC_add_to_black_list_stack((word)(bits)) -#endif /* PRINT_BLACK_LIST */ - -GC_INNER void GC_promote_black_lists(void); -/* Declare an end to a black listing phase. */ -GC_INNER void GC_unpromote_black_lists(void); -/* Approximately undo the effect of the above. */ -/* This actually loses some information, but */ -/* only in a reasonably safe way. */ - -GC_INNER ptr_t GC_scratch_alloc(size_t bytes); -/* GC internal memory allocation for */ -/* small objects. Deallocation is not */ -/* possible. May return NULL. */ - -#ifdef GWW_VDB -/* GC_scratch_recycle_no_gww() not used. */ -#else -#define GC_scratch_recycle_no_gww GC_scratch_recycle_inner -#endif -GC_INNER void GC_scratch_recycle_inner(void *ptr, size_t bytes); -/* Reuse the memory region by the heap. */ - -/* Heap block layout maps: */ -#ifdef MARK_BIT_PER_GRANULE -GC_INNER GC_bool GC_add_map_entry(size_t sz); -/* Add a heap block map for objects of */ -/* size sz to obj_map. */ -/* Return FALSE on failure. */ -#endif - -GC_INNER void GC_register_displacement_inner(size_t offset); -/* Version of GC_register_displacement */ -/* that assumes lock is already held. */ - -/* hblk allocation: */ -GC_INNER void GC_new_hblk(size_t size_in_granules, int kind); -/* Allocate a new heap block, and build */ -/* a free list in it. */ - -GC_INNER ptr_t GC_build_fl(struct hblk *h, size_t words, GC_bool clear, - ptr_t list); -/* Build a free list for objects of */ -/* size sz in block h. Append list to */ -/* end of the free lists. Possibly */ -/* clear objects on the list. Normally */ -/* called by GC_new_hblk, but also */ -/* called explicitly without GC lock. */ - -GC_INNER struct hblk *GC_allochblk(size_t size_in_bytes, int kind, - unsigned flags, size_t align_m1); -/* Allocate (and return pointer to) */ -/* a heap block for objects of the */ -/* given size and alignment (in bytes), */ -/* searching over the appropriate free */ -/* block lists; inform the marker */ -/* that the found block is valid for */ -/* objects of the indicated size. */ -/* The client is responsible for */ -/* clearing the block, if necessary. */ -/* Note: we set obj_map field in the */ -/* header correctly; the caller is */ -/* responsible for building an object */ -/* freelist in the block. */ - -GC_INNER ptr_t GC_alloc_large(size_t lb, int k, unsigned flags, - size_t align_m1); -/* Allocate a large block of size lb bytes with */ -/* the requested alignment (align_m1 plus one). */ -/* The block is not cleared. Assumes that */ -/* EXTRA_BYTES value is already added to lb. */ -/* The flags argument should be IGNORE_OFF_PAGE */ -/* or 0. Calls GC_allochblk() to do the actual */ -/* allocation, but also triggers GC and/or heap */ -/* expansion as appropriate. Updates value of */ -/* GC_bytes_allocd; does also other accounting. */ - -GC_INNER void GC_freehblk(struct hblk *p); -/* Deallocate a heap block and mark it */ -/* as invalid. */ - -/* Miscellaneous GC routines. */ - -GC_INNER GC_bool GC_expand_hp_inner(word n); -GC_INNER void GC_start_reclaim(GC_bool abort_if_found); -/* Restore unmarked objects to free */ -/* lists, or (if abort_if_found is */ -/* TRUE) report them. */ -/* Sweeping of small object pages is */ -/* largely deferred. */ -GC_INNER void GC_continue_reclaim(word sz, int kind); -/* Sweep pages of the given size and */ -/* kind, as long as possible, and */ -/* as long as the corresponding free */ -/* list is empty. sz is in granules. */ - -GC_INNER GC_bool GC_reclaim_all(GC_stop_func stop_func, GC_bool ignore_old); -/* Reclaim all blocks. Abort (in a */ -/* consistent state) if f returns TRUE. */ -GC_INNER ptr_t GC_reclaim_generic(struct hblk *hbp, hdr *hhdr, size_t sz, - GC_bool init, ptr_t list, - signed_word *count); -/* Rebuild free list in hbp with */ -/* header hhdr, with objects of size sz */ -/* bytes. Add list to the end of the */ -/* free list. Add the number of */ -/* reclaimed bytes to *count. */ -GC_INNER GC_bool GC_block_empty(hdr *hhdr); -/* Block completely unmarked? */ -GC_INNER int GC_CALLBACK GC_never_stop_func(void); -/* Always returns 0 (FALSE). */ -GC_INNER GC_bool GC_try_to_collect_inner(GC_stop_func f); - -/* Collect; caller must have acquired */ -/* lock. Collection is aborted if f */ -/* returns TRUE. Returns TRUE if it */ -/* completes successfully. */ -#define GC_gcollect_inner() \ - (void)GC_try_to_collect_inner(GC_never_stop_func) - -#ifdef THREADS -GC_EXTERN GC_bool GC_in_thread_creation; -/* We may currently be in thread creation or destruction. */ -/* Only set to TRUE while allocation lock is held. */ -/* When set, it is OK to run GC from unknown thread. */ -#endif - -GC_EXTERN GC_bool GC_is_initialized; /* GC_init() has been run. */ - -GC_INNER void GC_collect_a_little_inner(int n); -/* Do n units worth of garbage */ -/* collection work, if appropriate. */ -/* A unit is an amount appropriate for */ -/* HBLKSIZE bytes of allocation. */ - -GC_INNER void *GC_generic_malloc_aligned(size_t lb, int k, unsigned flags, - size_t align_m1); - -GC_INNER void *GC_generic_malloc_inner(size_t lb, int k, unsigned flags); -/* Allocate an object of the given */ -/* kind but assuming lock already held. */ -/* Should not be used to directly */ -/* allocate objects requiring special */ -/* handling on allocation. The flags */ -/* argument should be IGNORE_OFF_PAGE */ -/* or 0. In the first case the client */ -/* guarantees that there will always be */ -/* a pointer to the beginning (i.e. */ -/* within the first hblk) of the object */ -/* while it is live. */ - -GC_INNER GC_bool GC_collect_or_expand(word needed_blocks, unsigned flags, - GC_bool retry); - -GC_INNER ptr_t GC_allocobj(size_t gran, int kind); -/* Make the indicated free list */ -/* nonempty, and return its head. */ -/* The size (gran) is in granules. */ - -#ifdef GC_ADD_CALLER -/* GC_DBG_EXTRAS is used by GC debug API functions (unlike GC_EXTRAS */ -/* used by GC debug API macros) thus GC_RETURN_ADDR_PARENT (pointing */ -/* to client caller) should be used if possible. */ -#ifdef GC_HAVE_RETURN_ADDR_PARENT -#define GC_DBG_EXTRAS GC_RETURN_ADDR_PARENT, NULL, 0 -#else -#define GC_DBG_EXTRAS GC_RETURN_ADDR, NULL, 0 -#endif -#else -#define GC_DBG_EXTRAS "unknown", 0 -#endif /* !GC_ADD_CALLER */ - -#ifdef GC_COLLECT_AT_MALLOC -extern size_t GC_dbg_collect_at_malloc_min_lb; -/* variable visible outside for debugging */ -#define GC_DBG_COLLECT_AT_MALLOC(lb) \ - (void)((lb) >= GC_dbg_collect_at_malloc_min_lb ? (GC_gcollect(), 0) : 0) -#else -#define GC_DBG_COLLECT_AT_MALLOC(lb) (void)0 -#endif /* !GC_COLLECT_AT_MALLOC */ - -/* Allocation routines that bypass the thread local cache. */ -#if defined(THREAD_LOCAL_ALLOC) && defined(GC_GCJ_SUPPORT) -GC_INNER void *GC_core_gcj_malloc(size_t lb, void *, unsigned flags); -#endif - -GC_INNER void GC_init_headers(void); -GC_INNER struct hblkhdr *GC_install_header(struct hblk *h); -/* Install a header for block h. */ -/* Return 0 on failure, or the header */ -/* otherwise. */ -GC_INNER GC_bool GC_install_counts(struct hblk *h, size_t sz); -/* Set up forwarding counts for block */ -/* h of size sz. */ -/* Return FALSE on failure. */ -GC_INNER void GC_remove_header(struct hblk *h); -/* Remove the header for block h. */ -GC_INNER void GC_remove_counts(struct hblk *h, size_t sz); -/* Remove forwarding counts for h. */ -GC_INNER hdr *GC_find_header(ptr_t h); - -#ifdef USE_PROC_FOR_LIBRARIES -GC_INNER void GC_add_to_our_memory(ptr_t p, size_t bytes); -/* Add a chunk to GC_our_memory. */ -#else -#define GC_add_to_our_memory(p, bytes) \ - (GC_our_mem_bytes += (bytes), (void)(p)) -#endif - -GC_INNER void GC_print_all_errors(void); -/* Print smashed and leaked objects, if any. */ -/* Clear the lists of such objects. */ - -GC_EXTERN void (*GC_check_heap)(void); -/* Check that all objects in the heap with */ -/* debugging info are intact. */ -/* Add any that are not to GC_smashed list. */ -GC_EXTERN void (*GC_print_all_smashed)(void); -/* Print GC_smashed if it's not empty. */ -/* Clear GC_smashed list. */ -GC_EXTERN void (*GC_print_heap_obj)(ptr_t p); -/* If possible print (using GC_err_printf) */ -/* a more detailed description (terminated with */ -/* "\n") of the object referred to by p. */ - -#if defined(LINUX) && defined(__ELF__) && !defined(SMALL_CONFIG) -void GC_print_address_map(void); -/* Print an address map of the process. */ -#endif - -#ifndef SHORT_DBG_HDRS -GC_EXTERN GC_bool GC_findleak_delay_free; -/* Do not immediately deallocate object on */ -/* free() in the leak-finding mode, just mark */ -/* it as freed (and deallocate it after GC). */ -GC_INNER GC_bool GC_check_leaked(ptr_t base); /* from dbg_mlc.c */ -#endif - -#ifdef AO_HAVE_store -GC_EXTERN volatile AO_t GC_have_errors; -#define GC_SET_HAVE_ERRORS() AO_store(&GC_have_errors, (AO_t)TRUE) -#define get_have_errors() ((GC_bool)AO_load(&GC_have_errors)) -/* The barriers are not needed. */ -#else -GC_EXTERN GC_bool GC_have_errors; -#define GC_SET_HAVE_ERRORS() (void)(GC_have_errors = TRUE) -#define get_have_errors() GC_have_errors -#endif /* We saw a smashed or leaked object. */ - /* Call error printing routine */ - /* occasionally. It is OK to read it */ - /* without acquiring the lock. */ - /* If set to true, it is never cleared. */ - -#define VERBOSE 2 -#if !defined(NO_CLOCK) || !defined(SMALL_CONFIG) -GC_EXTERN int GC_print_stats; -/* Value 1 generates basic GC log; */ -/* VERBOSE generates additional messages. */ -#else /* SMALL_CONFIG */ -#define GC_print_stats 0 -/* Will this remove the message character strings from the executable? */ -/* With a particular level of optimizations, it should... */ -#endif - -#ifdef KEEP_BACK_PTRS -GC_EXTERN long GC_backtraces; -#endif - -#if defined(THREADS) || defined(LINT2) -/* A trivial (linear congruential) pseudo-random numbers generator, */ -/* safe for the concurrent usage. */ -#define GC_RAND_MAX ((int)(~0U >> 1)) -#if defined(AO_HAVE_store) && defined(THREAD_SANITIZER) -#define GC_RAND_STATE_T volatile AO_t -#define GC_RAND_NEXT(pseed) GC_rand_next(pseed) -GC_INLINE int GC_rand_next(GC_RAND_STATE_T *pseed) { - AO_t next = (AO_t)((AO_load(pseed) * 1103515245U + 12345) & (unsigned)GC_RAND_MAX); - AO_store(pseed, next); - return (int)next; -} -#else -#define GC_RAND_STATE_T unsigned -#define GC_RAND_NEXT(pseed) /* overflow and race are OK */ \ - (int)(*(pseed) = (*(pseed)*1103515245U + 12345) & (unsigned)GC_RAND_MAX) -#endif -#endif /* THREADS || LINT2 */ - -GC_EXTERN GC_bool GC_print_back_height; - -#ifdef MAKE_BACK_GRAPH -void GC_print_back_graph_stats(void); -#endif - -#ifdef THREADS -/* Explicitly deallocate the object when we already hold lock. */ -/* Only used for internally allocated objects. */ -GC_INNER void GC_free_inner(void *p); -#endif - -/* Macros used for collector internal allocation. */ -/* These assume the collector lock is held. */ -#ifdef DBG_HDRS_ALL -GC_INNER void *GC_debug_generic_malloc_inner(size_t lb, int k, - unsigned flags); -#define GC_INTERNAL_MALLOC(lb, k) GC_debug_generic_malloc_inner(lb, k, 0) -#define GC_INTERNAL_MALLOC_IGNORE_OFF_PAGE(lb, k) \ - GC_debug_generic_malloc_inner(lb, k, IGNORE_OFF_PAGE) -#ifdef THREADS -GC_INNER void GC_debug_free_inner(void *p); -#define GC_INTERNAL_FREE GC_debug_free_inner -#else -#define GC_INTERNAL_FREE GC_debug_free -#endif -#else -#define GC_INTERNAL_MALLOC(lb, k) GC_generic_malloc_inner(lb, k, 0) -#define GC_INTERNAL_MALLOC_IGNORE_OFF_PAGE(lb, k) \ - GC_generic_malloc_inner(lb, k, IGNORE_OFF_PAGE) -#ifdef THREADS -#define GC_INTERNAL_FREE GC_free_inner -#else -#define GC_INTERNAL_FREE GC_free -#endif -#endif /* !DBG_HDRS_ALL */ - -#ifdef USE_MUNMAP -/* Memory unmapping: */ -GC_INNER void GC_unmap_old(unsigned threshold); -GC_INNER void GC_merge_unmapped(void); -GC_INNER void GC_unmap(ptr_t start, size_t bytes); -GC_INNER void GC_remap(ptr_t start, size_t bytes); -GC_INNER void GC_unmap_gap(ptr_t start1, size_t bytes1, ptr_t start2, - size_t bytes2); - -#ifndef NOT_GCBUILD -/* Compute end address for an unmap operation on the indicated block. */ -GC_INLINE ptr_t GC_unmap_end(ptr_t start, size_t bytes) { - return (ptr_t)((word)(start + bytes) & ~(GC_page_size - 1)); -} -#endif -#endif /* USE_MUNMAP */ - -#ifdef CAN_HANDLE_FORK -GC_EXTERN int GC_handle_fork; -/* Fork-handling mode: */ -/* 0 means no fork handling requested (but client could */ -/* anyway call fork() provided it is surrounded with */ -/* GC_atfork_prepare/parent/child calls); */ -/* -1 means GC tries to use pthread_at_fork if it is */ -/* available (if it succeeds then GC_handle_fork value */ -/* is changed to 1), client should nonetheless surround */ -/* fork() with GC_atfork_prepare/parent/child (for the */ -/* case of pthread_at_fork failure or absence); */ -/* 1 (or other values) means client fully relies on */ -/* pthread_at_fork (so if it is missing or failed then */ -/* abort occurs in GC_init), GC_atfork_prepare and the */ -/* accompanying routines are no-op in such a case. */ -#endif - -#ifdef GC_DISABLE_INCREMENTAL -#define GC_incremental FALSE -#define GC_auto_incremental FALSE -#define GC_manual_vdb FALSE -#define GC_dirty(p) (void)(p) -#define REACHABLE_AFTER_DIRTY(p) (void)(p) - -#else /* !GC_DISABLE_INCREMENTAL */ -GC_EXTERN GC_bool GC_incremental; -/* Using incremental/generational collection. */ -/* Assumes dirty bits are being maintained. */ - -/* Virtual dirty bit implementation: */ -/* Each implementation exports the following: */ -GC_INNER void GC_read_dirty(GC_bool output_unneeded); -/* Retrieve dirty bits. Set output_unneeded to */ -/* indicate that reading of the retrieved dirty */ -/* bits is not planned till the next retrieval. */ -GC_INNER GC_bool GC_page_was_dirty(struct hblk *h); -/* Read retrieved dirty bits. */ - -GC_INNER void GC_remove_protection(struct hblk *h, word nblocks, - GC_bool pointerfree); -/* h is about to be written or allocated. Ensure that */ -/* it is not write protected by the virtual dirty bit */ -/* implementation. I.e., this is a call that: */ -/* - hints that [h, h+nblocks) is about to be written; */ -/* - guarantees that protection is removed; */ -/* - may speed up some dirty bit implementations; */ -/* - may be essential if we need to ensure that */ -/* pointer-free system call buffers in the heap are */ -/* not protected. */ - -#if !defined(NO_VDB_FOR_STATIC_ROOTS) && !defined(PROC_VDB) -GC_INNER GC_bool GC_is_vdb_for_static_roots(void); -/* Is VDB working for static roots? */ -#endif - -#ifdef CAN_HANDLE_FORK -#if defined(PROC_VDB) || defined(SOFT_VDB) -GC_INNER void GC_dirty_update_child(void); -/* Update pid-specific resources (like /proc file */ -/* descriptors) needed by the dirty bits implementation */ -/* after fork in the child process. */ -#else -#define GC_dirty_update_child() (void)0 -#endif -#endif /* CAN_HANDLE_FORK */ - -GC_INNER GC_bool GC_dirty_init(void); -/* Returns true if dirty bits are maintained (otherwise */ -/* it is OK to be called again if the client invokes */ -/* GC_enable_incremental once more). */ - -GC_EXTERN GC_bool GC_manual_vdb; -/* The incremental collection is in the manual VDB */ -/* mode. Assumes GC_incremental is true. Should not */ -/* be modified once GC_incremental is set to true. */ - -#define GC_auto_incremental (GC_incremental && !GC_manual_vdb) - -GC_INNER void GC_dirty_inner(const void *p); /* does not require locking */ -#define GC_dirty(p) (GC_manual_vdb ? GC_dirty_inner(p) : (void)0) -#define REACHABLE_AFTER_DIRTY(p) GC_reachable_here(p) -#endif /* !GC_DISABLE_INCREMENTAL */ - -/* Same as GC_base but excepts and returns a pointer to const object. */ -#define GC_base_C(p) ((const void *)GC_base((/* no const */ void *)(p))) - -/* Debugging print routines: */ -void GC_print_block_list(void); -void GC_print_hblkfreelist(void); -void GC_print_heap_sects(void); -void GC_print_static_roots(void); - -#ifdef KEEP_BACK_PTRS -GC_INNER void GC_store_back_pointer(ptr_t source, ptr_t dest); -GC_INNER void GC_marked_for_finalization(ptr_t dest); -#define GC_STORE_BACK_PTR(source, dest) GC_store_back_pointer(source, dest) -#define GC_MARKED_FOR_FINALIZATION(dest) GC_marked_for_finalization(dest) -#else -#define GC_STORE_BACK_PTR(source, dest) (void)(source) -#define GC_MARKED_FOR_FINALIZATION(dest) -#endif /* !KEEP_BACK_PTRS */ - -/* Make arguments appear live to compiler */ -void GC_noop6(word, word, word, word, word, word); - -#ifndef GC_ATTR_FORMAT_PRINTF -#if GC_GNUC_PREREQ(3, 0) -#define GC_ATTR_FORMAT_PRINTF(spec_argnum, first_checked) \ - __attribute__((__format__(__printf__, spec_argnum, first_checked))) -#else -#define GC_ATTR_FORMAT_PRINTF(spec_argnum, first_checked) -#endif -#endif - -/* Logging and diagnostic output: */ -/* GC_printf is used typically on client explicit print requests. */ -/* For all GC_X_printf routines, it is recommended to put "\n" at */ -/* 'format' string end (for output atomicity). */ -GC_API_PRIV void GC_printf(const char *format, ...) - GC_ATTR_FORMAT_PRINTF(1, 2); -/* A version of printf that doesn't allocate, */ -/* 1 KB total output length. */ -/* (We use sprintf. Hopefully that doesn't */ -/* allocate for long arguments.) */ -GC_API_PRIV void GC_err_printf(const char *format, ...) - GC_ATTR_FORMAT_PRINTF(1, 2); - -/* Basic logging routine. Typically, GC_log_printf is called directly */ -/* only inside various DEBUG_x blocks. */ -GC_API_PRIV void GC_log_printf(const char *format, ...) - GC_ATTR_FORMAT_PRINTF(1, 2); - -#ifndef GC_ANDROID_LOG -#define GC_PRINT_STATS_FLAG (GC_print_stats != 0) -#define GC_INFOLOG_PRINTF GC_COND_LOG_PRINTF -/* GC_verbose_log_printf is called only if GC_print_stats is VERBOSE. */ -#define GC_verbose_log_printf GC_log_printf -#else -extern GC_bool GC_quiet; -#define GC_PRINT_STATS_FLAG (!GC_quiet) -/* INFO/DBG loggers are enabled even if GC_print_stats is off. */ -#ifndef GC_INFOLOG_PRINTF -#define GC_INFOLOG_PRINTF \ - if (GC_quiet) { \ - } else \ - GC_info_log_printf -#endif -GC_INNER void GC_info_log_printf(const char *format, ...) - GC_ATTR_FORMAT_PRINTF(1, 2); -GC_INNER void GC_verbose_log_printf(const char *format, ...) - GC_ATTR_FORMAT_PRINTF(1, 2); -#endif /* GC_ANDROID_LOG */ - -#if defined(SMALL_CONFIG) || defined(GC_ANDROID_LOG) -#define GC_ERRINFO_PRINTF GC_INFOLOG_PRINTF -#else -#define GC_ERRINFO_PRINTF GC_log_printf -#endif - -/* Convenient macros for GC_[verbose_]log_printf invocation. */ -#define GC_COND_LOG_PRINTF \ - if (EXPECT(!GC_print_stats, TRUE)) { \ - } else \ - GC_log_printf -#define GC_VERBOSE_LOG_PRINTF \ - if (EXPECT(GC_print_stats != VERBOSE, TRUE)) { \ - } else \ - GC_verbose_log_printf -#ifndef GC_DBGLOG_PRINTF -#define GC_DBGLOG_PRINTF \ - if (!GC_PRINT_STATS_FLAG) { \ - } else \ - GC_log_printf -#endif - -void GC_err_puts(const char *s); -/* Write s to stderr, don't buffer, don't add */ -/* newlines, don't ... */ - -/* Handy macro for logging size values (of word type) in KiB (rounding */ -/* to nearest value). */ -#define TO_KiB_UL(v) ((unsigned long)(((v) + ((1 << 9) - 1)) >> 10)) - -GC_EXTERN unsigned GC_fail_count; -/* How many consecutive GC/expansion failures? */ -/* Reset by GC_allochblk(); defined in alloc.c. */ - -GC_EXTERN long GC_large_alloc_warn_interval; /* defined in misc.c */ - -GC_EXTERN signed_word GC_bytes_found; -/* Number of reclaimed bytes after garbage collection; */ -/* protected by GC lock; defined in reclaim.c. */ - -#ifndef GC_GET_HEAP_USAGE_NOT_NEEDED -GC_EXTERN word GC_reclaimed_bytes_before_gc; -/* Number of bytes reclaimed before this */ -/* collection cycle; used for statistics only. */ -#endif - -#ifdef USE_MUNMAP -GC_EXTERN unsigned GC_unmap_threshold; /* defined in alloc.c */ -GC_EXTERN GC_bool GC_force_unmap_on_gcollect; /* defined in misc.c */ -#endif - -#ifdef MSWIN32 -GC_EXTERN GC_bool GC_no_win32_dlls; /* defined in os_dep.c */ -GC_EXTERN GC_bool GC_wnt; /* Is Windows NT derivative; */ - /* defined and set in os_dep.c. */ -#endif - -#ifdef THREADS -#if (defined(MSWIN32) && !defined(CONSOLE_LOG)) || defined(MSWINCE) -GC_EXTERN CRITICAL_SECTION GC_write_cs; /* defined in misc.c */ -#ifdef GC_ASSERTIONS -GC_EXTERN GC_bool GC_write_disabled; -/* defined in win32_threads.c; */ -/* protected by GC_write_cs. */ - -#endif -#endif /* MSWIN32 || MSWINCE */ -#if defined(GC_DISABLE_INCREMENTAL) || defined(HAVE_LOCKFREE_AO_OR) -#define GC_acquire_dirty_lock() (void)0 -#define GC_release_dirty_lock() (void)0 -#else -/* Acquire the spin lock we use to update dirty bits. */ -/* Threads should not get stopped holding it. But we may */ -/* acquire and release it during GC_remove_protection call. */ -#define GC_acquire_dirty_lock() \ - do { /* empty */ \ - } while (AO_test_and_set_acquire(&GC_fault_handler_lock) == AO_TS_SET) -#define GC_release_dirty_lock() AO_CLEAR(&GC_fault_handler_lock) -GC_EXTERN volatile AO_TS_t GC_fault_handler_lock; -/* defined in os_dep.c */ -#endif -#ifdef MSWINCE -GC_EXTERN GC_bool GC_dont_query_stack_min; -/* Defined and set in os_dep.c. */ -#endif -#elif defined(IA64) -GC_EXTERN ptr_t GC_save_regs_ret_val; /* defined in mach_dep.c. */ - /* Previously set to backing store pointer. */ -#endif /* !THREADS */ - -#ifdef THREAD_LOCAL_ALLOC -GC_EXTERN GC_bool GC_world_stopped; /* defined in alloc.c */ -GC_INNER void GC_mark_thread_local_free_lists(void); -#endif - -#if defined(GLIBC_2_19_TSX_BUG) && defined(THREADS) -/* Parse string like [.[]] and return major value. */ -GC_INNER int GC_parse_version(int *pminor, const char *pverstr); -#endif - -#if defined(MPROTECT_VDB) && defined(GWW_VDB) -GC_INNER GC_bool GC_gww_dirty_init(void); -/* Returns TRUE if GetWriteWatch is available. */ -/* May be called repeatedly. May be called */ -/* with or without the GC lock held. */ -#endif - -#if defined(CHECKSUMS) || defined(PROC_VDB) -GC_INNER GC_bool GC_page_was_ever_dirty(struct hblk *h); -/* Could the page contain valid heap pointers? */ -#endif - -#ifdef CHECKSUMS -#if defined(MPROTECT_VDB) && !defined(DARWIN) -void GC_record_fault(struct hblk *h); -#endif -void GC_check_dirty(void); -#endif - -GC_INNER void GC_default_print_heap_obj_proc(ptr_t p); - -GC_INNER void GC_setpagesize(void); - -GC_INNER void GC_initialize_offsets(void); /* defined in obj_map.c */ - -GC_INNER void GC_bl_init(void); -GC_INNER void GC_bl_init_no_interiors(void); /* defined in blacklst.c */ - -GC_INNER void GC_start_debugging_inner(void); /* defined in dbg_mlc.c. */ - /* Should not be called if GC_debugging_started. */ - -/* Store debugging info into p. Return displaced pointer. */ -/* Assumes we hold the allocation lock. */ -GC_INNER void *GC_store_debug_info_inner(void *p, word sz, const char *str, - int linenum); - -#ifdef REDIRECT_MALLOC -#ifdef GC_LINUX_THREADS -GC_INNER GC_bool GC_text_mapping(char *nm, ptr_t *startp, ptr_t *endp); -/* from os_dep.c */ -#endif -#elif defined(USE_WINALLOC) -GC_INNER void GC_add_current_malloc_heap(void); -#endif /* !REDIRECT_MALLOC */ - -#ifdef MAKE_BACK_GRAPH -GC_INNER void GC_build_back_graph(void); -GC_INNER void GC_traverse_back_graph(void); -#endif - -#ifdef MSWIN32 -GC_INNER void GC_init_win32(void); -#endif - -#if !defined(MSWIN32) && !defined(MSWINCE) && !defined(CYGWIN32) -GC_INNER void *GC_roots_present(ptr_t); -/* The type is a lie, since the real type doesn't make sense here, */ -/* and we only test for NULL. */ -#endif - -#ifdef GC_WIN32_THREADS -GC_INNER void GC_get_next_stack(char *start, char *limit, char **lo, - char **hi); -#if defined(MPROTECT_VDB) && !defined(CYGWIN32) -GC_INNER void GC_set_write_fault_handler(void); -#endif -#if defined(WRAP_MARK_SOME) && !defined(GC_PTHREADS) -GC_INNER GC_bool GC_started_thread_while_stopped(void); -/* Did we invalidate mark phase with an unexpected thread start? */ -#endif -#endif /* GC_WIN32_THREADS */ - -#if defined(GC_DARWIN_THREADS) && defined(MPROTECT_VDB) -GC_INNER void GC_mprotect_stop(void); -GC_INNER void GC_mprotect_resume(void); -#ifndef GC_NO_THREADS_DISCOVERY -GC_INNER void GC_darwin_register_self_mach_handler(void); -#endif -#endif - -#ifdef THREADS -#ifndef GC_NO_FINALIZATION -GC_INNER void GC_reset_finalizer_nested(void); -GC_INNER unsigned char *GC_check_finalizer_nested(void); -#endif -GC_INNER void GC_do_blocking_inner(ptr_t data, void *context); -GC_INNER void GC_push_all_stacks(void); -#ifdef USE_PROC_FOR_LIBRARIES -GC_INNER GC_bool GC_segment_is_thread_stack(ptr_t lo, ptr_t hi); -#endif -#if (defined(HAVE_PTHREAD_ATTR_GET_NP) || defined(HAVE_PTHREAD_GETATTR_NP)) && defined(IA64) -GC_INNER ptr_t GC_greatest_stack_base_below(ptr_t bound); -#endif -#endif /* THREADS */ - -#ifdef DYNAMIC_LOADING -GC_INNER GC_bool GC_register_main_static_data(void); -#ifdef DARWIN -GC_INNER void GC_init_dyld(void); -#endif -#endif /* DYNAMIC_LOADING */ - -#ifdef SEARCH_FOR_DATA_START -GC_INNER void GC_init_linux_data_start(void); -void *GC_find_limit(void *, int); -#endif - -#ifdef UNIX_LIKE -GC_INNER void GC_set_and_save_fault_handler(void (*handler)(int)); -#endif - -#ifdef NEED_PROC_MAPS -#if defined(DYNAMIC_LOADING) && defined(USE_PROC_FOR_LIBRARIES) -GC_INNER const char *GC_parse_map_entry(const char *maps_ptr, - ptr_t *start, ptr_t *end, - const char **prot, - unsigned *maj_dev, - const char **mapping_name); -#endif -#if defined(IA64) || defined(INCLUDE_LINUX_THREAD_DESCR) -GC_INNER GC_bool GC_enclosing_mapping(ptr_t addr, - ptr_t *startp, ptr_t *endp); -#endif -GC_INNER const char *GC_get_maps(void); -#endif /* NEED_PROC_MAPS */ - -#ifdef GC_ASSERTIONS -GC_INNER word GC_compute_large_free_bytes(void); -GC_INNER word GC_compute_root_size(void); -#endif - -/* Check a compile time assertion at compile time. */ -#if _MSC_VER >= 1700 -#define GC_STATIC_ASSERT(expr) \ - static_assert(expr, "static assertion failed: " #expr) -#elif defined(static_assert) && !defined(CPPCHECK) && (__STDC_VERSION__ >= 201112L) -#define GC_STATIC_ASSERT(expr) static_assert(expr, #expr) -#elif defined(mips) && !defined(__GNUC__) && !defined(CPPCHECK) -/* DOB: MIPSPro C gets an internal error taking the sizeof an array type. - This code works correctly (ugliness is to avoid "unused var" warnings) */ -#define GC_STATIC_ASSERT(expr) \ - do { \ - if (0) { \ - char j[(expr) ? 1 : -1]; \ - j[0] = '\0'; \ - j[0] = j[0]; \ - } \ - } while (0) -#else -/* The error message for failure is a bit baroque, but ... */ -#define GC_STATIC_ASSERT(expr) (void)sizeof(char[(expr) ? 1 : -1]) -#endif - -/* Runtime check for an argument declared as non-null is actually not null. */ -#if GC_GNUC_PREREQ(4, 0) -/* Workaround tautological-pointer-compare Clang warning. */ -#define NONNULL_ARG_NOT_NULL(arg) (*(volatile void **)&(arg) != NULL) -#else -#define NONNULL_ARG_NOT_NULL(arg) (NULL != (arg)) -#endif - -#define COND_DUMP_CHECKS \ - do { \ - GC_ASSERT(GC_compute_large_free_bytes() == GC_large_free_bytes); \ - GC_ASSERT(GC_compute_root_size() == GC_root_size); \ - } while (0) - -#ifndef NO_DEBUGGING -GC_EXTERN GC_bool GC_dump_regularly; -/* Generate regular debugging dumps. */ -#define COND_DUMP \ - if (EXPECT(GC_dump_regularly, FALSE)) { \ - GC_dump_named(NULL); \ - } else \ - COND_DUMP_CHECKS -#else -#define COND_DUMP COND_DUMP_CHECKS -#endif - -#if defined(PARALLEL_MARK) -/* We need additional synchronization facilities from the thread */ -/* support. We believe these are less performance critical */ -/* than the main garbage collector lock; standard pthreads-based */ -/* implementations should be sufficient. */ - -#define GC_markers_m1 GC_parallel -/* Number of mark threads we would like to have */ -/* excluding the initiating thread. */ - -GC_EXTERN GC_bool GC_parallel_mark_disabled; -/* A flag to temporarily avoid parallel marking.*/ - -/* The mark lock and condition variable. If the GC lock is also */ -/* acquired, the GC lock must be acquired first. The mark lock is */ -/* used to both protect some variables used by the parallel */ -/* marker, and to protect GC_fl_builder_count, below. */ -/* GC_notify_all_marker() is called when */ -/* the state of the parallel marker changes */ -/* in some significant way (see gc_mark.h for details). The */ -/* latter set of events includes incrementing GC_mark_no. */ -/* GC_notify_all_builder() is called when GC_fl_builder_count */ -/* reaches 0. */ - -GC_INNER void GC_wait_for_markers_init(void); -GC_INNER void GC_acquire_mark_lock(void); -GC_INNER void GC_release_mark_lock(void); -GC_INNER void GC_notify_all_builder(void); -GC_INNER void GC_wait_for_reclaim(void); - -GC_EXTERN signed_word GC_fl_builder_count; /* Protected by mark lock. */ - -GC_INNER void GC_notify_all_marker(void); -GC_INNER void GC_wait_marker(void); -GC_EXTERN word GC_mark_no; /* Protected by mark lock. */ - -GC_INNER void GC_help_marker(word my_mark_no); -/* Try to help out parallel marker for mark cycle */ -/* my_mark_no. Returns if the mark cycle finishes or */ -/* was already done, or there was nothing to do for */ -/* some other reason. */ - -GC_INNER void GC_start_mark_threads_inner(void); -#endif /* PARALLEL_MARK */ - -#if defined(SIGNAL_BASED_STOP_WORLD) && !defined(SIG_SUSPEND) -/* We define the thread suspension signal here, so that we can refer */ -/* to it in the dirty bit implementation, if necessary. Ideally we */ -/* would allocate a (real-time?) signal using the standard mechanism. */ -/* unfortunately, there is no standard mechanism. (There is one */ -/* in Linux glibc, but it's not exported.) Thus we continue to use */ -/* the same hard-coded signals we've always used. */ -#ifdef THREAD_SANITIZER -/* Unfortunately, use of an asynchronous signal to suspend threads */ -/* leads to the situation when the signal is not delivered (is */ -/* stored to pending_signals in TSan runtime actually) while the */ -/* destination thread is blocked in pthread_mutex_lock. Thus, we */ -/* use some synchronous one instead (which is again unlikely to be */ -/* used by clients directly). */ -#define SIG_SUSPEND SIGSYS -#elif (defined(GC_LINUX_THREADS) || defined(GC_DGUX386_THREADS)) && !defined(GC_USESIGRT_SIGNALS) -#if defined(SPARC) && !defined(SIGPWR) -/* SPARC/Linux doesn't properly define SIGPWR in . */ -/* It is aliased to SIGLOST in asm/signal.h, though. */ -#define SIG_SUSPEND SIGLOST -#else -/* Linuxthreads itself uses SIGUSR1 and SIGUSR2. */ -#define SIG_SUSPEND SIGPWR -#endif -#elif defined(GC_FREEBSD_THREADS) && defined(__GLIBC__) && !defined(GC_USESIGRT_SIGNALS) -#define SIG_SUSPEND (32 + 6) -#elif (defined(GC_FREEBSD_THREADS) || defined(HURD) || defined(RTEMS)) && !defined(GC_USESIGRT_SIGNALS) -#define SIG_SUSPEND SIGUSR1 -/* SIGTSTP and SIGCONT could be used alternatively on FreeBSD. */ -#elif defined(GC_OPENBSD_THREADS) && !defined(GC_USESIGRT_SIGNALS) -#define SIG_SUSPEND SIGXFSZ -#elif defined(_SIGRTMIN) && !defined(CPPCHECK) -#define SIG_SUSPEND _SIGRTMIN + 6 -#else -#define SIG_SUSPEND SIGRTMIN + 6 -#endif -#endif /* GC_PTHREADS && !SIG_SUSPEND */ - -#if defined(GC_PTHREADS) && !defined(GC_SEM_INIT_PSHARED) -#define GC_SEM_INIT_PSHARED 0 -#endif - -/* Some macros for setjmp that works across signal handlers */ -/* were possible, and a couple of routines to facilitate */ -/* catching accesses to bad addresses when that's */ -/* possible/needed. */ -#if (defined(UNIX_LIKE) || (defined(NEED_FIND_LIMIT) && defined(CYGWIN32))) && !defined(GC_NO_SIGSETJMP) -#if defined(SUNOS5SIGS) && !defined(FREEBSD) && !defined(LINUX) -EXTERN_C_END -#include -EXTERN_C_BEGIN -#endif -/* Define SETJMP and friends to be the version that restores */ -/* the signal mask. */ -#define SETJMP(env) sigsetjmp(env, 1) -#define LONGJMP(env, val) siglongjmp(env, val) -#define JMP_BUF sigjmp_buf -#else -#ifdef ECOS -#define SETJMP(env) hal_setjmp(env) -#else -#define SETJMP(env) setjmp(env) -#endif -#define LONGJMP(env, val) longjmp(env, val) -#define JMP_BUF jmp_buf -#endif /* !UNIX_LIKE || GC_NO_SIGSETJMP */ - -#if defined(DATASTART_USES_BSDGETDATASTART) -EXTERN_C_END -#include -EXTERN_C_BEGIN -GC_INNER ptr_t GC_FreeBSDGetDataStart(size_t, ptr_t); -#define DATASTART_IS_FUNC -#endif /* DATASTART_USES_BSDGETDATASTART */ - -#if defined(NEED_FIND_LIMIT) || (defined(WRAP_MARK_SOME) && defined(NO_SEH_AVAILABLE)) || (defined(USE_PROC_FOR_LIBRARIES) && defined(THREADS)) -GC_EXTERN JMP_BUF GC_jmp_buf; - -/* Set up a handler for address faults which will longjmp to */ -/* GC_jmp_buf. */ -GC_INNER void GC_setup_temporary_fault_handler(void); -/* Undo the effect of GC_setup_temporary_fault_handler. */ -GC_INNER void GC_reset_fault_handler(void); -#endif /* NEED_FIND_LIMIT || USE_PROC_FOR_LIBRARIES || WRAP_MARK_SOME */ - -/* Some convenience macros for cancellation support. */ -#if defined(CANCEL_SAFE) -#if defined(GC_ASSERTIONS) && (defined(USE_COMPILER_TLS) || (defined(LINUX) && !defined(ARM32) && GC_GNUC_PREREQ(3, 3) || defined(HPUX) /* and probably others ... */)) -extern __thread unsigned char GC_cancel_disable_count; -#define NEED_CANCEL_DISABLE_COUNT -#define INCR_CANCEL_DISABLE() ++GC_cancel_disable_count -#define DECR_CANCEL_DISABLE() --GC_cancel_disable_count -#define ASSERT_CANCEL_DISABLED() GC_ASSERT(GC_cancel_disable_count > 0) -#else -#define INCR_CANCEL_DISABLE() -#define DECR_CANCEL_DISABLE() -#define ASSERT_CANCEL_DISABLED() (void)0 -#endif /* GC_ASSERTIONS & ... */ -#define DISABLE_CANCEL(state) \ - do { \ - pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, &state); \ - INCR_CANCEL_DISABLE(); \ - } while (0) -#define RESTORE_CANCEL(state) \ - do { \ - ASSERT_CANCEL_DISABLED(); \ - pthread_setcancelstate(state, NULL); \ - DECR_CANCEL_DISABLE(); \ - } while (0) -#else /* !CANCEL_SAFE */ -#define DISABLE_CANCEL(state) (void)0 -#define RESTORE_CANCEL(state) (void)0 -#define ASSERT_CANCEL_DISABLED() (void)0 -#endif /* !CANCEL_SAFE */ - -EXTERN_C_END - -#endif /* GC_PRIVATE_H */ diff --git a/vendor/bdwgc/private/gcconfig.h b/vendor/bdwgc/private/gcconfig.h deleted file mode 100644 index f8ffad27..00000000 --- a/vendor/bdwgc/private/gcconfig.h +++ /dev/null @@ -1,3224 +0,0 @@ -/* - * Copyright 1988, 1989 Hans-J. Boehm, Alan J. Demers - * Copyright (c) 1991-1994 by Xerox Corporation. All rights reserved. - * Copyright (c) 1996 by Silicon Graphics. All rights reserved. - * Copyright (c) 2000-2004 Hewlett-Packard Development Company, L.P. - * Copyright (c) 2009-2022 Ivan Maidanski - * - * THIS MATERIAL IS PROVIDED AS IS, WITH ABSOLUTELY NO WARRANTY EXPRESSED - * OR IMPLIED. ANY USE IS AT YOUR OWN RISK. - * - * Permission is hereby granted to use or copy this program - * for any purpose, provided the above notices are retained on all copies. - * Permission to modify the code and to distribute modified code is granted, - * provided the above notices are retained, and a notice that the code was - * modified is included with the above copyright notice. - */ - -/* - * This header is private to the gc. It is almost always included from - * gc_priv.h. However it is possible to include it by itself if just the - * configuration macros are needed. In that - * case, a few declarations relying on types declared in gc_priv.h will be - * omitted. - */ - -#ifndef GCCONFIG_H -#define GCCONFIG_H - -#ifdef CPPCHECK -#undef CLOCKS_PER_SEC -#undef FIXUP_POINTER -#undef POINTER_MASK -#undef POINTER_SHIFT -#undef REDIRECT_REALLOC -#undef _MAX_PATH -#endif - -#ifndef PTR_T_DEFINED -typedef char *ptr_t; -#define PTR_T_DEFINED -#endif - -#if !defined(sony_news) -#include /* For size_t, etc. */ -#endif - -/* Note: Only wrap our own declarations, and not the included headers. */ -/* In this case, wrap our entire file, but temporarily unwrap/rewrap */ -/* around #includes. Types and macros do not need such wrapping, only */ -/* the declared global data and functions. */ -#ifdef __cplusplus -#define EXTERN_C_BEGIN extern "C" { -#define EXTERN_C_END } /* extern "C" */ -#else -#define EXTERN_C_BEGIN /* empty */ -#define EXTERN_C_END /* empty */ -#endif - -EXTERN_C_BEGIN - -/* Convenient internal macro to test version of Clang. */ -#if defined(__clang__) && defined(__clang_major__) -#define GC_CLANG_PREREQ(major, minor) \ - ((__clang_major__ << 16) + __clang_minor__ >= ((major) << 16) + (minor)) -#define GC_CLANG_PREREQ_FULL(major, minor, patchlevel) \ - (GC_CLANG_PREREQ(major, (minor) + 1) || (__clang_major__ == (major) && __clang_minor__ == (minor) && __clang_patchlevel__ >= (patchlevel))) -#else -#define GC_CLANG_PREREQ(major, minor) 0 /* FALSE */ -#define GC_CLANG_PREREQ_FULL(major, minor, patchlevel) 0 -#endif - -#ifdef LINT2 -/* A macro (based on a tricky expression) to prevent false warnings */ -/* like "Array compared to 0", "Comparison of identical expressions", */ -/* "Untrusted loop bound" output by some static code analysis tools. */ -/* The argument should not be a literal value. The result is */ -/* converted to word type. (Actually, GC_word is used instead of */ -/* word type as the latter might be undefined at the place of use.) */ -#define COVERT_DATAFLOW(w) (~(GC_word)(w) ^ (~(GC_word)0)) -#else -#define COVERT_DATAFLOW(w) ((GC_word)(w)) -#endif - -/* Machine dependent parameters. Some tuning parameters can be found */ -/* near the top of gc_private.h. */ - -/* Machine specific parts contributed by various people. See README file. */ - -#if defined(__ANDROID__) && !defined(HOST_ANDROID) -/* __ANDROID__ macro is defined by Android NDK gcc. */ -#define HOST_ANDROID 1 -#endif - -#if defined(TIZEN) && !defined(HOST_TIZEN) -#define HOST_TIZEN 1 -#endif - -#if defined(__SYMBIAN32__) && !defined(SYMBIAN) -#define SYMBIAN -#ifdef __WINS__ -#pragma data_seg(".data2") -#endif -#endif - -/* First a unified test for Linux: */ -#if (defined(linux) || defined(__linux__) || defined(HOST_ANDROID)) && !defined(LINUX) && !defined(__native_client__) -#define LINUX -#endif - -/* And one for NetBSD: */ -#if defined(__NetBSD__) -#define NETBSD -#endif - -/* And one for OpenBSD: */ -#if defined(__OpenBSD__) -#define OPENBSD -#endif - -/* And one for FreeBSD: */ -#if (defined(__FreeBSD__) || defined(__DragonFly__) || defined(__FreeBSD_kernel__)) && !defined(FREEBSD) && !defined(GC_NO_FREEBSD) /* Orbis compiler defines __FreeBSD__ */ -#define FREEBSD -#endif - -#if defined(__QNX__) && !defined(QNX) -#define QNX -#endif - -/* And one for Darwin: */ -#if defined(macosx) || (defined(__APPLE__) && defined(__MACH__)) -#define DARWIN -EXTERN_C_END -#include -EXTERN_C_BEGIN -#endif - -/* Determine the machine type: */ -#if defined(__native_client__) -#define NACL -#if !defined(__portable_native_client__) && !defined(__arm__) -#define I386 -#define mach_type_known -#else -/* Here we will rely upon arch-specific defines. */ -#endif -#endif -#if defined(__aarch64__) && !defined(DARWIN) && !defined(LINUX) && !defined(FREEBSD) && !defined(NETBSD) && !defined(OPENBSD) && !defined(QNX) && !defined(NN_BUILD_TARGET_PLATFORM_NX) && !defined(_WIN32) -#define AARCH64 -#define NOSYS -#define mach_type_known -#endif -#if defined(__arm) || defined(__arm__) || defined(__thumb__) -#define ARM32 -#if defined(NACL) -#define mach_type_known -#elif !defined(DARWIN) && !defined(LINUX) && !defined(FREEBSD) && !defined(NETBSD) && !defined(OPENBSD) && !defined(QNX) && !defined(NN_PLATFORM_CTR) && !defined(SN_TARGET_PSP2) && !defined(SYMBIAN) && !defined(_WIN32) && !defined(__CEGCC__) && !defined(GC_NO_NOSYS) -#define NOSYS -#define mach_type_known -#endif -#endif -#if defined(sun) && defined(mc68000) && !defined(CPPCHECK) -#error SUNOS4 no longer supported -#endif -#if defined(hp9000s300) && !defined(CPPCHECK) -#error M68K based HP machines no longer supported -#endif -#if defined(vax) || defined(__vax__) -#define VAX -#ifdef ultrix -#define ULTRIX -#else -#define BSD -#endif -#define mach_type_known -#endif -#if defined(NETBSD) && defined(__vax__) -#define VAX -#define mach_type_known -#endif -#if (defined(mips) || defined(__mips) || defined(_mips)) && !defined(__TANDEM) && !defined(LINUX) && !defined(FREEBSD) && !defined(NETBSD) && !defined(OPENBSD) -#define MIPS -#if defined(nec_ews) || defined(_nec_ews) -#define EWS4800 -#define mach_type_known -#elif defined(ultrix) || defined(__ultrix) -#define ULTRIX -#define mach_type_known -#elif !defined(_WIN32_WCE) && !defined(__CEGCC__) && !defined(__MINGW32CE__) -#define IRIX5 /* or IRIX 6.X */ -#define mach_type_known -#endif /* !MSWINCE */ -#endif -#if defined(DGUX) && (defined(i386) || defined(__i386__)) -#define I386 -#ifndef _USING_DGUX -#define _USING_DGUX -#endif -#define mach_type_known -#endif -#if defined(sequent) && (defined(i386) || defined(__i386__)) -#define I386 -#define SEQUENT -#define mach_type_known -#endif -#if (defined(sun) || defined(__sun)) && (defined(i386) || defined(__i386__)) -#define I386 -#define SOLARIS -#define mach_type_known -#endif -#if (defined(sun) || defined(__sun)) && defined(__amd64) -#define X86_64 -#define SOLARIS -#define mach_type_known -#endif -#if (defined(__OS2__) || defined(__EMX__)) && defined(__32BIT__) -#define I386 -#define OS2 -#define mach_type_known -#endif -#if defined(ibm032) && !defined(CPPCHECK) -#error IBM PC/RT no longer supported -#endif -#if (defined(sun) || defined(__sun)) && (defined(sparc) || defined(__sparc)) -/* Test for SunOS 5.x */ -EXTERN_C_END -#include -EXTERN_C_BEGIN -#define SPARC -#define SOLARIS -#define mach_type_known -#elif defined(sparc) && defined(unix) && !defined(sun) && !defined(linux) && !defined(FREEBSD) && !defined(NETBSD) && !defined(OPENBSD) -#define SPARC -#define DRSNX -#define mach_type_known -#endif -#if defined(_IBMR2) -#define POWERPC -#define AIX -#define mach_type_known -#endif -#if defined(_M_XENIX) && defined(_M_SYSV) && defined(_M_I386) -/* TODO: The above test may need refinement. */ -#define I386 -#if defined(_SCO_ELF) -#define SCO_ELF -#else -#define SCO -#endif -#define mach_type_known -#endif -#if defined(_AUX_SOURCE) && !defined(CPPCHECK) -#error A/UX no longer supported -#endif -#if defined(_PA_RISC1_0) || defined(_PA_RISC1_1) || defined(_PA_RISC2_0) || defined(hppa) || defined(__hppa__) -#define HP_PA -#if !defined(LINUX) && !defined(HPUX) && !defined(OPENBSD) -#define HPUX -#endif -#define mach_type_known -#endif -#if defined(__ia64) && (defined(_HPUX_SOURCE) || defined(__HP_aCC)) -#define IA64 -#ifndef HPUX -#define HPUX -#endif -#define mach_type_known -#endif -#if (defined(__BEOS__) || defined(__HAIKU__)) && defined(_X86_) -#define I386 -#define HAIKU -#define mach_type_known -#endif -#if defined(__HAIKU__) && (defined(__amd64__) || defined(__x86_64__)) -#define X86_64 -#define HAIKU -#define mach_type_known -#endif -#if defined(__alpha) || defined(__alpha__) -#define ALPHA -#if !defined(LINUX) && !defined(NETBSD) && !defined(OPENBSD) && !defined(FREEBSD) -#define OSF1 /* a.k.a Digital Unix */ -#endif -#define mach_type_known -#endif -#if defined(_AMIGA) && !defined(AMIGA) -#define AMIGA -#endif -#ifdef AMIGA -#define M68K -#define mach_type_known -#endif -#if defined(THINK_C) || (defined(__MWERKS__) && !defined(__powerc) && !defined(SYMBIAN)) -#define M68K -#define MACOS -#define mach_type_known -#endif -#if defined(__MWERKS__) && defined(__powerc) && !defined(__MACH__) && !defined(SYMBIAN) -#define POWERPC -#define MACOS -#define mach_type_known -#endif -#if defined(__rtems__) && (defined(i386) || defined(__i386__)) -#define I386 -#define RTEMS -#define mach_type_known -#endif -#if defined(NeXT) && defined(mc68000) -#define M68K -#define NEXT -#define mach_type_known -#endif -#if defined(NeXT) && (defined(i386) || defined(__i386__)) -#define I386 -#define NEXT -#define mach_type_known -#endif -#if defined(bsdi) && (defined(i386) || defined(__i386__)) -#define I386 -#define BSDI -#define mach_type_known -#endif -#if defined(__386BSD__) && !defined(mach_type_known) -#define I386 -#define THREE86BSD -#define mach_type_known -#endif -#if defined(_CX_UX) && defined(_M88K) -#define M88K -#define CX_UX -#define mach_type_known -#endif -#if defined(DGUX) && defined(m88k) -#define M88K -/* DGUX defined */ -#define mach_type_known -#endif -#if defined(_WIN32_WCE) || defined(__CEGCC__) || defined(__MINGW32CE__) -/* SH3, SH4, MIPS already defined for corresponding architectures */ -#if defined(SH3) || defined(SH4) -#define SH -#endif -#if defined(x86) || defined(__i386__) -#define I386 -#endif -#if defined(_M_ARM) || defined(ARM) || defined(_ARM_) -#define ARM32 -#endif -#define MSWINCE -#define mach_type_known -#else -#if ((defined(_MSDOS) || defined(_MSC_VER)) && (_M_IX86 >= 300)) || (defined(_WIN32) && !defined(__CYGWIN32__) && !defined(__CYGWIN__) && !defined(__INTERIX) && !defined(SYMBIAN)) -#if defined(__LP64__) || defined(_M_X64) -#define X86_64 -#elif defined(_M_ARM) -#define ARM32 -#elif defined(_M_ARM64) -#define AARCH64 -#else /* _M_IX86 */ -#define I386 -#endif -#ifdef _XBOX_ONE -#define MSWIN_XBOX1 -#else -#ifndef MSWIN32 -#define MSWIN32 /* or Win64 */ -#endif -#if defined(WINAPI_FAMILY) && (WINAPI_FAMILY == WINAPI_FAMILY_APP) -#define MSWINRT_FLAVOR -#endif -#endif -#define mach_type_known -#endif -#if defined(_MSC_VER) && defined(_M_IA64) -#define IA64 -#define MSWIN32 /* Really Win64, but we do not treat 64-bit */ -/* variants as a different platform. */ -#endif -#endif /* !_WIN32_WCE && !__CEGCC__ && !__MINGW32CE__ */ -#if defined(__DJGPP__) -#define I386 -#ifndef DJGPP -#define DJGPP /* MSDOS running the DJGPP port of GCC */ -#endif -#define mach_type_known -#endif -#if defined(__CYGWIN32__) || defined(__CYGWIN__) -#if defined(__LP64__) -#define X86_64 -#else -#define I386 -#endif -#define CYGWIN32 -#define mach_type_known -#endif /* __CYGWIN__ */ -#if defined(__INTERIX) -#define I386 -#define INTERIX -#define mach_type_known -#endif -#if (defined(__BORLANDC__) || defined(__MINGW32__)) && !defined(mach_type_known) -#define I386 -#define MSWIN32 -#define mach_type_known -#endif -#if defined(_UTS) && !defined(mach_type_known) -#define S370 -#define UTS4 -#define mach_type_known -#endif -#if defined(__pj__) && !defined(CPPCHECK) -#error PicoJava no longer supported -/* The implementation had problems, and I haven't heard of users */ -/* in ages. If you want it resurrected, let me know. */ -#endif -#if defined(__embedded__) && defined(PPC) -#define POWERPC -#define NOSYS -#define mach_type_known -#endif -#if defined(__WATCOMC__) && defined(__386__) -#define I386 -#if !defined(OS2) && !defined(MSWIN32) && !defined(DOS4GW) -#if defined(__OS2__) -#define OS2 -#else -#if defined(__WINDOWS_386__) || defined(__NT__) -#define MSWIN32 -#else -#define DOS4GW -#endif -#endif -#endif -#define mach_type_known -#endif /* __WATCOMC__ */ -#if defined(__i386__) && defined(__GNU__) -/* The Debian Hurd running on generic PC */ -#define HURD -#define I386 -#define mach_type_known -#endif -#if defined(__TANDEM) -/* Nonstop S-series */ -/* FIXME: Should recognize Integrity series? */ -#define MIPS -#define NONSTOP -#define mach_type_known -#endif -#if defined(__tile__) && defined(LINUX) -#ifdef __tilegx__ -#define TILEGX -#else -#define TILEPRO -#endif -#define mach_type_known -#endif /* __tile__ */ -#if defined(NN_BUILD_TARGET_PLATFORM_NX) -#define AARCH64 -#define NINTENDO_SWITCH -#define mach_type_known -#endif -#if defined(SYMBIAN) -#define mach_type_known -#endif -#if defined(__EMSCRIPTEN__) || defined(EMSCRIPTEN) -#define WEBASSEMBLY -#ifndef EMSCRIPTEN -#define EMSCRIPTEN -#endif -#define mach_type_known -#endif -#if defined(__wasi__) -#define WEBASSEMBLY -#define WASI -#define mach_type_known -#endif - -#if defined(__aarch64__) && (defined(DARWIN) || defined(LINUX) || defined(FREEBSD) || defined(NETBSD) || defined(OPENBSD) || defined(QNX)) -#define AARCH64 -#define mach_type_known -#elif defined(__arc__) && defined(LINUX) -#define ARC -#define mach_type_known -#elif (defined(__arm) || defined(__arm__) || defined(__arm32__) || defined(__ARM__)) && (defined(DARWIN) || defined(LINUX) || defined(FREEBSD) || defined(NETBSD) || defined(OPENBSD) || defined(QNX) || defined(NN_PLATFORM_CTR) || defined(SN_TARGET_PSP2)) -#define ARM32 -#define mach_type_known -#elif defined(__avr32__) && defined(LINUX) -#define AVR32 -#define mach_type_known -#elif defined(__cris__) && defined(LINUX) -#ifndef CRIS -#define CRIS -#endif -#define mach_type_known -#elif defined(__e2k__) && defined(LINUX) -#define E2K -#define mach_type_known -#elif defined(__hexagon__) && defined(LINUX) -#define HEXAGON -#define mach_type_known -#elif (defined(__i386__) || defined(i386) || defined(__X86__)) && (defined(DARWIN) || defined(LINUX) || defined(FREEBSD) || defined(NETBSD) || defined(OPENBSD) || defined(QNX)) -#define I386 -#define mach_type_known -#elif (defined(__ia64) || defined(__ia64__)) && defined(LINUX) -#define IA64 -#define mach_type_known -#elif defined(__loongarch__) && defined(LINUX) -#define LOONGARCH -#define mach_type_known -#elif defined(__m32r__) && defined(LINUX) -#define M32R -#define mach_type_known -#elif ((defined(__m68k__) || defined(m68k)) && (defined(NETBSD) || defined(OPENBSD))) || (defined(__mc68000__) && defined(LINUX)) -#define M68K -#define mach_type_known -#elif (defined(__mips) || defined(_mips) || defined(mips)) && (defined(LINUX) || defined(FREEBSD) || defined(NETBSD) || defined(OPENBSD)) -#define MIPS -#define mach_type_known -#elif (defined(__NIOS2__) || defined(__NIOS2) || defined(__nios2__)) && defined(LINUX) -#define NIOS2 /* Altera NIOS2 */ -#define mach_type_known -#elif defined(__or1k__) && defined(LINUX) -#define OR1K /* OpenRISC (or1k) */ -#define mach_type_known -#elif (defined(__powerpc__) || defined(__powerpc64__) || defined(__ppc__) || defined(__ppc64__) || defined(powerpc) || defined(powerpc64)) && (defined(DARWIN) || defined(LINUX) || defined(FREEBSD) || defined(NETBSD) || defined(OPENBSD)) -#define POWERPC -#define mach_type_known -#elif defined(__riscv) && (defined(LINUX) || defined(FREEBSD) || defined(OPENBSD)) -#define RISCV -#define mach_type_known -#elif defined(__s390__) && defined(LINUX) -#define S390 -#define mach_type_known -#elif defined(__sh__) && (defined(LINUX) || defined(NETBSD) || defined(OPENBSD)) -#define SH -#define mach_type_known -#elif (defined(__sparc__) || defined(sparc)) && (defined(LINUX) || defined(FREEBSD) || defined(NETBSD) || defined(OPENBSD)) -#define SPARC -#define mach_type_known -#elif defined(__sw_64__) && defined(LINUX) -#define SW_64 -#define mach_type_known -#elif (defined(__x86_64) || defined(__x86_64__) || defined(__amd64__) || defined(__X86_64__)) && (defined(DARWIN) || defined(LINUX) || defined(FREEBSD) || defined(NETBSD) || defined(OPENBSD) || defined(QNX)) -#define X86_64 -#define mach_type_known -#endif - -/* Feel free to add more clauses here. Or manually define the machine */ -/* type here. A machine type is characterized by the architecture. */ -/* Some machine types are further subdivided by OS. Macros such as */ -/* LINUX, FREEBSD, etc. distinguish them. The distinction in these */ -/* cases is usually the stack starting address. */ - -#if !defined(mach_type_known) && !defined(CPPCHECK) -#error The collector has not been ported to this machine/OS combination -#endif - -/* Mapping is: M68K ==> Motorola 680X0 */ -/* (NEXT, and SYSV (A/UX), */ -/* MACOS and AMIGA variants) */ -/* I386 ==> Intel 386 */ -/* (SEQUENT, OS2, SCO, LINUX, NETBSD, */ -/* FREEBSD, THREE86BSD, MSWIN32, */ -/* BSDI, SOLARIS, NEXT and others) */ -/* NS32K ==> Encore Multimax */ -/* MIPS ==> R2000 through R14K */ -/* (many variants) */ -/* VAX ==> DEC VAX */ -/* (BSD, ULTRIX variants) */ -/* HP_PA ==> HP9000/700 & /800 */ -/* HP/UX, LINUX */ -/* SPARC ==> SPARC v7/v8/v9 */ -/* (SOLARIS, LINUX, DRSNX variants) */ -/* ALPHA ==> DEC Alpha */ -/* (OSF1 and LINUX variants) */ -/* LOONGARCH ==> Loongson LoongArch */ -/* (LINUX 32- and 64-bit variants) */ -/* M88K ==> Motorola 88XX0 */ -/* (CX_UX and DGUX) */ -/* S370 ==> 370-like machine */ -/* running Amdahl UTS4 */ -/* S390 ==> 390-like machine */ -/* running LINUX */ -/* AARCH64 ==> ARM AArch64 */ -/* (LP64 and ILP32 variants) */ -/* E2K ==> Elbrus 2000 */ -/* running LINUX */ -/* ARM32 ==> Intel StrongARM */ -/* (many variants) */ -/* IA64 ==> Intel IPF */ -/* (e.g. Itanium) */ -/* (LINUX and HPUX) */ -/* SH ==> Hitachi SuperH */ -/* (LINUX & MSWINCE) */ -/* SW_64 ==> Sunway (Shenwei) */ -/* running LINUX */ -/* X86_64 ==> AMD x86-64 */ -/* POWERPC ==> IBM/Apple PowerPC */ -/* (MACOS(<=9),DARWIN(incl.MACOSX),*/ -/* LINUX, NETBSD, AIX, NOSYS */ -/* variants) */ -/* Handles 32 and 64-bit variants. */ -/* ARC ==> Synopsys ARC */ -/* AVR32 ==> Atmel RISC 32-bit */ -/* CRIS ==> Axis Etrax */ -/* M32R ==> Renesas M32R */ -/* NIOS2 ==> Altera NIOS2 */ -/* HEXAGON ==> Qualcomm Hexagon */ -/* OR1K ==> OpenRISC/or1k */ -/* RISCV ==> RISC-V 32/64-bit */ -/* TILEPRO ==> Tilera TILEPro */ -/* TILEGX ==> Tilera TILE-Gx */ - -/* - * For each architecture and OS, the following need to be defined: - * - * CPP_WORDSZ is a simple integer constant representing the word size. - * in bits. We assume byte addressability, where a byte has 8 bits. - * We also assume CPP_WORDSZ is either 32 or 64. - * (We care about the length of pointers, not hardware - * bus widths. Thus a 64 bit processor with a C compiler that uses - * 32 bit pointers should use CPP_WORDSZ of 32, not 64. Default is 32.) - * - * MACH_TYPE is a string representation of the machine type. - * OS_TYPE is analogous for the OS. - * - * ALIGNMENT is the largest N, such that - * all pointer are guaranteed to be aligned on N byte boundaries. - * defining it to be 1 will always work, but perform poorly. - * - * DATASTART is the beginning of the data segment. - * On some platforms SEARCH_FOR_DATA_START is defined. - * The latter will cause GC_data_start to - * be set to an address determined by accessing data backwards from _end - * until an unmapped page is found. DATASTART will be defined to be - * GC_data_start. - * On UNIX-like systems, the collector will scan the area between DATASTART - * and DATAEND for root pointers. - * - * DATAEND, if not "end", where "end" is defined as "extern int end[]". - * RTH suggests gaining access to linker script synth'd values with - * this idiom instead of "&end", where "end" is defined as "extern int end". - * Otherwise, "GCC will assume these are in .sdata/.sbss" and it will, e.g., - * cause failures on alpha*-*-* with -msmall-data or -fpic or mips-*-* - * without any special options. - * - * STACKBOTTOM is the cold end of the stack, which is usually the - * highest address in the stack. - * Under PCR or OS/2, we have other ways of finding thread stacks. - * For each machine, the following should: - * 1) define STACK_GROWS_UP if the stack grows toward higher addresses, and - * 2) define exactly one of - * STACKBOTTOM (should be defined to be an expression) - * LINUX_STACKBOTTOM - * HEURISTIC1 - * HEURISTIC2 - * If STACKBOTTOM is defined, then its value will be used directly (as the - * stack bottom). If LINUX_STACKBOTTOM is defined, then it will be determined - * with a method appropriate for most Linux systems. Currently we look - * first for __libc_stack_end (currently only if USE_LIBC_PRIVATES is - * defined), and if that fails read it from /proc. (If USE_LIBC_PRIVATES - * is not defined and NO_PROC_STAT is defined, we revert to HEURISTIC2.) - * If either of the last two macros are defined, then STACKBOTTOM is computed - * during collector startup using one of the following two heuristics: - * HEURISTIC1: Take an address inside GC_init's frame, and round it up to - * the next multiple of STACK_GRAN. - * HEURISTIC2: Take an address inside GC_init's frame, increment it repeatedly - * in small steps (decrement if STACK_GROWS_UP), and read the value - * at each location. Remember the value when the first - * Segmentation violation or Bus error is signaled. Round that - * to the nearest plausible page boundary, and use that instead - * of STACKBOTTOM. - * - * Gustavo Rodriguez-Rivera points out that on most (all?) Unix machines, - * the value of environ is a pointer that can serve as STACKBOTTOM. - * I expect that HEURISTIC2 can be replaced by this approach, which - * interferes far less with debugging. However it has the disadvantage - * that it's confused by a putenv call before the collector is initialized. - * This could be dealt with by intercepting putenv ... - * - * If no expression for STACKBOTTOM can be found, and neither of the above - * heuristics are usable, the collector can still be used with all of the above - * undefined, provided one of the following is done: - * 1) GC_mark_roots can be changed to somehow mark from the correct stack(s) - * without reference to STACKBOTTOM. This is appropriate for use in - * conjunction with thread packages, since there will be multiple stacks. - * (Allocating thread stacks in the heap, and treating them as ordinary - * heap data objects is also possible as a last resort. However, this is - * likely to introduce significant amounts of excess storage retention - * unless the dead parts of the thread stacks are periodically cleared.) - * 2) Client code may set GC_stackbottom before calling any GC_ routines. - * If the author of the client code controls the main program, this is - * easily accomplished by introducing a new main program, setting - * GC_stackbottom to the address of a local variable, and then calling - * the original main program. The new main program would read something - * like (provided real_main() is not inlined by the compiler): - * - * #include "gc/gc.h" - * - * main(argc, argv, envp) - * int argc; - * char **argv, **envp; - * { - * volatile int dummy; - * - * GC_stackbottom = (ptr_t)(&dummy); - * return real_main(argc, argv, envp); - * } - * - * - * Each architecture may also define the style of virtual dirty bit - * implementation to be used: - * GWW_VDB: Use Win32 GetWriteWatch primitive. - * MPROTECT_VDB: Write protect the heap and catch faults. - * PROC_VDB: Use the SVR4 /proc primitives to read dirty bits. - * SOFT_VDB: Use the Linux /proc primitives to track dirty bits. - * - * The first and second one may be combined, in which case a runtime - * selection will be made, based on GetWriteWatch availability. - * - * An architecture may define DYNAMIC_LOADING if dyn_load.c - * defined GC_register_dynamic_libraries() for the architecture. - * - * An architecture may define PREFETCH(x) to preload the cache with *x. - * This defaults to GCC built-in operation (or a no-op for other compilers). - * - * GC_PREFETCH_FOR_WRITE(x) is used if *x is about to be written. - * - * An architecture may also define CLEAR_DOUBLE(x) to be a fast way to - * clear the two words at GC_malloc-aligned address x. By default, - * word stores of 0 are used instead. - * - * HEAP_START may be defined as the initial address hint for mmap-based - * allocation. - */ - -#ifdef LINUX /* TODO: FreeBSD too? */ -EXTERN_C_END -#include /* for __GLIBC__ and __GLIBC_MINOR__, at least */ -EXTERN_C_BEGIN -#endif - -/* Convenient internal macro to test glibc version (if compiled against). */ -#if defined(__GLIBC__) && defined(__GLIBC_MINOR__) -#define GC_GLIBC_PREREQ(major, minor) \ - ((__GLIBC__ << 16) + __GLIBC_MINOR__ >= ((major) << 16) + (minor)) -#else -#define GC_GLIBC_PREREQ(major, minor) 0 /* FALSE */ -#endif - -/* If available, we can use __builtin_unwind_init() to push the */ -/* relevant registers onto the stack. */ -#if GC_GNUC_PREREQ(2, 8) && !GC_GNUC_PREREQ(11, 0) /* broken at least in 11.2.0 on cygwin64 */ \ - && !defined(__INTEL_COMPILER) && !defined(__PATHCC__) && !defined(__FUJITSU) /* for FX10 system */ \ - && !(defined(POWERPC) && defined(DARWIN)) /* for MacOS X 10.3.9 */ \ - && !defined(E2K) && !defined(RTEMS) && !defined(__ARMCC_VERSION) /* does not exist in armcc gnu emu */ \ - && (!defined(__clang__) || GC_CLANG_PREREQ(8, 0) /* was no-op in clang-3 at least */) -#define HAVE_BUILTIN_UNWIND_INIT -#endif - -/* The common OS-specific definitions (should be applicable to */ -/* all (or most, at least) supported architectures). */ - -#ifdef CYGWIN32 -#define OS_TYPE "CYGWIN32" -#define RETRY_GET_THREAD_CONTEXT -#ifdef USE_WINALLOC -#define GWW_VDB -#elif defined(USE_MMAP) -#define USE_MMAP_ANON -#endif -#endif /* CYGWIN32 */ - -#ifdef DARWIN -#define OS_TYPE "DARWIN" -#define DYNAMIC_LOADING -/* TODO: see get_end(3), get_etext() and get_end() should not be used. */ -/* These aren't used when dyld support is enabled (it is by default). */ -#define DATASTART ((ptr_t)get_etext()) -#define DATAEND ((ptr_t)get_end()) -#define USE_MMAP_ANON -/* There seems to be some issues with trylock hanging on darwin. */ -/* TODO: This should be looked into some more. */ -#define NO_PTHREAD_TRYLOCK -#endif /* DARWIN */ - -#ifdef FREEBSD -#define OS_TYPE "FREEBSD" -#define FREEBSD_STACKBOTTOM -#ifdef __ELF__ -#define DYNAMIC_LOADING -#endif -#if !defined(ALPHA) && !defined(SPARC) -extern char etext[]; -#define DATASTART GC_FreeBSDGetDataStart(0x1000, (ptr_t)etext) -#define DATASTART_USES_BSDGETDATASTART -#ifndef GC_FREEBSD_THREADS -#define MPROTECT_VDB -#endif -#endif -#endif /* FREEBSD */ - -#ifdef HAIKU -#define OS_TYPE "HAIKU" -#define DYNAMIC_LOADING -#define MPROTECT_VDB -EXTERN_C_END -#include -EXTERN_C_BEGIN -#define GETPAGESIZE() (unsigned)B_PAGE_SIZE -#endif /* HAIKU */ - -#ifdef HPUX -#define OS_TYPE "HPUX" -extern int __data_start[]; -#define DATASTART ((ptr_t)(__data_start)) -#ifdef USE_MMAP -#define USE_MMAP_ANON -#endif -#define DYNAMIC_LOADING -EXTERN_C_END -#include -EXTERN_C_BEGIN -#define GETPAGESIZE() (unsigned)sysconf(_SC_PAGE_SIZE) -#endif /* HPUX */ - -#ifdef LINUX -#define OS_TYPE "LINUX" -#if defined(FORCE_MPROTECT_BEFORE_MADVISE) || defined(PREFER_MMAP_PROT_NONE) -#define COUNT_UNMAPPED_REGIONS -#endif -#define RETRY_TKILL_ON_EAGAIN -#if !defined(MIPS) && !defined(POWERPC) -#define LINUX_STACKBOTTOM -#endif -#if defined(__ELF__) && !defined(IA64) -#define DYNAMIC_LOADING -#endif -#if defined(__ELF__) && !defined(ARC) && !defined(RISCV) && !defined(S390) && !defined(TILEGX) && !defined(TILEPRO) -extern int _end[]; -#define DATAEND ((ptr_t)(_end)) -#endif -#if !defined(REDIRECT_MALLOC) && !defined(E2K) -/* Requires Linux 2.3.47 or later. */ -#define MPROTECT_VDB -#else -/* We seem to get random errors in the incremental mode, */ -/* possibly because the Linux threads implementation */ -/* itself is a malloc client and cannot deal with the */ -/* signals. fread() uses malloc too. */ -/* In case of e2k, unless -fsemi-spec-ld (or -O0) option */ -/* is passed to gcc (both when compiling libgc and the */ -/* client), a semi-speculative optimization may lead to */ -/* SIGILL (with ILL_ILLOPN si_code) instead of SIGSEGV. */ -#endif -#endif /* LINUX */ - -#ifdef MACOS -#define OS_TYPE "MACOS" -#ifndef __LOWMEM__ -EXTERN_C_END -#include -EXTERN_C_BEGIN -#endif -/* See os_dep.c for details of global data segments. */ -#define STACKBOTTOM ((ptr_t)LMGetCurStackBase()) -#define DATAEND /* not needed */ -#endif /* MACOS */ - -#ifdef MSWIN32 -#define OS_TYPE "MSWIN32" -/* STACKBOTTOM and DATASTART are handled specially in os_dep.c. */ -#define DATAEND /* not needed */ -#define GWW_VDB -#endif - -#ifdef MSWINCE -#define OS_TYPE "MSWINCE" -#define DATAEND /* not needed */ -#endif - -#ifdef NETBSD -#define OS_TYPE "NETBSD" -#define HEURISTIC2 -#ifdef __ELF__ -#define SEARCH_FOR_DATA_START -#define DYNAMIC_LOADING -#elif !defined(MIPS) /* TODO: probably do not exclude it */ -extern char etext[]; -#define DATASTART ((ptr_t)(etext)) -#endif -#endif /* NETBSD */ - -#ifdef NEXT -#define OS_TYPE "NEXT" -#define DATASTART ((ptr_t)get_etext()) -#define DATASTART_IS_FUNC -#define DATAEND /* not needed */ -#endif - -#ifdef OPENBSD -#define OS_TYPE "OPENBSD" -#ifndef GC_OPENBSD_THREADS -#define HEURISTIC2 -#endif -#ifdef __ELF__ -extern int __data_start[]; -#define DATASTART ((ptr_t)__data_start) -extern int _end[]; -#define DATAEND ((ptr_t)(&_end)) -#define DYNAMIC_LOADING -#else -extern char etext[]; -#define DATASTART ((ptr_t)(etext)) -#endif -#endif /* OPENBSD */ - -#ifdef QNX -#define OS_TYPE "QNX" -#define SA_RESTART 0 -#ifndef QNX_STACKBOTTOM /* TODO: not the default one for now */ -#define HEURISTIC1 -#endif -extern char etext[]; -#define DATASTART ((ptr_t)etext) -extern int _end[]; -#define DATAEND ((ptr_t)_end) -#endif /* QNX */ - -#ifdef SOLARIS -#define OS_TYPE "SOLARIS" -extern int _etext[], _end[]; -ptr_t GC_SysVGetDataStart(size_t, ptr_t); -#define DATASTART_IS_FUNC -#define DATAEND ((ptr_t)(_end)) -#if !defined(USE_MMAP) && defined(REDIRECT_MALLOC) -#define USE_MMAP 1 -/* Otherwise we now use calloc. Mmap may result in the */ -/* heap interleaved with thread stacks, which can result in */ -/* excessive blacklisting. Sbrk is unusable since it */ -/* doesn't interact correctly with the system malloc. */ -#endif -#ifdef USE_MMAP -#define HEAP_START (ptr_t)0x40000000 -#else -#define HEAP_START DATAEND -#endif -#ifndef GC_THREADS -#define MPROTECT_VDB -#endif -#define DYNAMIC_LOADING -/* Define STACKBOTTOM as (ptr_t)_start worked through 2.7, */ -/* but reportedly breaks under 2.8. It appears that the stack */ -/* base is a property of the executable, so this should not */ -/* break old executables. */ -/* HEURISTIC1 reportedly no longer works under Solaris 2.7. */ -/* HEURISTIC2 probably works, but this appears to be preferable.*/ -/* Apparently USRSTACK is defined to be USERLIMIT, but in some */ -/* installations that's undefined. We work around this with a */ -/* gross hack: */ -EXTERN_C_END -#include -#include -EXTERN_C_BEGIN -#ifdef USERLIMIT -/* This should work everywhere, but doesn't. */ -#define STACKBOTTOM ((ptr_t)USRSTACK) -#else -#define HEURISTIC2 -#endif -#endif /* SOLARIS */ - -#define STACK_GRAN 0x1000000 - -#ifdef SYMBIAN -#define MACH_TYPE "SYMBIAN" -#define OS_TYPE "SYMBIAN" -#define CPP_WORDSZ 32 -#define DATASTART (ptr_t) ALIGNMENT /* cannot be null */ -#define DATAEND (ptr_t) ALIGNMENT -#endif - -#ifdef M68K -#define MACH_TYPE "M68K" -#define ALIGNMENT 2 -#ifdef OPENBSD -/* Nothing specific. */ -#endif -#ifdef NETBSD -/* Nothing specific. */ -#endif -#ifdef LINUX -#ifdef __ELF__ -#if GC_GLIBC_PREREQ(2, 0) -#define SEARCH_FOR_DATA_START -#else -extern char **__environ; -#define DATASTART ((ptr_t)(&__environ)) -/* hideous kludge: __environ is the first */ -/* word in crt0.o, and delimits the start */ -/* of the data segment, no matter which */ -/* ld options were passed through. */ -/* We could use _etext instead, but that */ -/* would include .rodata, which may */ -/* contain large read-only data tables */ -/* that we'd rather not scan. */ -#endif -#else -extern int etext[]; -#define DATASTART ((ptr_t)((((word)(etext)) + 0xfff) & ~0xfff)) -#endif -#endif -#ifdef AMIGA -#define OS_TYPE "AMIGA" -/* STACKBOTTOM and DATASTART handled specially */ -/* in os_dep.c */ -#define DATAEND /* not needed */ -#define GETPAGESIZE() 4096 -#endif -#ifdef MACOS -#define GETPAGESIZE() 4096 -#endif -#ifdef NEXT -#define STACKBOTTOM ((ptr_t)0x4000000) -#endif -#endif - -#ifdef POWERPC -#define MACH_TYPE "POWERPC" -#ifdef MACOS -#define ALIGNMENT 2 /* Still necessary? Could it be 4? */ -#endif -#ifdef LINUX -#if defined(__powerpc64__) -#define CPP_WORDSZ 64 -#ifndef HBLKSIZE -#define HBLKSIZE 4096 -#endif -#else -#define ALIGNMENT 4 -#endif -/* HEURISTIC1 has been reliably reported to fail for a 32-bit */ -/* executable on a 64 bit kernel. */ -#if defined(__bg__) -/* The Linux Compute Node Kernel (used on BlueGene systems) */ -/* does not support LINUX_STACKBOTTOM way. */ -#define HEURISTIC2 -#define NO_PTHREAD_GETATTR_NP -#else -#define LINUX_STACKBOTTOM -#endif -#define SEARCH_FOR_DATA_START -#ifndef SOFT_VDB -#define SOFT_VDB -#endif -#endif -#ifdef DARWIN -#if defined(__ppc64__) -#define CPP_WORDSZ 64 -#define STACKBOTTOM ((ptr_t)0x7fff5fc00000) -#define CACHE_LINE_SIZE 64 -#ifndef HBLKSIZE -#define HBLKSIZE 4096 -#endif -#else -#define ALIGNMENT 4 -#define STACKBOTTOM ((ptr_t)0xc0000000) -#endif -#define MPROTECT_VDB -#if defined(USE_PPC_PREFETCH) && defined(__GNUC__) -/* The performance impact of prefetches is untested */ -#define PREFETCH(x) \ - __asm__ __volatile__("dcbt 0,%0" \ - : \ - : "r"((const void *)(x))) -#define GC_PREFETCH_FOR_WRITE(x) \ - __asm__ __volatile__("dcbtst 0,%0" \ - : \ - : "r"((const void *)(x))) -#endif -#endif -#ifdef OPENBSD -#if defined(__powerpc64__) -#define CPP_WORDSZ 64 -#else -#define ALIGNMENT 4 -#endif -#endif -#ifdef FREEBSD -#if defined(__powerpc64__) -#define CPP_WORDSZ 64 -#ifndef HBLKSIZE -#define HBLKSIZE 4096 -#endif -#else -#define ALIGNMENT 4 -#endif -#endif -#ifdef NETBSD -#define ALIGNMENT 4 -#endif -#ifdef SN_TARGET_PS3 -#define OS_TYPE "SN_TARGET_PS3" -#define NO_GETENV -#define CPP_WORDSZ 32 -extern int _end[]; -extern int __bss_start; -#define DATASTART ((ptr_t)(__bss_start)) -#define DATAEND ((ptr_t)(_end)) -#define STACKBOTTOM ((ptr_t)ps3_get_stack_bottom()) -#define NO_PTHREAD_TRYLOCK -/* Current GC LOCK() implementation for PS3 explicitly */ -/* use pthread_mutex_lock for some reason. */ -#endif -#ifdef AIX -#define OS_TYPE "AIX" -#undef ALIGNMENT /* in case it's defined */ -#undef IA64 -/* DOB: some AIX installs stupidly define IA64 in */ -/* /usr/include/sys/systemcfg.h */ -#ifdef __64BIT__ -#define CPP_WORDSZ 64 -#define STACKBOTTOM ((ptr_t)0x1000000000000000) -#else -#define CPP_WORDSZ 32 -#define STACKBOTTOM ((ptr_t)((ulong)&errno)) -#endif -#define USE_MMAP_ANON -/* From AIX linker man page: -_text Specifies the first location of the program. -_etext Specifies the first location after the program. -_data Specifies the first location of the data. -_edata Specifies the first location after the initialized data -_end or end Specifies the first location after all data. -*/ -extern int _data[], _end[]; -#define DATASTART ((ptr_t)((ulong)_data)) -#define DATAEND ((ptr_t)((ulong)_end)) -extern int errno; -#define MPROTECT_VDB -#define DYNAMIC_LOADING -/* For really old versions of AIX, this may have to be removed. */ -#endif -#ifdef NOSYS -#define OS_TYPE "NOSYS" -#define ALIGNMENT 4 -extern void __end[], __dso_handle[]; -#define DATASTART ((ptr_t)__dso_handle) /* OK, that's ugly. */ -#define DATAEND ((ptr_t)(__end)) -/* Stack starts at 0xE0000000 for the simulator. */ -#undef STACK_GRAN -#define STACK_GRAN 0x10000000 -#define HEURISTIC1 -#endif -#endif - -#ifdef NACL -#define OS_TYPE "NACL" -#if defined(__GLIBC__) -#define DYNAMIC_LOADING -#endif -#define DATASTART ((ptr_t)0x10020000) -extern int _end[]; -#define DATAEND ((ptr_t)_end) -#undef STACK_GRAN -#define STACK_GRAN 0x10000 -#define HEURISTIC1 -#define NO_PTHREAD_GETATTR_NP -#define USE_MMAP_ANON -#define GETPAGESIZE() 65536 /* FIXME: Not real page size */ -#define MAX_NACL_GC_THREADS 1024 -#endif - -#ifdef VAX -#define MACH_TYPE "VAX" -#define ALIGNMENT 4 /* Pointers are longword aligned by 4.2 C compiler */ -extern char etext[]; -#define DATASTART ((ptr_t)(etext)) -#ifdef BSD -#define OS_TYPE "BSD" -#define HEURISTIC1 -/* HEURISTIC2 may be OK, but it's hard to test. */ -#endif -#ifdef ULTRIX -#define OS_TYPE "ULTRIX" -#define STACKBOTTOM ((ptr_t)0x7fffc800) -#endif -#endif - -#ifdef SPARC -#define MACH_TYPE "SPARC" -#if defined(__arch64__) || defined(__sparcv9) -#define CPP_WORDSZ 64 -#define ELF_CLASS ELFCLASS64 -#else -#define ALIGNMENT 4 /* Required by hardware */ -#define CPP_WORDSZ 32 -#endif -/* Don't define USE_ASM_PUSH_REGS. We do use an asm helper, but */ -/* not to push the registers on the mark stack. */ -#ifdef SOLARIS -#define DATASTART GC_SysVGetDataStart(0x10000, (ptr_t)_etext) -#define PROC_VDB -#define GETPAGESIZE() (unsigned)sysconf(_SC_PAGESIZE) -/* getpagesize() appeared to be missing from at least */ -/* one Solaris 5.4 installation. Weird. */ -#endif -#ifdef DRSNX -#define OS_TYPE "DRSNX" -extern int etext[]; -ptr_t GC_SysVGetDataStart(size_t, ptr_t); -#define DATASTART GC_SysVGetDataStart(0x10000, (ptr_t)etext) -#define DATASTART_IS_FUNC -#define MPROTECT_VDB -#define STACKBOTTOM ((ptr_t)0xdfff0000) -#define DYNAMIC_LOADING -#endif -#ifdef LINUX -#if !defined(__ELF__) && !defined(CPPCHECK) -#error Linux SPARC a.out not supported -#endif -#define SVR4 -extern int _etext[]; -ptr_t GC_SysVGetDataStart(size_t, ptr_t); -#ifdef __arch64__ -#define DATASTART GC_SysVGetDataStart(0x100000, (ptr_t)_etext) -#else -#define DATASTART GC_SysVGetDataStart(0x10000, (ptr_t)_etext) -#endif -#define DATASTART_IS_FUNC -#endif -#ifdef OPENBSD -/* Nothing specific. */ -#endif -#ifdef NETBSD -/* Nothing specific. */ -#endif -#ifdef FREEBSD -extern char etext[]; -extern char edata[]; -#if !defined(CPPCHECK) -extern char end[]; -#endif -#define NEED_FIND_LIMIT -#define DATASTART ((ptr_t)(&etext)) -void *GC_find_limit(void *, int); -#define DATAEND (ptr_t) GC_find_limit(DATASTART, TRUE) -#define DATAEND_IS_FUNC -#define GC_HAVE_DATAREGION2 -#define DATASTART2 ((ptr_t)(&edata)) -#define DATAEND2 ((ptr_t)(&end)) -#endif -#endif /* SPARC */ - -#ifdef I386 -#define MACH_TYPE "I386" -#if (defined(__LP64__) || defined(_WIN64)) && !defined(CPPCHECK) -#error This should be handled as X86_64 -#else -#define CPP_WORDSZ 32 -#define ALIGNMENT 4 -/* Appears to hold for all "32 bit" compilers */ -/* except Borland. The -a4 option fixes */ -/* Borland. For Watcom the option is -zp4. */ -#endif -#ifdef SEQUENT -#define OS_TYPE "SEQUENT" -extern int etext[]; -#define DATASTART ((ptr_t)((((word)(etext)) + 0xfff) & ~0xfff)) -#define STACKBOTTOM ((ptr_t)0x3ffff000) -#endif -#ifdef HAIKU -extern int etext[]; -#define DATASTART ((ptr_t)((((word)(etext)) + 0xfff) & ~0xfff)) -#endif -#ifdef QNX -/* Nothing specific. */ -#endif -#ifdef SOLARIS -#define DATASTART GC_SysVGetDataStart(0x1000, (ptr_t)_etext) -/* At least in Solaris 2.5, PROC_VDB gives wrong values for */ -/* dirty bits. It appears to be fixed in 2.8 and 2.9. */ -#ifdef SOLARIS25_PROC_VDB_BUG_FIXED -#define PROC_VDB -#endif -#endif -#ifdef SCO -#define OS_TYPE "SCO" -extern int etext[]; -#define DATASTART ((ptr_t)((((word)(etext)) + 0x3fffff) & ~0x3fffff) + ((word)(etext)&0xfff)) -#define STACKBOTTOM ((ptr_t)0x7ffffffc) -#endif -#ifdef SCO_ELF -#define OS_TYPE "SCO_ELF" -extern int etext[]; -#define DATASTART ((ptr_t)(etext)) -#define STACKBOTTOM ((ptr_t)0x08048000) -#define DYNAMIC_LOADING -#define ELF_CLASS ELFCLASS32 -#endif -#ifdef DGUX -#define OS_TYPE "DGUX" -extern int _etext, _end; -ptr_t GC_SysVGetDataStart(size_t, ptr_t); -#define DATASTART GC_SysVGetDataStart(0x1000, (ptr_t)(&_etext)) -#define DATASTART_IS_FUNC -#define DATAEND ((ptr_t)(&_end)) -#define HEURISTIC2 -EXTERN_C_END -#include -EXTERN_C_BEGIN -#define GETPAGESIZE() (unsigned)sysconf(_SC_PAGESIZE) -#define DYNAMIC_LOADING -#ifndef USE_MMAP -#define USE_MMAP 1 -#endif -#define MAP_FAILED (void *)((word)-1) -#define HEAP_START (ptr_t)0x40000000 -#endif /* DGUX */ -#ifdef LINUX -#define HEAP_START (ptr_t)0x1000 -/* This encourages mmap to give us low addresses, */ -/* thus allowing the heap to grow to ~3 GB. */ -#ifdef __ELF__ -#if GC_GLIBC_PREREQ(2, 0) || defined(HOST_ANDROID) -#define SEARCH_FOR_DATA_START -#else -extern char **__environ; -#define DATASTART ((ptr_t)(&__environ)) -/* hideous kludge: __environ is the first */ -/* word in crt0.o, and delimits the start */ -/* of the data segment, no matter which */ -/* ld options were passed through. */ -/* We could use _etext instead, but that */ -/* would include .rodata, which may */ -/* contain large read-only data tables */ -/* that we'd rather not scan. */ -#endif -#if !defined(GC_NO_SIGSETJMP) && (defined(HOST_TIZEN) || (defined(HOST_ANDROID) && !(GC_GNUC_PREREQ(4, 8) || GC_CLANG_PREREQ(3, 2) || __ANDROID_API__ >= 18))) -/* Older Android NDK releases lack sigsetjmp in x86 libc */ -/* (setjmp is used instead to find data_start). The bug */ -/* is fixed in Android NDK r8e (so, ok to use sigsetjmp */ -/* if gcc4.8+, clang3.2+ or Android API level 18+). */ -#define GC_NO_SIGSETJMP 1 -#endif -#else -extern int etext[]; -#define DATASTART ((ptr_t)((((word)(etext)) + 0xfff) & ~0xfff)) -#endif -#ifdef USE_I686_PREFETCH -#define PREFETCH(x) \ - __asm__ __volatile__("prefetchnta %0" \ - : \ - : "m"(*(char *)(x))) -/* Empirically prefetcht0 is much more effective at reducing */ -/* cache miss stalls for the targeted load instructions. But it */ -/* seems to interfere enough with other cache traffic that the */ -/* net result is worse than prefetchnta. */ -#ifdef FORCE_WRITE_PREFETCH -/* Using prefetches for write seems to have a slight negative */ -/* impact on performance, at least for a PIII/500. */ -#define GC_PREFETCH_FOR_WRITE(x) \ - __asm__ __volatile__("prefetcht0 %0" \ - : \ - : "m"(*(char *)(x))) -#else -#define GC_NO_PREFETCH_FOR_WRITE -#endif -#elif defined(USE_3DNOW_PREFETCH) -#define PREFETCH(x) \ - __asm__ __volatile__("prefetch %0" \ - : \ - : "m"(*(char *)(x))) -#define GC_PREFETCH_FOR_WRITE(x) \ - __asm__ __volatile__("prefetchw %0" \ - : \ - : "m"(*(char *)(x))) -#endif -#if defined(__GLIBC__) && !defined(__UCLIBC__) && !defined(GLIBC_TSX_BUG_FIXED) -/* Workaround lock elision implementation for some glibc. */ -#define GLIBC_2_19_TSX_BUG -EXTERN_C_END -#include /* for gnu_get_libc_version() */ -EXTERN_C_BEGIN -#endif -#ifndef SOFT_VDB -#define SOFT_VDB -#endif -#endif -#ifdef CYGWIN32 -#define WOW64_THREAD_CONTEXT_WORKAROUND -#define DATASTART ((ptr_t)GC_DATASTART) /* From gc.h */ -#define DATAEND ((ptr_t)GC_DATAEND) -#ifndef USE_WINALLOC -#/* MPROTECT_VDB does not work, it leads to a spurious exit. */ -#endif -#endif -#ifdef INTERIX -#define OS_TYPE "INTERIX" -extern int _data_start__[]; -extern int _bss_end__[]; -#define DATASTART ((ptr_t)_data_start__) -#define DATAEND ((ptr_t)_bss_end__) -#define STACKBOTTOM ({ ptr_t rv; \ - __asm__ __volatile__ ("movl %%fs:4, %%eax" \ - : "=a" (rv)); \ - rv; }) -#define USE_MMAP_ANON -#endif -#ifdef OS2 -#define OS_TYPE "OS2" -/* STACKBOTTOM and DATASTART are handled specially in */ -/* os_dep.c. OS2 actually has the right */ -/* system call! */ -#define DATAEND /* not needed */ -#define GETPAGESIZE() os2_getpagesize() -#endif -#ifdef MSWIN32 -#define WOW64_THREAD_CONTEXT_WORKAROUND -#define RETRY_GET_THREAD_CONTEXT -#define MPROTECT_VDB -#endif -#ifdef MSWINCE -/* Nothing specific. */ -#endif -#ifdef DJGPP -#define OS_TYPE "DJGPP" -EXTERN_C_END -#include "stubinfo.h" -EXTERN_C_BEGIN -extern int etext[]; -extern int _stklen; -extern int __djgpp_stack_limit; -#define DATASTART ((ptr_t)((((word)(etext)) + 0x1ff) & ~0x1ff)) -/* #define STACKBOTTOM ((ptr_t)((word)_stubinfo+_stubinfo->size+_stklen)) */ -#define STACKBOTTOM ((ptr_t)((word)__djgpp_stack_limit + _stklen)) -/* This may not be right. */ -#endif -#ifdef OPENBSD -/* Nothing specific. */ -#endif -#ifdef FREEBSD -#if defined(__GLIBC__) -extern int _end[]; -#define DATAEND ((ptr_t)(_end)) -#endif -#endif -#ifdef NETBSD -/* Nothing specific. */ -#endif -#ifdef THREE86BSD -#define OS_TYPE "THREE86BSD" -#define HEURISTIC2 -extern char etext[]; -#define DATASTART ((ptr_t)(etext)) -#endif -#ifdef BSDI -#define OS_TYPE "BSDI" -#define HEURISTIC2 -extern char etext[]; -#define DATASTART ((ptr_t)(etext)) -#endif -#ifdef NEXT -#define STACKBOTTOM ((ptr_t)0xc0000000) -#endif -#ifdef RTEMS -#define OS_TYPE "RTEMS" -EXTERN_C_END -#include -EXTERN_C_BEGIN -extern int etext[]; -void *rtems_get_stack_bottom(void); -#define InitStackBottom rtems_get_stack_bottom() -#define DATASTART ((ptr_t)etext) -#define STACKBOTTOM ((ptr_t)InitStackBottom) -#endif -#ifdef DOS4GW -#define OS_TYPE "DOS4GW" -extern long __nullarea; -extern char _end; -extern char *_STACKTOP; -/* Depending on calling conventions Watcom C either precedes */ -/* or does not precedes with underscore names of C-variables. */ -/* Make sure startup code variables always have the same names. */ -#pragma aux __nullarea "*"; -#pragma aux _end "*"; -#define STACKBOTTOM ((ptr_t)_STACKTOP) -/* confused? me too. */ -#define DATASTART ((ptr_t)(&__nullarea)) -#define DATAEND ((ptr_t)(&_end)) -#endif -#ifdef HURD -#define OS_TYPE "HURD" -#define HEURISTIC2 -#define SEARCH_FOR_DATA_START -extern int _end[]; -#define DATAEND ((ptr_t)(_end)) -/* # define MPROTECT_VDB Not quite working yet? */ -#define DYNAMIC_LOADING -#define USE_MMAP_ANON -#endif -#ifdef DARWIN -#define DARWIN_DONT_PARSE_STACK 1 -#define STACKBOTTOM ((ptr_t)0xc0000000) -#define MPROTECT_VDB -#if TARGET_OS_IPHONE && !defined(NO_DYLD_BIND_FULLY_IMAGE) -/* iPhone/iPad simulator */ -#define NO_DYLD_BIND_FULLY_IMAGE -#endif -#endif /* DARWIN */ -#endif - -#ifdef NS32K -#define MACH_TYPE "NS32K" -#define ALIGNMENT 4 -extern char **environ; -#define DATASTART ((ptr_t)(&environ)) -/* hideous kludge: environ is the first */ -/* word in crt0.o, and delimits the start */ -/* of the data segment, no matter which */ -/* ld options were passed through. */ -#define STACKBOTTOM ((ptr_t)0xfffff000) /* for Encore */ -#endif - -#ifdef LOONGARCH -#define MACH_TYPE "LoongArch" -#define CPP_WORDSZ _LOONGARCH_SZPTR -#ifdef LINUX -#pragma weak __data_start -extern int __data_start[]; -#define DATASTART ((ptr_t)(__data_start)) -#endif -#endif /* LOONGARCH */ - -#ifdef SW_64 -#define MACH_TYPE "SW_64" -#define CPP_WORDSZ 64 -#ifdef LINUX -/* Nothing specific. */ -#endif -#endif /* SW_64 */ - -#ifdef MIPS -#define MACH_TYPE "MIPS" -#ifdef LINUX -#pragma weak __data_start -extern int __data_start[]; -#define DATASTART ((ptr_t)(__data_start)) -#ifdef _MIPS_SZPTR -#define CPP_WORDSZ _MIPS_SZPTR -#else -#define ALIGNMENT 4 -#endif -#ifndef HBLKSIZE -#define HBLKSIZE 4096 -#endif -#if GC_GLIBC_PREREQ(2, 2) -#define LINUX_STACKBOTTOM -#else -#define STACKBOTTOM ((ptr_t)0x7fff8000) -#endif -#endif /* Linux */ -#ifdef EWS4800 -#define OS_TYPE "EWS4800" -#define HEURISTIC2 -#if defined(_MIPS_SZPTR) && (_MIPS_SZPTR == 64) -extern int _fdata[], _end[]; -#define DATASTART ((ptr_t)_fdata) -#define DATAEND ((ptr_t)_end) -#define CPP_WORDSZ _MIPS_SZPTR -#else -extern int etext[], edata[]; -#if !defined(CPPCHECK) -extern int end[]; -#endif -extern int _DYNAMIC_LINKING[], _gp[]; -#define DATASTART ((ptr_t)((((word)(etext) + 0x3ffff) & ~0x3ffff) + ((word)(etext)&0xffff))) -#define DATAEND ((ptr_t)(edata)) -#define GC_HAVE_DATAREGION2 -#define DATASTART2 (_DYNAMIC_LINKING \ - ? (ptr_t)(((word)_gp + 0x8000 + 0x3ffff) & ~0x3ffff) \ - : (ptr_t)edata) -#define DATAEND2 ((ptr_t)(end)) -#define ALIGNMENT 4 -#endif -#endif -#ifdef ULTRIX -#define OS_TYPE "ULTRIX" -#define HEURISTIC2 -#define DATASTART ((ptr_t)0x10000000) -/* Could probably be slightly higher since */ -/* startup code allocates lots of stuff. */ -#define ALIGNMENT 4 -#endif -#ifdef IRIX5 -#define OS_TYPE "IRIX5" -#define HEURISTIC2 -extern int _fdata[]; -#define DATASTART ((ptr_t)(_fdata)) -#ifdef USE_MMAP -#define HEAP_START (ptr_t)0x30000000 -#else -#define HEAP_START DATASTART -#endif -/* Lowest plausible heap address. */ -/* In the MMAP case, we map there. */ -/* In either case it is used to identify */ -/* heap sections so they're not */ -/* considered as roots. */ -/*# define MPROTECT_VDB DOB: this should work, but there is evidence */ -/* of recent breakage. */ -#ifdef _MIPS_SZPTR -#define CPP_WORDSZ _MIPS_SZPTR -#else -#define ALIGNMENT 4 -#endif -#define DYNAMIC_LOADING -#endif -#ifdef MSWINCE -#define ALIGNMENT 4 -#endif -#ifdef NETBSD -#define ALIGNMENT 4 -#ifndef __ELF__ -#define DATASTART ((ptr_t)0x10000000) -#define STACKBOTTOM ((ptr_t)0x7ffff000) -#endif -#endif -#ifdef OPENBSD -#define CPP_WORDSZ 64 /* all OpenBSD/mips platforms are 64-bit */ -#endif -#ifdef FREEBSD -#define ALIGNMENT 4 -#endif -#ifdef NONSTOP -#define OS_TYPE "NONSTOP" -#define CPP_WORDSZ 32 -#define DATASTART ((ptr_t)0x08000000) -extern char **environ; -#define DATAEND ((ptr_t)(environ - 0x10)) -#define STACKBOTTOM ((ptr_t)0x4fffffff) -#endif -#endif /* MIPS */ - -#ifdef NIOS2 -#define MACH_TYPE "NIOS2" -#define CPP_WORDSZ 32 -#ifndef HBLKSIZE -#define HBLKSIZE 4096 -#endif -#ifdef LINUX -extern int __data_start[]; -#define DATASTART ((ptr_t)(__data_start)) -#endif -#endif /* NIOS2 */ - -#ifdef OR1K -#define MACH_TYPE "OR1K" -#define CPP_WORDSZ 32 -#ifndef HBLKSIZE -#define HBLKSIZE 4096 -#endif -#ifdef LINUX -extern int __data_start[]; -#define DATASTART ((ptr_t)(__data_start)) -#endif -#endif /* OR1K */ - -#ifdef HP_PA -#define MACH_TYPE "HP_PA" -#ifdef __LP64__ -#define CPP_WORDSZ 64 -#else -#define CPP_WORDSZ 32 -#endif -#define STACK_GROWS_UP -#ifdef HPUX -#ifndef GC_THREADS -#define MPROTECT_VDB -#endif -#ifdef USE_HPUX_FIXED_STACKBOTTOM -/* The following appears to work for 7xx systems running HP/UX */ -/* 9.xx. Furthermore, it might result in much faster */ -/* collections than HEURISTIC2, which may involve scanning */ -/* segments that directly precede the stack. It is not the */ -/* default, since it may not work on older machine/OS */ -/* combinations. (Thanks to Raymond X.T. Nijssen for uncovering */ -/* this.) */ -/* This technique also doesn't work with HP/UX 11.xx. The */ -/* stack size is settable using the kernel maxssiz variable, */ -/* and in 11.23 and latter, the size can be set dynamically. */ -/* It also doesn't handle SHMEM_MAGIC binaries which have */ -/* stack and data in the first quadrant. */ -#define STACKBOTTOM ((ptr_t)0x7b033000) /* from /etc/conf/h/param.h */ -#elif defined(USE_ENVIRON_POINTER) -/* Gustavo Rodriguez-Rivera suggested changing HEURISTIC2 */ -/* to this. Note that the GC must be initialized before the */ -/* first putenv call. Unfortunately, some clients do not obey. */ -extern char **environ; -#define STACKBOTTOM ((ptr_t)environ) -#elif !defined(HEURISTIC2) -/* This uses pst_vm_status support. */ -#define HPUX_MAIN_STACKBOTTOM -#endif -#ifndef __GNUC__ -#define PREFETCH(x) \ - do { \ - register long addr = (long)(x); \ - (void)_asm("LDW", 0, 0, addr, 0); \ - } while (0) -#endif -#endif /* HPUX */ -#ifdef LINUX -#define SEARCH_FOR_DATA_START -#endif -#ifdef OPENBSD -/* Nothing specific. */ -#endif -#endif /* HP_PA */ - -#ifdef ALPHA -#define MACH_TYPE "ALPHA" -#define CPP_WORDSZ 64 -#ifdef NETBSD -#define ELFCLASS32 32 -#define ELFCLASS64 64 -#define ELF_CLASS ELFCLASS64 -#endif -#ifdef OPENBSD -/* Nothing specific. */ -#endif -#ifdef FREEBSD -/* MPROTECT_VDB is not yet supported at all on FreeBSD/alpha. */ -/* Handle unmapped hole alpha*-*-freebsd[45]* puts between etext and edata. */ -extern char etext[]; -extern char edata[]; -#if !defined(CPPCHECK) -extern char end[]; -#endif -#define NEED_FIND_LIMIT -#define DATASTART ((ptr_t)(&etext)) -void *GC_find_limit(void *, int); -#define DATAEND (ptr_t) GC_find_limit(DATASTART, TRUE) -#define DATAEND_IS_FUNC -#define GC_HAVE_DATAREGION2 -#define DATASTART2 ((ptr_t)(&edata)) -#define DATAEND2 ((ptr_t)(&end)) -#endif -#ifdef OSF1 -#define OS_TYPE "OSF1" -#define DATASTART ((ptr_t)0x140000000) -extern int _end[]; -#define DATAEND ((ptr_t)(&_end)) -extern char **environ; -EXTERN_C_END -#include -EXTERN_C_BEGIN -/* round up from the value of environ to the nearest page boundary */ -/* Probably breaks if putenv is called before collector */ -/* initialization. */ -#define STACKBOTTOM ((ptr_t)(((word)(environ) | (getpagesize() - 1)) + 1)) -/* #define HEURISTIC2 */ -/* Normally HEURISTIC2 is too conservative, since */ -/* the text segment immediately follows the stack. */ -/* Hence we give an upper pound. */ -/* This is currently unused, since we disabled HEURISTIC2 */ -extern int __start[]; -#define HEURISTIC2_LIMIT ((ptr_t)((word)(__start) & ~(getpagesize() - 1))) -#ifndef GC_OSF1_THREADS -/* Unresolved signal issues with threads. */ -#define MPROTECT_VDB -#endif -#define DYNAMIC_LOADING -#endif -#ifdef LINUX -#ifdef __ELF__ -#define SEARCH_FOR_DATA_START -#else -#define DATASTART ((ptr_t)0x140000000) -extern int _end[]; -#define DATAEND ((ptr_t)(_end)) -#endif -#endif -#endif /* ALPHA */ - -#ifdef IA64 -#define MACH_TYPE "IA64" -#ifdef HPUX -#ifdef _ILP32 -#define CPP_WORDSZ 32 -/* Requires 8 byte alignment for malloc */ -#define ALIGNMENT 4 -#else -#if !defined(_LP64) && !defined(CPPCHECK) -#error Unknown ABI -#endif -#define CPP_WORDSZ 64 -/* Requires 16 byte alignment for malloc */ -#define ALIGNMENT 8 -#endif -/* Note that the GC must be initialized before the 1st putenv call. */ -extern char **environ; -#define STACKBOTTOM ((ptr_t)environ) -#define HPUX_STACKBOTTOM -/* The following was empirically determined, and is probably */ -/* not very robust. */ -/* Note that the backing store base seems to be at a nice */ -/* address minus one page. */ -#define BACKING_STORE_DISPLACEMENT 0x1000000 -#define BACKING_STORE_ALIGNMENT 0x1000 -/* Known to be wrong for recent HP/UX versions!!! */ -#endif -#ifdef LINUX -#define CPP_WORDSZ 64 -/* The following works on NUE and older kernels: */ -/* define STACKBOTTOM ((ptr_t)0xa000000000000000l) */ -/* TODO: LINUX_STACKBOTTOM does not work on NUE. */ -/* We also need the base address of the register stack */ -/* backing store. */ -#define SEARCH_FOR_DATA_START -#ifdef __GNUC__ -#define DYNAMIC_LOADING -#else -/* In the Intel compiler environment, we seem to end up with */ -/* statically linked executables and an undefined reference */ -/* to _DYNAMIC */ -#endif -#ifdef __GNUC__ -#ifndef __INTEL_COMPILER -#define PREFETCH(x) \ - __asm__(" lfetch [%0]" \ - : \ - : "r"(x)) -#define GC_PREFETCH_FOR_WRITE(x) \ - __asm__(" lfetch.excl [%0]" \ - : \ - : "r"(x)) -#define CLEAR_DOUBLE(x) \ - __asm__(" stf.spill [%0]=f0" \ - : \ - : "r"((void *)(x))) -#else -EXTERN_C_END -#include -EXTERN_C_BEGIN -#define PREFETCH(x) __lfetch(__lfhint_none, (x)) -#define GC_PREFETCH_FOR_WRITE(x) __lfetch(__lfhint_nta, (x)) -#define CLEAR_DOUBLE(x) __stf_spill((void *)(x), 0) -#endif /* __INTEL_COMPILER */ -#endif -#endif -#ifdef MSWIN32 -/* FIXME: This is a very partial guess. There is no port, yet. */ -#if defined(_WIN64) -#define CPP_WORDSZ 64 -#else -#define CPP_WORDSZ 32 /* Is this possible? */ -#define ALIGNMENT 8 -#endif -#endif -#endif - -#ifdef E2K -#define MACH_TYPE "E2K" -#ifdef __LP64__ -#define CPP_WORDSZ 64 -#else -#define CPP_WORDSZ 32 -#endif -#ifndef HBLKSIZE -#define HBLKSIZE 4096 -#endif -#ifdef LINUX -extern int __dso_handle[]; -#define DATASTART ((ptr_t)__dso_handle) -#ifdef REDIRECT_MALLOC -#define NO_PROC_FOR_LIBRARIES -#endif -#endif -#endif /* E2K */ - -#ifdef M88K -#define MACH_TYPE "M88K" -#define ALIGNMENT 4 -#define STACKBOTTOM ((char *)0xf0000000) /* determined empirically */ -extern int etext[]; -#ifdef CX_UX -#define OS_TYPE "CX_UX" -#define DATASTART ((ptr_t)((((word)(etext) + 0x3fffff) & ~0x3fffff) + 0x10000)) -#endif -#ifdef DGUX -#define OS_TYPE "DGUX" -ptr_t GC_SysVGetDataStart(size_t, ptr_t); -#define DATASTART GC_SysVGetDataStart(0x10000, (ptr_t)etext) -#define DATASTART_IS_FUNC -#endif -#endif - -#ifdef S370 -/* If this still works, and if anyone cares, this should probably */ -/* be moved to the S390 category. */ -#define MACH_TYPE "S370" -#define ALIGNMENT 4 /* Required by hardware */ -#ifdef UTS4 -#define OS_TYPE "UTS4" -extern int etext[]; -extern int _etext[]; -extern int _end[]; -ptr_t GC_SysVGetDataStart(size_t, ptr_t); -#define DATASTART GC_SysVGetDataStart(0x10000, (ptr_t)_etext) -#define DATASTART_IS_FUNC -#define DATAEND ((ptr_t)(_end)) -#define HEURISTIC2 -#endif -#endif - -#ifdef S390 -#define MACH_TYPE "S390" -#ifndef __s390x__ -#define CPP_WORDSZ 32 -#else -#define CPP_WORDSZ 64 -#ifndef HBLKSIZE -#define HBLKSIZE 4096 -#endif -#endif -#ifdef LINUX -extern int __data_start[] __attribute__((__weak__)); -#define DATASTART ((ptr_t)(__data_start)) -extern int _end[] __attribute__((__weak__)); -#define DATAEND ((ptr_t)(_end)) -#define CACHE_LINE_SIZE 256 -#define GETPAGESIZE() 4096 -#ifndef SOFT_VDB -#define SOFT_VDB -#endif -#endif -#endif /* S390 */ - -#ifdef AARCH64 -#define MACH_TYPE "AARCH64" -#ifdef __ILP32__ -#define CPP_WORDSZ 32 -#else -#define CPP_WORDSZ 64 -#endif -#ifndef HBLKSIZE -#define HBLKSIZE 4096 -#endif -#ifdef LINUX -#if defined(HOST_ANDROID) -#define SEARCH_FOR_DATA_START -#else -extern int __data_start[] __attribute__((__weak__)); -#define DATASTART ((ptr_t)__data_start) -#endif -#endif -#ifdef DARWIN -/* iOS */ -#define DARWIN_DONT_PARSE_STACK 1 -#define STACKBOTTOM ((ptr_t)0x16fdfffff) -/* MPROTECT_VDB causes use of non-public API like exc_server, */ -/* this could be a reason for blocking the client application in */ -/* the store. */ -#if TARGET_OS_IPHONE && !defined(NO_DYLD_BIND_FULLY_IMAGE) -#define NO_DYLD_BIND_FULLY_IMAGE -#endif -#endif -#ifdef FREEBSD -/* Nothing specific. */ -#endif -#ifdef NETBSD -#define ELF_CLASS ELFCLASS64 -#endif -#ifdef OPENBSD -/* Nothing specific. */ -#endif -#ifdef NINTENDO_SWITCH -#define OS_TYPE "NINTENDO_SWITCH" -extern int __bss_end[]; -#define NO_HANDLE_FORK 1 -#define DATASTART (ptr_t) ALIGNMENT /* cannot be null */ -#define DATAEND (ptr_t)(&__bss_end) -void *switch_get_stack_bottom(void); -#define STACKBOTTOM ((ptr_t)switch_get_stack_bottom()) -#ifndef HAVE_CLOCK_GETTIME -#define HAVE_CLOCK_GETTIME 1 -#endif -#endif -#ifdef QNX -/* Nothing specific. */ -#endif -#ifdef MSWIN32 /* UWP */ - /* TODO: Enable MPROTECT_VDB */ -#endif -#ifdef NOSYS -#define OS_TYPE "NOSYS" -/* __data_start is usually defined in the target linker script. */ -extern int __data_start[]; -#define DATASTART ((ptr_t)__data_start) -extern void *__stack_base__; -#define STACKBOTTOM ((ptr_t)__stack_base__) -#endif -#endif - -#ifdef ARM32 -#if defined(NACL) -#define MACH_TYPE "NACL" -#else -#define MACH_TYPE "ARM32" -#endif -#define CPP_WORDSZ 32 -#ifdef NETBSD -/* Nothing specific. */ -#endif -#ifdef LINUX -#if GC_GLIBC_PREREQ(2, 0) || defined(HOST_ANDROID) -#define SEARCH_FOR_DATA_START -#else -extern char **__environ; -#define DATASTART ((ptr_t)(&__environ)) -/* hideous kludge: __environ is the first */ -/* word in crt0.o, and delimits the start */ -/* of the data segment, no matter which */ -/* ld options were passed through. */ -/* We could use _etext instead, but that */ -/* would include .rodata, which may */ -/* contain large read-only data tables */ -/* that we'd rather not scan. */ -#endif -#endif -#ifdef MSWINCE -/* Nothing specific. */ -#endif -#ifdef FREEBSD -/* Nothing specific. */ -#endif -#ifdef DARWIN -/* iOS */ -#define DARWIN_DONT_PARSE_STACK 1 -#define STACKBOTTOM ((ptr_t)0x30000000) -/* MPROTECT_VDB causes use of non-public API. */ -#if TARGET_OS_IPHONE && !defined(NO_DYLD_BIND_FULLY_IMAGE) -#define NO_DYLD_BIND_FULLY_IMAGE -#endif -#endif -#ifdef OPENBSD -/* Nothing specific. */ -#endif -#ifdef QNX -/* Nothing specific. */ -#endif -#ifdef SN_TARGET_PSP2 -#define OS_TYPE "SN_TARGET_PSP2" -#define NO_HANDLE_FORK 1 -#ifndef HBLKSIZE -#define HBLKSIZE 65536 /* page size is 64 KB */ -#endif -#define DATASTART (ptr_t) ALIGNMENT -#define DATAEND (ptr_t) ALIGNMENT -void *psp2_get_stack_bottom(void); -#define STACKBOTTOM ((ptr_t)psp2_get_stack_bottom()) -#endif -#ifdef NN_PLATFORM_CTR -#define OS_TYPE "NN_PLATFORM_CTR" -extern unsigned char Image$$ZI$$ZI$$Base[]; -#define DATASTART (ptr_t)(Image$$ZI$$ZI$$Base) -extern unsigned char Image$$ZI$$ZI$$Limit[]; -#define DATAEND (ptr_t)(Image$$ZI$$ZI$$Limit) -void *n3ds_get_stack_bottom(void); -#define STACKBOTTOM ((ptr_t)n3ds_get_stack_bottom()) -#endif -#ifdef MSWIN32 /* UWP */ - /* TODO: Enable MPROTECT_VDB */ -#endif -#ifdef NOSYS -#define OS_TYPE "NOSYS" -/* __data_start is usually defined in the target linker script. */ -extern int __data_start[]; -#define DATASTART ((ptr_t)(__data_start)) -/* __stack_base__ is set in newlib/libc/sys/arm/crt0.S */ -extern void *__stack_base__; -#define STACKBOTTOM ((ptr_t)(__stack_base__)) -#endif -#endif - -#ifdef CRIS -#define MACH_TYPE "CRIS" -#define CPP_WORDSZ 32 -#define ALIGNMENT 1 -#ifdef LINUX -#define SEARCH_FOR_DATA_START -#endif -#endif /* CRIS */ - -#if defined(SH) && !defined(SH4) -#define MACH_TYPE "SH" -#define ALIGNMENT 4 -#ifdef LINUX -#define SEARCH_FOR_DATA_START -#endif -#ifdef NETBSD -/* Nothing specific. */ -#endif -#ifdef OPENBSD -/* Nothing specific. */ -#endif -#ifdef MSWINCE -/* Nothing specific. */ -#endif -#endif - -#ifdef SH4 -#define MACH_TYPE "SH4" -#define ALIGNMENT 4 -#ifdef MSWINCE -/* Nothing specific. */ -#endif -#endif - -#ifdef AVR32 -#define MACH_TYPE "AVR32" -#define CPP_WORDSZ 32 -#ifdef LINUX -#define SEARCH_FOR_DATA_START -#endif -#endif /* AVR32 */ - -#ifdef M32R -#define MACH_TYPE "M32R" -#define CPP_WORDSZ 32 -#ifdef LINUX -#define SEARCH_FOR_DATA_START -#endif -#endif /* M32R */ - -#ifdef X86_64 -#define MACH_TYPE "X86_64" -#ifdef __ILP32__ -#define CPP_WORDSZ 32 -#else -#define CPP_WORDSZ 64 -#endif -#ifndef HBLKSIZE -#define HBLKSIZE 4096 -#endif -#ifndef CACHE_LINE_SIZE -#define CACHE_LINE_SIZE 64 -#endif -#ifdef PLATFORM_GETMEM -#define OS_TYPE "PLATFORM_GETMEM" -#define DATASTART (ptr_t) ALIGNMENT -#define DATAEND (ptr_t) ALIGNMENT -EXTERN_C_END -#include -EXTERN_C_BEGIN -void *platform_get_stack_bottom(void); -#define STACKBOTTOM ((ptr_t)platform_get_stack_bottom()) -#endif -#ifdef LINUX -#define SEARCH_FOR_DATA_START -#if defined(__GLIBC__) && !defined(__UCLIBC__) -/* A workaround for GCF (Google Cloud Function) which does */ -/* not support mmap() for "/dev/zero". Should not cause any */ -/* harm to other targets. */ -#define USE_MMAP_ANON -#endif -#if defined(__GLIBC__) && !defined(__UCLIBC__) && !defined(GETCONTEXT_FPU_BUG_FIXED) -/* At present, there's a bug in glibc getcontext() on */ -/* Linux/x64 (it clears FPU exception mask). We define this */ -/* macro to workaround it. */ -/* TODO: This seems to be fixed in glibc 2.14. */ -#define GETCONTEXT_FPU_EXCMASK_BUG -#endif -#if defined(__GLIBC__) && !defined(__UCLIBC__) && !defined(GLIBC_TSX_BUG_FIXED) -/* Workaround lock elision implementation for some glibc. */ -#define GLIBC_2_19_TSX_BUG -EXTERN_C_END -#include /* for gnu_get_libc_version() */ -EXTERN_C_BEGIN -#endif -#ifndef SOFT_VDB -#define SOFT_VDB -#endif -#endif -#ifdef DARWIN -#define DARWIN_DONT_PARSE_STACK 1 -#define STACKBOTTOM ((ptr_t)0x7fff5fc00000) -#define MPROTECT_VDB -#if TARGET_OS_IPHONE && !defined(NO_DYLD_BIND_FULLY_IMAGE) -/* iPhone/iPad simulator */ -#define NO_DYLD_BIND_FULLY_IMAGE -#endif -#endif -#ifdef FREEBSD -#if defined(__GLIBC__) -extern int _end[]; -#define DATAEND ((ptr_t)(_end)) -#endif -#if defined(__DragonFly__) -/* DragonFly BSD still has vm.max_proc_mmap, according to */ -/* its mmap(2) man page. */ -#define COUNT_UNMAPPED_REGIONS -#endif -#endif -#ifdef NETBSD -/* Nothing specific. */ -#endif -#ifdef OPENBSD -/* Nothing specific. */ -#endif -#ifdef HAIKU -#define HEURISTIC2 -#define SEARCH_FOR_DATA_START -#endif -#ifdef QNX -/* Nothing specific. */ -#endif -#ifdef SOLARIS -#define ELF_CLASS ELFCLASS64 -#define DATASTART GC_SysVGetDataStart(0x1000, (ptr_t)_etext) -#ifdef SOLARIS25_PROC_VDB_BUG_FIXED -#define PROC_VDB -#endif -#endif -#ifdef CYGWIN32 -#ifndef USE_WINALLOC -#if defined(THREAD_LOCAL_ALLOC) -/* TODO: For an unknown reason, thread-local allocations */ -/* lead to spurious process exit after the fault handler is */ -/* once invoked. */ -#else -#define MPROTECT_VDB -#endif -#endif -#endif -#ifdef MSWIN_XBOX1 -#define OS_TYPE "MSWIN_XBOX1" -#define NO_GETENV -#define DATASTART (ptr_t) ALIGNMENT -#define DATAEND (ptr_t) ALIGNMENT -LONG64 durango_get_stack_bottom(void); -#define STACKBOTTOM ((ptr_t)durango_get_stack_bottom()) -#define GETPAGESIZE() 4096 -#ifndef USE_MMAP -#define USE_MMAP 1 -#endif -/* The following is from sys/mman.h: */ -#define PROT_NONE 0 -#define PROT_READ 1 -#define PROT_WRITE 2 -#define PROT_EXEC 4 -#define MAP_PRIVATE 2 -#define MAP_FIXED 0x10 -#define MAP_FAILED ((void *)-1) -#endif -#ifdef MSWIN32 -#define RETRY_GET_THREAD_CONTEXT -#if !defined(__GNUC__) || defined(__INTEL_COMPILER) || (GC_GNUC_PREREQ(4, 7) && !defined(__MINGW64__)) -/* Older GCC and Mingw-w64 (both GCC and Clang) do not */ -/* support SetUnhandledExceptionFilter() properly on x64. */ -#define MPROTECT_VDB -#endif -#endif -#endif /* X86_64 */ - -#ifdef ARC -#define MACH_TYPE "ARC" -#define CPP_WORDSZ 32 -#define CACHE_LINE_SIZE 64 -#ifdef LINUX -extern int __data_start[] __attribute__((__weak__)); -#define DATASTART ((ptr_t)__data_start) -#endif -#endif /* ARC */ - -#ifdef HEXAGON -#define MACH_TYPE "HEXAGON" -#define CPP_WORDSZ 32 -#ifdef LINUX -#if defined(__GLIBC__) -#define SEARCH_FOR_DATA_START -#elif !defined(CPPCHECK) -#error Unknown Hexagon libc configuration -#endif -#endif -#endif /* HEXAGON */ - -#ifdef TILEPRO -#define MACH_TYPE "TILEPro" -#define CPP_WORDSZ 32 -#define PREFETCH(x) __insn_prefetch(x) -#define CACHE_LINE_SIZE 64 -#ifdef LINUX -extern int __data_start[]; -#define DATASTART ((ptr_t)__data_start) -#endif -#endif /* TILEPRO */ - -#ifdef TILEGX -#define MACH_TYPE "TILE-Gx" -#define CPP_WORDSZ (__SIZEOF_POINTER__ * 8) -#if CPP_WORDSZ < 64 -#define CLEAR_DOUBLE(x) (*(long long *)(x) = 0) -#endif -#define PREFETCH(x) __insn_prefetch_l1(x) -#define CACHE_LINE_SIZE 64 -#ifdef LINUX -extern int __data_start[]; -#define DATASTART ((ptr_t)__data_start) -#endif -#endif /* TILEGX */ - -#ifdef RISCV -#define MACH_TYPE "RISC-V" -#define CPP_WORDSZ __riscv_xlen /* 32 or 64 */ -#ifdef FREEBSD -/* Nothing specific. */ -#endif -#ifdef LINUX -extern int __data_start[] __attribute__((__weak__)); -#define DATASTART ((ptr_t)__data_start) -#endif -#ifdef OPENBSD -/* Nothing specific. */ -#endif -#endif /* RISCV */ - -#ifdef WEBASSEMBLY -#define MACH_TYPE "WebAssembly" -#if defined(__wasm64__) && !defined(CPPCHECK) -#error 64-bit WebAssembly is not yet supported -#endif -#define ALIGNMENT 4 -#ifdef EMSCRIPTEN -#define OS_TYPE "EMSCRIPTEN" -#define DATASTART (ptr_t) ALIGNMENT -#define DATAEND (ptr_t) ALIGNMENT -/* Emscripten does emulate mmap and munmap, but those should */ -/* not be used in the collector, since WebAssembly lacks the */ -/* native support of memory mapping. Use sbrk() instead. */ -#undef USE_MMAP -#undef USE_MUNMAP -#if defined(GC_THREADS) && !defined(CPPCHECK) -#error No threads support yet -#endif -#endif -#ifdef WASI -#define OS_TYPE "WASI" -extern char __global_base, __heap_base; -#define STACKBOTTOM ((ptr_t)&__global_base) -#define DATASTART ((ptr_t)&__global_base) -#define DATAEND ((ptr_t)&__heap_base) -#ifndef GC_NO_SIGSETJMP -#define GC_NO_SIGSETJMP 1 /* no support of signals */ -#endif -#ifndef NO_CLOCK -#define NO_CLOCK 1 /* no support of clock */ -#endif -#undef USE_MMAP /* similar to Emscripten */ -#undef USE_MUNMAP -#if defined(GC_THREADS) && !defined(CPPCHECK) -#error No threads support yet -#endif -#endif -#endif /* WEBASSEMBLY */ - -#if defined(__GLIBC__) && !defined(DONT_USE_LIBC_PRIVATES) -/* Use glibc's stack-end marker. */ -#define USE_LIBC_PRIVATES -#endif - -#ifdef NO_RETRY_GET_THREAD_CONTEXT -#undef RETRY_GET_THREAD_CONTEXT -#endif - -#if defined(LINUX_STACKBOTTOM) && defined(NO_PROC_STAT) && !defined(USE_LIBC_PRIVATES) -/* This combination will fail, since we have no way to get */ -/* the stack bottom. Use HEURISTIC2 instead. */ -#undef LINUX_STACKBOTTOM -#define HEURISTIC2 -/* This may still fail on some architectures like IA64. */ -/* We tried ... */ -#endif - -#if defined(USE_MMAP_ANON) && !defined(USE_MMAP) -#define USE_MMAP 1 -#elif (defined(LINUX) || defined(OPENBSD)) && defined(USE_MMAP) -/* The kernel may do a somewhat better job merging mappings etc. */ -/* with anonymous mappings. */ -#define USE_MMAP_ANON -#endif - -#if defined(GC_LINUX_THREADS) && defined(REDIRECT_MALLOC) && !defined(USE_PROC_FOR_LIBRARIES) && !defined(NO_PROC_FOR_LIBRARIES) -/* Nptl allocates thread stacks with mmap, which is fine. But it */ -/* keeps a cache of thread stacks. Thread stacks contain the */ -/* thread control blocks. These in turn contain a pointer to */ -/* (sizeof (void *) from the beginning of) the dtv for thread-local */ -/* storage, which is calloc allocated. If we don't scan the cached */ -/* thread stacks, we appear to lose the dtv. This tends to */ -/* result in something that looks like a bogus dtv count, which */ -/* tends to result in a memset call on a block that is way too */ -/* large. Sometimes we're lucky and the process just dies ... */ -/* There seems to be a similar issue with some other memory */ -/* allocated by the dynamic loader. */ -/* This should be avoidable by either: */ -/* - Defining USE_PROC_FOR_LIBRARIES here. */ -/* That performs very poorly, precisely because we end up */ -/* scanning cached stacks. */ -/* - Have calloc look at its callers. */ -/* In spite of the fact that it is gross and disgusting. */ -/* In fact neither seems to suffice, probably in part because */ -/* even with USE_PROC_FOR_LIBRARIES, we don't scan parts of stack */ -/* segments that appear to be out of bounds. Thus we actually */ -/* do both, which seems to yield the best results. */ -#define USE_PROC_FOR_LIBRARIES -#endif - -#ifndef STACK_GROWS_UP -#define STACK_GROWS_DOWN -#endif - -#ifndef CPP_WORDSZ -#define CPP_WORDSZ 32 -#endif - -#ifndef OS_TYPE -#define OS_TYPE "" -#endif - -#ifndef DATAEND -#if !defined(CPPCHECK) -extern int end[]; -#endif -#define DATAEND ((ptr_t)(end)) -#endif - -/* Workaround for Android NDK clang 3.5+ (as of NDK r10e) which does */ -/* not provide correct _end symbol. Unfortunately, alternate __end__ */ -/* symbol is provided only by NDK "bfd" linker. */ -#if defined(HOST_ANDROID) && defined(__clang__) && !defined(BROKEN_UUENDUU_SYM) -#undef DATAEND -#pragma weak __end__ -extern int __end__[]; -#define DATAEND (__end__ != 0 ? (ptr_t)__end__ : (ptr_t)_end) -#endif - -#if !defined(MSWIN32) && !defined(MSWINCE) && !defined(CYGWIN32) && !defined(GETPAGESIZE) -#if defined(AIX) || defined(DARWIN) || defined(IRIX5) || defined(LINUX) || defined(FREEBSD) || defined(NETBSD) || defined(SOLARIS) -EXTERN_C_END -#include -EXTERN_C_BEGIN -#endif -#if defined(HOST_ANDROID) || defined(HOST_TIZEN) || defined(SVR4) -#define GETPAGESIZE() (unsigned)sysconf(_SC_PAGESIZE) -#else -#define GETPAGESIZE() (unsigned)getpagesize() -#endif -#endif /* !MSWIN32 && !GETPAGESIZE */ - -#if defined(HOST_ANDROID) && !(__ANDROID_API__ >= 23) && ((defined(MIPS) && (CPP_WORDSZ == 32)) || defined(ARM32) || defined(I386) /* but not x32 */) -/* tkill() exists only on arm32/mips(32)/x86. */ -/* NDK r11+ deprecates tkill() but keeps it for Mono clients. */ -#define USE_TKILL_ON_ANDROID -#endif - -#if defined(SOLARIS) || defined(DRSNX) || defined(UTS4) -/* OS has SVR4 generic features. */ -/* Probably others also qualify. */ -#define SVR4 -#endif - -#if defined(MPROTECT_VDB) && defined(__GLIBC__) && !GC_GLIBC_PREREQ(2, 2) -#error glibc too old? -#endif - -#if defined(SOLARIS) || defined(DRSNX) -/* OS has SOLARIS style semi-undocumented interface */ -/* to dynamic loader. */ -#define SOLARISDL -/* OS has SOLARIS style signal handlers. */ -#define SUNOS5SIGS -#endif - -#if defined(HPUX) -#define SUNOS5SIGS -#endif - -#if defined(FREEBSD) && (defined(__DragonFly__) || __FreeBSD__ >= 4 || __FreeBSD_kernel__ >= 4 || defined(__GLIBC__)) -#define SUNOS5SIGS -#endif - -#if defined(FREEBSD) || defined(HAIKU) || defined(HURD) || defined(IRIX5) || defined(NETBSD) || defined(OPENBSD) || defined(OSF1) || defined(SUNOS5SIGS) -#define USE_SEGV_SIGACT -#if defined(IRIX5) && defined(_sigargs) /* Irix 5.x, not 6.x */ \ - || (defined(FREEBSD) && defined(SUNOS5SIGS)) || defined(HPUX) || defined(HURD) || defined(NETBSD) -/* We may get SIGBUS. */ -#define USE_BUS_SIGACT -#endif -#endif - -#if !defined(GC_EXPLICIT_SIGNALS_UNBLOCK) && defined(SUNOS5SIGS) && !defined(GC_NO_PTHREAD_SIGMASK) -#define GC_EXPLICIT_SIGNALS_UNBLOCK -#endif - -#if !defined(NO_SIGNALS_UNBLOCK_IN_MAIN) && defined(GC_NO_PTHREAD_SIGMASK) -#define NO_SIGNALS_UNBLOCK_IN_MAIN -#endif - -#if !defined(NO_MARKER_SPECIAL_SIGMASK) && (defined(NACL) || defined(GC_WIN32_PTHREADS)) -/* Either there is no pthread_sigmask(), or GC marker thread cannot */ -/* steal and drop user signal calls. */ -#define NO_MARKER_SPECIAL_SIGMASK -#endif - -#ifdef GC_NETBSD_THREADS -#define SIGRTMIN 33 -#define SIGRTMAX 63 -/* It seems to be necessary to wait until threads have restarted. */ -/* But it is unclear why that is the case. */ -#define GC_NETBSD_THREADS_WORKAROUND -#endif - -#ifdef GC_OPENBSD_THREADS -EXTERN_C_END -#include -EXTERN_C_BEGIN -#endif /* GC_OPENBSD_THREADS */ - -#if defined(SVR4) || defined(LINUX) || defined(IRIX5) || defined(HPUX) || defined(OPENBSD) || defined(NETBSD) || defined(FREEBSD) || defined(DGUX) || defined(BSD) || defined(HAIKU) || defined(HURD) || defined(AIX) || defined(DARWIN) || defined(OSF1) || defined(QNX) -#define UNIX_LIKE /* Basic Unix-like system calls work. */ -#endif - -#if defined(CPPCHECK) -#undef CPP_WORDSZ -#define CPP_WORDSZ (__SIZEOF_POINTER__ * 8) -#elif CPP_WORDSZ != 32 && CPP_WORDSZ != 64 -#error Bad word size -#endif - -#ifndef ALIGNMENT -#if !defined(CPP_WORDSZ) && !defined(CPPCHECK) -#error Undefined both ALIGNMENT and CPP_WORDSZ -#endif -#define ALIGNMENT (CPP_WORDSZ >> 3) -#endif /* !ALIGNMENT */ - -#ifdef PCR -#undef DYNAMIC_LOADING -#undef STACKBOTTOM -#undef HEURISTIC1 -#undef HEURISTIC2 -#undef PROC_VDB -#undef MPROTECT_VDB -#define PCR_VDB -#endif - -#if !defined(STACKBOTTOM) && (defined(ECOS) || defined(NOSYS)) && !defined(CPPCHECK) -#error Undefined STACKBOTTOM -#endif - -#ifdef IGNORE_DYNAMIC_LOADING -#undef DYNAMIC_LOADING -#endif - -#if defined(SMALL_CONFIG) && !defined(GC_DISABLE_INCREMENTAL) -/* Presumably not worth the space it takes. */ -#define GC_DISABLE_INCREMENTAL -#endif - -#if (defined(MSWIN32) || defined(MSWINCE)) && !defined(USE_WINALLOC) -/* USE_WINALLOC is only an option for Cygwin. */ -#define USE_WINALLOC 1 -#endif - -#ifdef USE_WINALLOC -#undef USE_MMAP -#endif - -#if defined(DARWIN) || defined(FREEBSD) || defined(HAIKU) || defined(IRIX5) || defined(LINUX) || defined(NETBSD) || defined(OPENBSD) || defined(SOLARIS) || ((defined(CYGWIN32) || defined(USE_MMAP) || defined(USE_MUNMAP)) && !defined(USE_WINALLOC)) -/* Try both sbrk and mmap, in that order. */ -#define MMAP_SUPPORTED -#endif - -/* Xbox One (DURANGO) may not need to be this aggressive, but the */ -/* default is likely too lax under heavy allocation pressure. */ -/* The platform does not have a virtual paging system, so it does not */ -/* have a large virtual address space that a standard x64 platform has. */ -#if defined(USE_MUNMAP) && !defined(MUNMAP_THRESHOLD) && (defined(SN_TARGET_PS3) || defined(SN_TARGET_PSP2) || defined(MSWIN_XBOX1)) -#define MUNMAP_THRESHOLD 3 -#endif - -#if defined(USE_MUNMAP) && defined(COUNT_UNMAPPED_REGIONS) && !defined(GC_UNMAPPED_REGIONS_SOFT_LIMIT) -/* The default limit of vm.max_map_count on Linux is ~65530. */ -/* There is approximately one mapped region to every unmapped region. */ -/* Therefore if we aim to use up to half of vm.max_map_count for the */ -/* GC (leaving half for the rest of the process) then the number of */ -/* unmapped regions should be one quarter of vm.max_map_count. */ -#if defined(__DragonFly__) -#define GC_UNMAPPED_REGIONS_SOFT_LIMIT (1000000 / 4) -#else -#define GC_UNMAPPED_REGIONS_SOFT_LIMIT 16384 -#endif -#endif - -#if defined(GC_DISABLE_INCREMENTAL) || defined(DEFAULT_VDB) -#undef GWW_VDB -#undef MPROTECT_VDB -#undef PCR_VDB -#undef PROC_VDB -#undef SOFT_VDB -#endif - -#ifdef NO_GWW_VDB -#undef GWW_VDB -#endif - -#ifdef NO_MPROTECT_VDB -#undef MPROTECT_VDB -#endif - -#ifdef NO_SOFT_VDB -#undef SOFT_VDB -#endif - -#if defined(SOFT_VDB) && defined(SOFT_VDB_LINUX_VER_STATIC_CHECK) -EXTERN_C_END -#include /* for LINUX_VERSION[_CODE] */ -EXTERN_C_BEGIN -#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 18, 0) -/* Not reliable in kernels prior to v3.18. */ -#undef SOFT_VDB -#endif -#endif /* SOFT_VDB */ - -#ifdef GC_DISABLE_INCREMENTAL -#undef CHECKSUMS -#endif - -#ifdef USE_GLOBAL_ALLOC -/* Cannot pass MEM_WRITE_WATCH to GlobalAlloc(). */ -#undef GWW_VDB -#endif - -#if defined(BASE_ATOMIC_OPS_EMULATED) -/* GC_write_fault_handler() cannot use lock-based atomic primitives */ -/* as this could lead to a deadlock. */ -#undef MPROTECT_VDB -#endif - -#if defined(USE_PROC_FOR_LIBRARIES) && defined(GC_LINUX_THREADS) -/* Incremental GC based on mprotect is incompatible with /proc roots. */ -#undef MPROTECT_VDB -#endif - -#if defined(MPROTECT_VDB) && defined(GC_PREFER_MPROTECT_VDB) -/* Choose MPROTECT_VDB manually (if multiple strategies available). */ -#undef PCR_VDB -#undef PROC_VDB -/* GWW_VDB, SOFT_VDB are handled in os_dep.c. */ -#endif - -#ifdef PROC_VDB -/* Mutually exclusive VDB implementations (for now). */ -#undef MPROTECT_VDB -/* For a test purpose only. */ -#undef SOFT_VDB -#endif - -#if defined(MPROTECT_VDB) && !defined(MSWIN32) && !defined(MSWINCE) -#include /* for SA_SIGINFO, SIGBUS */ -#endif - -#if defined(SIGBUS) && !defined(HAVE_SIGBUS) && !defined(CPPCHECK) -#define HAVE_SIGBUS -#endif - -#ifndef SA_SIGINFO -#define NO_SA_SIGACTION -#endif - -#if (defined(NO_SA_SIGACTION) || defined(GC_NO_SIGSETJMP)) && defined(MPROTECT_VDB) && !defined(DARWIN) && !defined(MSWIN32) && !defined(MSWINCE) -#undef MPROTECT_VDB -#endif - -#if !defined(PCR_VDB) && !defined(PROC_VDB) && !defined(MPROTECT_VDB) && !defined(GWW_VDB) && !defined(SOFT_VDB) && !defined(DEFAULT_VDB) && !defined(GC_DISABLE_INCREMENTAL) -#define DEFAULT_VDB -#endif - -#if !defined(PROC_VDB) && !defined(SOFT_VDB) && !defined(NO_VDB_FOR_STATIC_ROOTS) -/* Cannot determine whether a static root page is dirty? */ -#define NO_VDB_FOR_STATIC_ROOTS -#endif - -#if ((defined(UNIX_LIKE) && (defined(DARWIN) || defined(HAIKU) || defined(HURD) || defined(OPENBSD) || defined(QNX) || defined(ARM32) || defined(AVR32) || defined(MIPS) || defined(NIOS2) || defined(OR1K))) || (defined(LINUX) && !defined(__gnu_linux__)) || (defined(RTEMS) && defined(I386)) || defined(HOST_ANDROID)) && !defined(NO_GETCONTEXT) -#define NO_GETCONTEXT 1 -#endif - -#ifndef PREFETCH -#if GC_GNUC_PREREQ(3, 0) && !defined(NO_PREFETCH) -#define PREFETCH(x) __builtin_prefetch((x), 0, 0) -#else -#define PREFETCH(x) (void)0 -#endif -#endif - -#ifndef GC_PREFETCH_FOR_WRITE -#if GC_GNUC_PREREQ(3, 0) && !defined(GC_NO_PREFETCH_FOR_WRITE) -#define GC_PREFETCH_FOR_WRITE(x) __builtin_prefetch((x), 1) -#else -#define GC_PREFETCH_FOR_WRITE(x) (void)0 -#endif -#endif - -#ifndef CACHE_LINE_SIZE -#define CACHE_LINE_SIZE 32 /* Wild guess */ -#endif - -#ifndef STATIC -#ifdef GC_ASSERTIONS -#define STATIC /* ignore to aid debugging (or profiling) */ -#else -#define STATIC static -#endif -#endif - -/* Do we need the GC_find_limit machinery to find the end of */ -/* a data segment (or the backing store base)? */ -#if defined(HEURISTIC2) || defined(SEARCH_FOR_DATA_START) || defined(HPUX_MAIN_STACKBOTTOM) || defined(IA64) || (defined(CYGWIN32) && defined(I386) && defined(USE_MMAP) && !defined(USE_WINALLOC)) || (defined(NETBSD) && defined(__ELF__)) || defined(OPENBSD) || ((defined(SVR4) || defined(AIX) || defined(DGUX) || defined(DATASTART_USES_BSDGETDATASTART) || (defined(LINUX) && defined(SPARC))) && !defined(PCR)) -#define NEED_FIND_LIMIT -#endif - -#if defined(LINUX) && (defined(USE_PROC_FOR_LIBRARIES) || defined(IA64) || !defined(SMALL_CONFIG)) -#define NEED_PROC_MAPS -#endif - -#if defined(LINUX) || defined(HURD) || defined(__GLIBC__) -#define REGISTER_LIBRARIES_EARLY -/* We sometimes use dl_iterate_phdr, which may acquire an internal */ -/* lock. This isn't safe after the world has stopped. So we must */ -/* call GC_register_dynamic_libraries before stopping the world. */ -/* For performance reasons, this may be beneficial on other */ -/* platforms as well, though it should be avoided on Windows. */ -#endif /* LINUX */ - -#if defined(SEARCH_FOR_DATA_START) -extern ptr_t GC_data_start; -#define DATASTART GC_data_start -#endif - -#ifndef HEAP_START -#define HEAP_START ((ptr_t)0) -#endif - -#ifndef CLEAR_DOUBLE -#define CLEAR_DOUBLE(x) (((word *)(x))[0] = 0, ((word *)(x))[1] = 0) -#endif - -/* Some libc implementations like bionic, musl and glibc 2.34 */ -/* do not have libpthread.so because the pthreads-related code */ -/* is located in libc.so, thus potential calloc calls from such */ -/* code are forwarded to real (libc) calloc without any special */ -/* handling on the libgc side. Checking glibc version at */ -/* compile time for the purpose seems to be fine. */ -#if defined(GC_LINUX_THREADS) && defined(REDIRECT_MALLOC) && defined(__GLIBC__) && !GC_GLIBC_PREREQ(2, 34) && !defined(HAVE_LIBPTHREAD_SO) -#define HAVE_LIBPTHREAD_SO -#endif - -#if defined(GC_LINUX_THREADS) && defined(REDIRECT_MALLOC) && !defined(INCLUDE_LINUX_THREAD_DESCR) -/* Will not work, since libc and the dynamic loader use thread */ -/* locals, sometimes as the only reference. */ -#define INCLUDE_LINUX_THREAD_DESCR -#endif - -#if !defined(CPPCHECK) -#if defined(GC_IRIX_THREADS) && !defined(IRIX5) -#error Inconsistent configuration -#endif -#if defined(GC_LINUX_THREADS) && !defined(LINUX) && !defined(NACL) -#error Inconsistent configuration -#endif -#if defined(GC_NETBSD_THREADS) && !defined(NETBSD) -#error Inconsistent configuration -#endif -#if defined(GC_FREEBSD_THREADS) && !defined(FREEBSD) -#error Inconsistent configuration -#endif -#if defined(GC_SOLARIS_THREADS) && !defined(SOLARIS) -#error Inconsistent configuration -#endif -#if defined(GC_HPUX_THREADS) && !defined(HPUX) -#error Inconsistent configuration -#endif -#if defined(GC_AIX_THREADS) && !defined(_AIX) -#error Inconsistent configuration -#endif -#if defined(GC_WIN32_THREADS) && !defined(CYGWIN32) && !defined(MSWIN32) && !defined(MSWINCE) && !defined(MSWIN_XBOX1) -#error Inconsistent configuration -#endif -#if defined(GC_WIN32_PTHREADS) && defined(CYGWIN32) -#error Inconsistent configuration -#endif -#endif /* !CPPCHECK */ - -#if defined(PCR) || defined(GC_WIN32_THREADS) || defined(GC_PTHREADS) || ((defined(NN_PLATFORM_CTR) || defined(NINTENDO_SWITCH) || defined(SN_TARGET_PS3) || defined(SN_TARGET_PSP2)) && defined(GC_THREADS)) -#define THREADS -#endif - -#if defined(PARALLEL_MARK) && !defined(THREADS) && !defined(CPPCHECK) -#error Invalid config: PARALLEL_MARK requires GC_THREADS -#endif - -#if defined(GWW_VDB) && !defined(USE_WINALLOC) && !defined(CPPCHECK) -#error Invalid config: GWW_VDB requires USE_WINALLOC -#endif - -/* Whether GC_page_size is to be set to a value other than page size. */ -#if defined(CYGWIN32) && (defined(MPROTECT_VDB) || defined(USE_MUNMAP)) || (!defined(MSWIN32) && !defined(MSWINCE) && !defined(CYGWIN32) && (defined(GC_DISABLE_INCREMENTAL) || defined(DEFAULT_VDB)) && !defined(USE_MMAP)) -/* Cygwin: use the allocation granularity instead. */ -/* Other than Windows: use HBLKSIZE instead (unless mmap() is used). */ -#define ALT_PAGESIZE_USED -#ifndef GC_NO_VALLOC -/* Nonetheless, we need the real page size is some extra functions. */ -#define REAL_PAGESIZE_NEEDED -#endif -#endif - -#if defined(GC_PTHREADS) && !defined(GC_DARWIN_THREADS) && !defined(GC_WIN32_THREADS) && !defined(PLATFORM_STOP_WORLD) && !defined(SN_TARGET_PSP2) -#define PTHREAD_STOP_WORLD_IMPL -#endif - -#if defined(PTHREAD_STOP_WORLD_IMPL) && !defined(NACL) -#define SIGNAL_BASED_STOP_WORLD -#endif - -#if (defined(E2K) || defined(HP_PA) || defined(IA64) || defined(M68K) || defined(NO_SA_SIGACTION)) && defined(SIGNAL_BASED_STOP_WORLD) -#define SUSPEND_HANDLER_NO_CONTEXT -#endif - -#if (defined(MSWIN32) || defined(MSWINCE) || (defined(USE_PROC_FOR_LIBRARIES) && defined(THREADS))) && !defined(NO_CRT) && !defined(NO_WRAP_MARK_SOME) -/* Under rare conditions, we may end up marking from nonexistent */ -/* memory. Hence we need to be prepared to recover by running */ -/* GC_mark_some with a suitable handler in place. */ -/* TODO: Should we also define it for Cygwin? */ -#define WRAP_MARK_SOME -#endif - -#if !defined(MSWIN32) && !defined(MSWINCE) || defined(__GNUC__) || defined(NO_CRT) -#define NO_SEH_AVAILABLE -#endif - -#ifdef GC_WIN32_THREADS -/* The number of copied registers in copy_ptr_regs. */ -#if defined(I386) -#ifdef WOW64_THREAD_CONTEXT_WORKAROUND -#define PUSHED_REGS_COUNT 9 -#else -#define PUSHED_REGS_COUNT 7 -#endif -#elif defined(X86_64) || defined(SHx) -#define PUSHED_REGS_COUNT 15 -#elif defined(ARM32) -#define PUSHED_REGS_COUNT 13 -#elif defined(AARCH64) -#define PUSHED_REGS_COUNT 30 -#elif defined(MIPS) || defined(ALPHA) -#define PUSHED_REGS_COUNT 28 -#elif defined(PPC) -#define PUSHED_REGS_COUNT 29 -#endif -#endif /* GC_WIN32_THREADS */ - -#if defined(PARALLEL_MARK) && defined(GC_PTHREADS) && !defined(GC_PTHREADS_PARAMARK) && !defined(__MINGW32__) -/* Use pthread-based parallel mark implementation. */ -/* Except for MinGW 32/64 to workaround a deadlock in */ -/* winpthreads-3.0b internals. */ -#define GC_PTHREADS_PARAMARK -#else -#undef GC_PTHREADS_PARAMARK /* just in case defined by client */ -#endif - -#ifndef GC_NO_THREADS_DISCOVERY -#ifdef GC_DARWIN_THREADS -/* Task-based thread registration requires stack-frame-walking code. */ -#if defined(DARWIN_DONT_PARSE_STACK) -#define GC_NO_THREADS_DISCOVERY -#endif -#elif defined(GC_WIN32_THREADS) -/* DllMain-based thread registration is currently incompatible */ -/* with thread-local allocation, pthreads and WinCE. */ -#if (!defined(GC_DLL) && !defined(GC_INSIDE_DLL)) || defined(GC_PTHREADS) || defined(MSWINCE) || defined(NO_CRT) || defined(THREAD_LOCAL_ALLOC) -#define GC_NO_THREADS_DISCOVERY -#endif -#else -#define GC_NO_THREADS_DISCOVERY -#endif -#endif /* !GC_NO_THREADS_DISCOVERY */ - -#if defined(GC_DISCOVER_TASK_THREADS) && defined(GC_NO_THREADS_DISCOVERY) && !defined(CPPCHECK) -#error Defined both GC_DISCOVER_TASK_THREADS and GC_NO_THREADS_DISCOVERY -#endif - -#if defined(PARALLEL_MARK) && !defined(DEFAULT_STACK_MAYBE_SMALL) && (defined(HPUX) || defined(GC_DGUX386_THREADS) || defined(NO_GETCONTEXT) /* e.g. musl */) -/* TODO: Test default stack size in configure. */ -#define DEFAULT_STACK_MAYBE_SMALL -#endif - -#ifdef PARALLEL_MARK -/* The minimum stack size for a marker thread. */ -#define MIN_STACK_SIZE (8 * HBLKSIZE * sizeof(word)) -#endif - -#if defined(HOST_ANDROID) && !defined(THREADS) && !defined(USE_GET_STACKBASE_FOR_MAIN) -/* Always use pthread_attr_getstack on Android ("-lpthread" option is */ -/* not needed to be specified manually) since GC_linux_main_stack_base */ -/* causes app crash if invoked inside Dalvik VM. */ -#define USE_GET_STACKBASE_FOR_MAIN -#endif - -/* Outline pthread primitives to use in GC_get_[main_]stack_base. */ -#if ((defined(FREEBSD) && defined(__GLIBC__)) /* kFreeBSD */ \ - || defined(LINUX) || defined(NETBSD) || defined(HOST_ANDROID)) && \ - !defined(NO_PTHREAD_GETATTR_NP) -#define HAVE_PTHREAD_GETATTR_NP 1 -#elif defined(FREEBSD) && !defined(__GLIBC__) && !defined(NO_PTHREAD_ATTR_GET_NP) -#define HAVE_PTHREAD_NP_H 1 /* requires include pthread_np.h */ -#define HAVE_PTHREAD_ATTR_GET_NP 1 -#endif - -#if !defined(HAVE_CLOCK_GETTIME) && defined(_POSIX_TIMERS) && (defined(CYGWIN32) || (defined(LINUX) && defined(__USE_POSIX199309))) -#define HAVE_CLOCK_GETTIME 1 -#endif - -#if defined(UNIX_LIKE) && defined(THREADS) && !defined(NO_CANCEL_SAFE) && !defined(HOST_ANDROID) -/* Make the code cancellation-safe. This basically means that we */ -/* ensure that cancellation requests are ignored while we are in */ -/* the collector. This applies only to Posix deferred cancellation; */ -/* we don't handle Posix asynchronous cancellation. */ -/* Note that this only works if pthread_setcancelstate is */ -/* async-signal-safe, at least in the absence of asynchronous */ -/* cancellation. This appears to be true for the glibc version, */ -/* though it is not documented. Without that assumption, there */ -/* seems to be no way to safely wait in a signal handler, which */ -/* we need to do for thread suspension. */ -/* Also note that little other code appears to be cancellation-safe. */ -/* Hence it may make sense to turn this off for performance. */ -#define CANCEL_SAFE -#endif - -#ifdef CANCEL_SAFE -#define IF_CANCEL(x) x -#else -#define IF_CANCEL(x) /* empty */ -#endif - -#if !defined(CAN_HANDLE_FORK) && !defined(NO_HANDLE_FORK) && !defined(HAVE_NO_FORK) && ((defined(GC_PTHREADS) && !defined(NACL) && !defined(GC_WIN32_PTHREADS) && !defined(USE_WINALLOC)) || (defined(DARWIN) && defined(MPROTECT_VDB)) || defined(HANDLE_FORK)) -/* Attempts (where supported and requested) to make GC_malloc work in */ -/* a child process fork'ed from a multi-threaded parent. */ -#define CAN_HANDLE_FORK -#endif - -/* Workaround "failed to create new win32 semaphore" Cygwin fatal error */ -/* during semaphores fixup-after-fork. */ -#if defined(CYGWIN32) && defined(GC_WIN32_THREADS) && defined(CAN_HANDLE_FORK) && !defined(EMULATE_PTHREAD_SEMAPHORE) && !defined(CYGWIN_SEM_FIXUP_AFTER_FORK_BUG_FIXED) -#define EMULATE_PTHREAD_SEMAPHORE -#endif - -#if defined(CAN_HANDLE_FORK) && !defined(CAN_CALL_ATFORK) && !defined(GC_NO_CAN_CALL_ATFORK) && !defined(HOST_TIZEN) && !defined(HURD) && (!defined(HOST_ANDROID) || __ANDROID_API__ >= 21) -/* Have working pthread_atfork(). */ -#define CAN_CALL_ATFORK -#endif - -#if !defined(CAN_HANDLE_FORK) && !defined(HAVE_NO_FORK) && !(defined(CYGWIN32) || defined(SOLARIS) || defined(UNIX_LIKE)) -#define HAVE_NO_FORK -#endif - -#if !defined(USE_MARK_BITS) && !defined(USE_MARK_BYTES) && defined(PARALLEL_MARK) -/* Minimize compare-and-swap usage. */ -#define USE_MARK_BYTES -#endif - -#if (defined(MSWINCE) && !defined(__CEGCC__) || defined(MSWINRT_FLAVOR)) && !defined(NO_GETENV) -#define NO_GETENV -#endif - -#if (defined(NO_GETENV) || defined(MSWINCE)) && !defined(NO_GETENV_WIN32) -#define NO_GETENV_WIN32 -#endif - -#if !defined(MSGBOX_ON_ERROR) && !defined(NO_MSGBOX_ON_ERROR) && !defined(SMALL_CONFIG) && defined(MSWIN32) && !defined(MSWINRT_FLAVOR) && !defined(MSWIN_XBOX1) -/* Show a Windows message box with "OK" button on a GC fatal error. */ -/* Client application is terminated once the user clicks the button. */ -#define MSGBOX_ON_ERROR -#endif - -#ifndef STRTOULL -#if defined(_WIN64) && !defined(__GNUC__) -#define STRTOULL _strtoui64 -#elif defined(_LLP64) || defined(__LLP64__) || defined(_WIN64) -#define STRTOULL strtoull -#else -/* strtoul() fits since sizeof(long) >= sizeof(word). */ -#define STRTOULL strtoul -#endif -#endif /* !STRTOULL */ - -#ifndef GC_WORD_C -#if defined(_WIN64) && !defined(__GNUC__) -#define GC_WORD_C(val) val##ui64 -#elif defined(_LLP64) || defined(__LLP64__) || defined(_WIN64) -#define GC_WORD_C(val) val##ULL -#else -#define GC_WORD_C(val) ((word)val##UL) -#endif -#endif /* !GC_WORD_C */ - -#if defined(__has_feature) -/* __has_feature() is supported. */ -#if __has_feature(address_sanitizer) -#define ADDRESS_SANITIZER -#endif -#if __has_feature(memory_sanitizer) -#define MEMORY_SANITIZER -#endif -#if __has_feature(thread_sanitizer) && defined(THREADS) -#define THREAD_SANITIZER -#endif -#else -#ifdef __SANITIZE_ADDRESS__ -/* GCC v4.8+ */ -#define ADDRESS_SANITIZER -#endif -#if defined(__SANITIZE_THREAD__) && defined(THREADS) -/* GCC v7.1+ */ -#define THREAD_SANITIZER -#endif -#endif /* !__has_feature */ - -#if defined(SPARC) -#define ASM_CLEAR_CODE /* Stack clearing is crucial, and we */ - /* include assembly code to do it well. */ -#endif - -/* Can we save call chain in objects for debugging? */ -/* SET NFRAMES (# of saved frames) and NARGS (#of args for each */ -/* frame) to reasonable values for the platform. */ -/* Set SAVE_CALL_CHAIN if we can. SAVE_CALL_COUNT can be specified */ -/* at build time, though we feel free to adjust it slightly. */ -/* Define NEED_CALLINFO if we either save the call stack or */ -/* GC_ADD_CALLER is defined. */ -/* GC_CAN_SAVE_CALL_STACKS is set in gc.h. */ -#if defined(SPARC) -#define CAN_SAVE_CALL_ARGS -#endif -#if (defined(I386) || defined(X86_64)) && (defined(LINUX) || defined(__GLIBC__)) -/* SAVE_CALL_CHAIN is supported if the code is compiled to save */ -/* frame pointers by default, i.e. no -fomit-frame-pointer flag. */ -#define CAN_SAVE_CALL_ARGS -#endif - -#if defined(SAVE_CALL_COUNT) && !defined(GC_ADD_CALLER) && defined(GC_CAN_SAVE_CALL_STACKS) -#define SAVE_CALL_CHAIN -#endif -#ifdef SAVE_CALL_CHAIN -#if defined(SAVE_CALL_NARGS) && defined(CAN_SAVE_CALL_ARGS) -#define NARGS SAVE_CALL_NARGS -#else -#define NARGS 0 /* Number of arguments to save for each call. */ -#endif -#endif -#ifdef SAVE_CALL_CHAIN -#if !defined(SAVE_CALL_COUNT) || defined(CPPCHECK) -#define NFRAMES 6 /* Number of frames to save. Even for */ - /* alignment reasons. */ -#else -#define NFRAMES ((SAVE_CALL_COUNT + 1) & ~1) -#endif -#define NEED_CALLINFO -#endif /* SAVE_CALL_CHAIN */ -#ifdef GC_ADD_CALLER -#define NFRAMES 1 -#define NARGS 0 -#define NEED_CALLINFO -#endif - -#if (defined(FREEBSD) || (defined(DARWIN) && !defined(_POSIX_C_SOURCE)) || (defined(SOLARIS) && (!defined(_XOPEN_SOURCE) || defined(__EXTENSIONS__))) || defined(LINUX)) && !defined(HAVE_DLADDR) -#define HAVE_DLADDR 1 -#endif - -#if defined(MAKE_BACK_GRAPH) && !defined(DBG_HDRS_ALL) -#define DBG_HDRS_ALL 1 -#endif - -#if defined(POINTER_MASK) && !defined(POINTER_SHIFT) -#define POINTER_SHIFT 0 -#endif - -#if defined(POINTER_SHIFT) && !defined(POINTER_MASK) -#define POINTER_MASK ((word)(signed_word)(-1)) -#endif - -#if !defined(FIXUP_POINTER) && defined(POINTER_MASK) -#define FIXUP_POINTER(p) (p = ((p)&POINTER_MASK) << POINTER_SHIFT) -#endif - -#if defined(FIXUP_POINTER) -#define NEED_FIXUP_POINTER -#else -#define FIXUP_POINTER(p) -#endif - -#if !defined(MARK_BIT_PER_GRANULE) && !defined(MARK_BIT_PER_OBJ) -#define MARK_BIT_PER_GRANULE /* Usually faster */ -#endif - -/* Some static sanity tests. */ -#if !defined(CPPCHECK) -#if defined(MARK_BIT_PER_GRANULE) && defined(MARK_BIT_PER_OBJ) -#error Define only one of MARK_BIT_PER_GRANULE and MARK_BIT_PER_OBJ -#endif -#if defined(STACK_GROWS_UP) && defined(STACK_GROWS_DOWN) -#error Only one of STACK_GROWS_UP and STACK_GROWS_DOWN should be defined -#endif -#if !defined(STACK_GROWS_UP) && !defined(STACK_GROWS_DOWN) -#error One of STACK_GROWS_UP and STACK_GROWS_DOWN should be defined -#endif -#if defined(REDIRECT_MALLOC) && defined(THREADS) && !defined(LINUX) && !defined(REDIRECT_MALLOC_IN_HEADER) -/* May work on other platforms (e.g. Darwin) provided the client */ -/* ensures all the client threads are registered with the GC, */ -/* e.g. by using the preprocessor-based interception of the thread */ -/* primitives (i.e., define GC_THREADS and include gc.h from all */ -/* the client files those are using pthread_create and friends). */ -#endif -#endif /* !CPPCHECK */ - -#ifdef GC_PRIVATE_H -/* This relies on some type definitions from gc_priv.h, from */ -/* where it's normally included. */ -/* */ -/* How to get heap memory from the OS: */ -/* Note that sbrk()-like allocation is preferred, since it */ -/* usually makes it possible to merge consecutively allocated */ -/* chunks. It also avoids unintended recursion with */ -/* REDIRECT_MALLOC macro defined. */ -/* GET_MEM() argument should be of size_t type and have */ -/* no side-effect. GET_MEM() returns HBLKSIZE-aligned chunk; */ -/* 0 is taken to mean failure. */ -/* In case of MMAP_SUPPORTED, the argument must also be */ -/* a multiple of a physical page size. */ -/* GET_MEM is currently not assumed to retrieve 0 filled space, */ -/* though we should perhaps take advantage of the case in which */ -/* does. */ -#define hblk GC_hblk_s -struct hblk; /* See gc_priv.h. */ -#if defined(PCR) -char *real_malloc(size_t bytes); -#define GET_MEM(bytes) HBLKPTR(real_malloc(SIZET_SAT_ADD(bytes, \ - GC_page_size)) + \ - GC_page_size - 1) -#elif defined(OS2) -void *os2_alloc(size_t bytes); -#define GET_MEM(bytes) HBLKPTR((ptr_t)os2_alloc( \ - SIZET_SAT_ADD(bytes, \ - GC_page_size)) + \ - GC_page_size - 1) -#elif defined(NEXT) || defined(DOS4GW) || defined(NONSTOP) || (defined(AMIGA) && !defined(GC_AMIGA_FASTALLOC)) || (defined(SOLARIS) && !defined(USE_MMAP)) || defined(RTEMS) || defined(__CC_ARM) -#define GET_MEM(bytes) HBLKPTR((size_t)calloc(1, \ - SIZET_SAT_ADD(bytes, \ - GC_page_size)) + \ - GC_page_size - 1) -#elif defined(MSWIN_XBOX1) -ptr_t GC_durango_get_mem(size_t bytes); -#define GET_MEM(bytes) (struct hblk *)GC_durango_get_mem(bytes) -#elif defined(MSWIN32) || defined(CYGWIN32) -ptr_t GC_win32_get_mem(size_t bytes); -#define GET_MEM(bytes) (struct hblk *)GC_win32_get_mem(bytes) -#elif defined(MACOS) -#if defined(USE_TEMPORARY_MEMORY) -Ptr GC_MacTemporaryNewPtr(size_t size, Boolean clearMemory); -#define GET_MEM(bytes) HBLKPTR(GC_MacTemporaryNewPtr( \ - SIZET_SAT_ADD(bytes, \ - GC_page_size), \ - true) + \ - GC_page_size - 1) -#else -#define GET_MEM(bytes) HBLKPTR(NewPtrClear(SIZET_SAT_ADD(bytes, \ - GC_page_size)) + \ - GC_page_size - 1) -#endif -#elif defined(MSWINCE) -ptr_t GC_wince_get_mem(size_t bytes); -#define GET_MEM(bytes) (struct hblk *)GC_wince_get_mem(bytes) -#elif defined(AMIGA) && defined(GC_AMIGA_FASTALLOC) -void *GC_amiga_get_mem(size_t bytes); -#define GET_MEM(bytes) HBLKPTR((size_t)GC_amiga_get_mem( \ - SIZET_SAT_ADD(bytes, \ - GC_page_size)) + \ - GC_page_size - 1) -#elif defined(PLATFORM_GETMEM) -void *platform_get_mem(size_t bytes); -#define GET_MEM(bytes) (struct hblk *)platform_get_mem(bytes) -#elif defined(SN_TARGET_PS3) -void *ps3_get_mem(size_t bytes); -#define GET_MEM(bytes) (struct hblk *)ps3_get_mem(bytes) -#elif defined(SN_TARGET_PSP2) -void *psp2_get_mem(size_t bytes); -#define GET_MEM(bytes) (struct hblk *)psp2_get_mem(bytes) -#elif defined(NINTENDO_SWITCH) -void *switch_get_mem(size_t bytes); -#define GET_MEM(bytes) (struct hblk *)switch_get_mem(bytes) -#elif defined(HAIKU) -ptr_t GC_haiku_get_mem(size_t bytes); -#define GET_MEM(bytes) (struct hblk *)GC_haiku_get_mem(bytes) -#elif defined(EMSCRIPTEN_TINY) -void *emmalloc_memalign(size_t alignment, size_t size); -#define GET_MEM(bytes) (struct hblk *)emmalloc_memalign(GC_page_size, bytes) -#else -ptr_t GC_unix_get_mem(size_t bytes); -#define GET_MEM(bytes) (struct hblk *)GC_unix_get_mem(bytes) -#endif -#endif /* GC_PRIVATE_H */ - -EXTERN_C_END - -#endif /* GCCONFIG_H */ diff --git a/vendor/bdwgc/private/specific.h b/vendor/bdwgc/private/specific.h deleted file mode 100644 index e7dbb3c5..00000000 --- a/vendor/bdwgc/private/specific.h +++ /dev/null @@ -1,132 +0,0 @@ -/* - * Copyright (c) 2000 by Hewlett-Packard Company. All rights reserved. - * - * THIS MATERIAL IS PROVIDED AS IS, WITH ABSOLUTELY NO WARRANTY EXPRESSED - * OR IMPLIED. ANY USE IS AT YOUR OWN RISK. - * - * Permission is hereby granted to use or copy this program - * for any purpose, provided the above notices are retained on all copies. - * Permission to modify the code and to distribute modified code is granted, - * provided the above notices are retained, and a notice that the code was - * modified is included with the above copyright notice. - */ - -/* - * This is a reimplementation of a subset of the pthread_getspecific/setspecific - * interface. This appears to outperform the standard linuxthreads one - * by a significant margin. - * The major restriction is that each thread may only make a single - * pthread_setspecific call on a single key. (The current data structure - * doesn't really require that. The restriction should be easily removable.) - * We don't currently support the destruction functions, though that - * could be done. - * We also currently assume that only one pthread_setspecific call - * can be executed at a time, though that assumption would be easy to remove - * by adding a lock. - */ - -#ifndef GC_SPECIFIC_H -#define GC_SPECIFIC_H - -#include - -EXTERN_C_BEGIN - -/* Called during key creation or setspecific. */ -/* For the GC we already hold lock. */ -/* Currently allocated objects leak on thread exit. */ -/* That's hard to fix, but OK if we allocate garbage */ -/* collected memory. */ -#define MALLOC_CLEAR(n) GC_INTERNAL_MALLOC(n, NORMAL) - -#define TS_CACHE_SIZE 1024 -#define CACHE_HASH(n) ((((n) >> 8) ^ (n)) & (TS_CACHE_SIZE - 1)) - -#define TS_HASH_SIZE 1024 -#define HASH(p) \ - ((unsigned)((((word)(p)) >> 8) ^ (word)(p)) & (TS_HASH_SIZE - 1)) - -#ifdef GC_ASSERTIONS -/* Thread-local storage is not guaranteed to be scanned by GC. */ -/* We hide values stored in "specific" entries for a test purpose. */ -typedef GC_hidden_pointer ts_entry_value_t; -#define TS_HIDE_VALUE(p) GC_HIDE_POINTER(p) -#define TS_REVEAL_PTR(p) GC_REVEAL_POINTER(p) -#else -typedef void *ts_entry_value_t; -#define TS_HIDE_VALUE(p) (p) -#define TS_REVEAL_PTR(p) (p) -#endif - -/* An entry describing a thread-specific value for a given thread. */ -/* All such accessible structures preserve the invariant that if either */ -/* thread is a valid pthread id or qtid is a valid "quick thread id" */ -/* for a thread, then value holds the corresponding thread specific */ -/* value. This invariant must be preserved at ALL times, since */ -/* asynchronous reads are allowed. */ -typedef struct thread_specific_entry { - volatile AO_t qtid; /* quick thread id, only for cache */ - ts_entry_value_t value; - struct thread_specific_entry *next; - pthread_t thread; -} tse; - -/* We represent each thread-specific datum as two tables. The first is */ -/* a cache, indexed by a "quick thread identifier". The "quick" thread */ -/* identifier is an easy to compute value, which is guaranteed to */ -/* determine the thread, though a thread may correspond to more than */ -/* one value. We typically use the address of a page in the stack. */ -/* The second is a hash table, indexed by pthread_self(). It is used */ -/* only as a backup. */ - -/* Return the "quick thread id". Default version. Assumes page size, */ -/* or at least thread stack separation, is at least 4 KB. */ -/* Must be defined so that it never returns 0. (Page 0 can't really be */ -/* part of any stack, since that would make 0 a valid stack pointer.) */ -#define quick_thread_id() (((word)GC_approx_sp()) >> 12) - -#define INVALID_QTID ((word)0) -#define INVALID_THREADID ((pthread_t)0) - -union ptse_ao_u { - tse *p; - volatile AO_t ao; -}; - -typedef struct thread_specific_data { - tse *volatile cache[TS_CACHE_SIZE]; - /* A faster index to the hash table */ - union ptse_ao_u hash[TS_HASH_SIZE]; - pthread_mutex_t lock; -} tsd; - -typedef tsd *GC_key_t; - -#define GC_key_create(key, d) GC_key_create_inner(key) -GC_INNER int GC_key_create_inner(tsd **key_ptr); -GC_INNER int GC_setspecific(tsd *key, void *value); -#define GC_remove_specific(key) \ - GC_remove_specific_after_fork(key, pthread_self()) -GC_INNER void GC_remove_specific_after_fork(tsd *key, pthread_t t); - -/* An internal version of getspecific that assumes a cache miss. */ -GC_INNER void *GC_slow_getspecific(tsd *key, word qtid, - tse *volatile *cache_entry); - -/* GC_INLINE is defined in gc_priv.h. */ -GC_INLINE void *GC_getspecific(tsd *key) { - word qtid = quick_thread_id(); - tse *volatile *entry_ptr = &key->cache[CACHE_HASH(qtid)]; - tse *entry = *entry_ptr; /* Must be loaded only once. */ - - GC_ASSERT(qtid != INVALID_QTID); - if (EXPECT(entry->qtid == qtid, TRUE)) { - GC_ASSERT(entry->thread == pthread_self()); - return TS_REVEAL_PTR(entry->value); - } - return GC_slow_getspecific(key, qtid, entry_ptr); -} - -EXTERN_C_END - -#endif /* GC_SPECIFIC_H */ diff --git a/vendor/bdwgc/ptr_chck.c b/vendor/bdwgc/ptr_chck.c deleted file mode 100644 index be55258c..00000000 --- a/vendor/bdwgc/ptr_chck.c +++ /dev/null @@ -1,293 +0,0 @@ -/* - * Copyright (c) 1991-1994 by Xerox Corporation. All rights reserved. - * - * THIS MATERIAL IS PROVIDED AS IS, WITH ABSOLUTELY NO WARRANTY EXPRESSED - * OR IMPLIED. ANY USE IS AT YOUR OWN RISK. - * - * Permission is hereby granted to use or copy this program - * for any purpose, provided the above notices are retained on all copies. - * Permission to modify the code and to distribute modified code is granted, - * provided the above notices are retained, and a notice that the code was - * modified is included with the above copyright notice. - */ - -#ifdef _WIN32 -#define WIN32_LEAN_AND_MEAN -#include -#endif -#include "private/gc_pmark.h" - -/* - * These are checking routines calls to which could be inserted by a - * preprocessor to validate C pointer arithmetic. - */ - -STATIC void GC_CALLBACK GC_default_same_obj_print_proc(void *p, void *q) { - ABORT_ARG2("GC_same_obj test failed", - ": %p and %p are not in the same object", p, q); -} - -GC_same_obj_print_proc_t GC_same_obj_print_proc = - GC_default_same_obj_print_proc; - -GC_API void *GC_CALL GC_same_obj(void *p, void *q) { - struct hblk *h; - hdr *hhdr; - ptr_t base, limit; - word sz; - - if (!EXPECT(GC_is_initialized, TRUE)) GC_init(); - hhdr = HDR((word)p); - if (NULL == hhdr) { - if (divHBLKSZ((word)p) != divHBLKSZ((word)q) && HDR((word)q) != NULL) { - goto fail; - } - return p; - } - /* If it's a pointer to the middle of a large object, move it */ - /* to the beginning. */ - if (IS_FORWARDING_ADDR_OR_NIL(hhdr)) { - h = HBLKPTR(p) - (word)hhdr; - hhdr = HDR(h); - while (IS_FORWARDING_ADDR_OR_NIL(hhdr)) { - h = FORWARDED_ADDR(h, hhdr); - hhdr = HDR(h); - } - limit = (ptr_t)h + hhdr->hb_sz; - if ((word)p >= (word)limit || (word)q >= (word)limit || (word)q < (word)h) { - goto fail; - } - return p; - } - sz = hhdr->hb_sz; - if (sz > MAXOBJBYTES) { - base = (ptr_t)HBLKPTR(p); - limit = base + sz; - if ((word)p >= (word)limit) { - goto fail; - } - } else { - size_t offset; - size_t pdispl = HBLKDISPL(p); - - offset = pdispl % sz; - if (HBLKPTR(p) != HBLKPTR(q)) goto fail; - /* W/o this check, we might miss an error if */ - /* q points to the first object on a page, and */ - /* points just before the page. */ - base = (ptr_t)p - offset; - limit = base + sz; - } - /* [base, limit) delimits the object containing p, if any. */ - /* If p is not inside a valid object, then either q is */ - /* also outside any valid object, or it is outside */ - /* [base, limit). */ - if ((word)q >= (word)limit || (word)q < (word)base) { - goto fail; - } - return p; -fail: - (*GC_same_obj_print_proc)((ptr_t)p, (ptr_t)q); - return p; -} - -STATIC void GC_CALLBACK GC_default_is_valid_displacement_print_proc(void *p) { - ABORT_ARG1("GC_is_valid_displacement test failed", ": %p not valid", p); -} - -GC_valid_ptr_print_proc_t GC_is_valid_displacement_print_proc = - GC_default_is_valid_displacement_print_proc; - -GC_API void *GC_CALL GC_is_valid_displacement(void *p) { - hdr *hhdr; - word pdispl; - word offset; - struct hblk *h; - word sz; - - if (!EXPECT(GC_is_initialized, TRUE)) GC_init(); - if (NULL == p) return NULL; - hhdr = HDR((word)p); - if (NULL == hhdr) return p; - h = HBLKPTR(p); - if (GC_all_interior_pointers) { - while (IS_FORWARDING_ADDR_OR_NIL(hhdr)) { - h = FORWARDED_ADDR(h, hhdr); - hhdr = HDR(h); - } - } else if (IS_FORWARDING_ADDR_OR_NIL(hhdr)) { - goto fail; - } - sz = hhdr->hb_sz; - pdispl = HBLKDISPL(p); - offset = pdispl % sz; - if ((sz > MAXOBJBYTES && (word)p >= (word)h + sz) || !GC_valid_offsets[offset] || ((word)p + (sz - offset) > (word)(h + 1) && !IS_FORWARDING_ADDR_OR_NIL(HDR(h + 1)))) { - goto fail; - } - return p; -fail: - (*GC_is_valid_displacement_print_proc)((ptr_t)p); - return p; -} - -STATIC void GC_CALLBACK GC_default_is_visible_print_proc(void *p) { - ABORT_ARG1("GC_is_visible test failed", ": %p not GC-visible", p); -} - -GC_valid_ptr_print_proc_t GC_is_visible_print_proc = - GC_default_is_visible_print_proc; - -#ifndef THREADS -/* Could p be a stack address? */ -STATIC GC_bool GC_on_stack(void *p) { -#ifdef STACK_GROWS_DOWN - if ((word)p >= (word)GC_approx_sp() && (word)p < (word)GC_stackbottom) { - return TRUE; - } -#else - if ((word)p <= (word)GC_approx_sp() && (word)p > (word)GC_stackbottom) { - return TRUE; - } -#endif - return FALSE; -} -#endif /* !THREADS */ - -GC_API void *GC_CALL GC_is_visible(void *p) { - hdr *hhdr; - - if ((word)p & (ALIGNMENT - 1)) goto fail; - if (!EXPECT(GC_is_initialized, TRUE)) GC_init(); -#ifdef THREADS - hhdr = HDR((word)p); - if (hhdr != NULL && NULL == GC_base(p)) { - goto fail; - } else { - /* May be inside thread stack. We can't do much. */ - return p; - } -#else - /* Check stack first: */ - if (GC_on_stack(p)) return p; - hhdr = HDR((word)p); - if (NULL == hhdr) { - if (GC_is_static_root(p)) return p; - /* Else do it again correctly: */ -#if defined(DYNAMIC_LOADING) || defined(MSWIN32) || defined(MSWINCE) || defined(CYGWIN32) || defined(PCR) - if (!GC_no_dls) { - GC_register_dynamic_libraries(); - if (GC_is_static_root(p)) return p; - } -#endif - goto fail; - } else { - /* p points to the heap. */ - word descr; - ptr_t base = (ptr_t)GC_base(p); - /* TODO: should GC_base be manually inlined? */ - - if (NULL == base) goto fail; - if (HBLKPTR(base) != HBLKPTR(p)) - hhdr = HDR(base); - descr = hhdr->hb_descr; - retry: - switch (descr & GC_DS_TAGS) { - case GC_DS_LENGTH: - if ((word)p - (word)base > descr) goto fail; - break; - case GC_DS_BITMAP: - if ((word)p - (word)base >= WORDS_TO_BYTES(BITMAP_BITS) || ((word)p & (sizeof(word) - 1))) goto fail; - if (!(((word)1 << (WORDSZ - ((ptr_t)p - (ptr_t)base) - 1)) & descr)) goto fail; - break; - case GC_DS_PROC: - /* We could try to decipher this partially. */ - /* For now we just punt. */ - break; - case GC_DS_PER_OBJECT: - if (!(descr & SIGNB)) { - descr = *(word *)((ptr_t)base + (descr & ~GC_DS_TAGS)); - } else { - ptr_t type_descr = *(ptr_t *)base; - descr = *(word *)(type_descr - (descr - (word)(GC_DS_PER_OBJECT - GC_INDIR_PER_OBJ_BIAS))); - } - goto retry; - } - return p; - } -#endif -fail: - (*GC_is_visible_print_proc)((ptr_t)p); - return p; -} - -GC_API void *GC_CALL GC_pre_incr(void **p, ptrdiff_t how_much) { - void *initial = *p; - void *result = GC_same_obj((void *)((ptr_t)initial + how_much), initial); - - if (!GC_all_interior_pointers) { - (void)GC_is_valid_displacement(result); - } - *p = result; - return result; /* updated pointer */ -} - -GC_API void *GC_CALL GC_post_incr(void **p, ptrdiff_t how_much) { - void *initial = *p; - void *result = GC_same_obj((void *)((ptr_t)initial + how_much), initial); - - if (!GC_all_interior_pointers) { - (void)GC_is_valid_displacement(result); - } - *p = result; - return initial; /* original *p */ -} - -GC_API void GC_CALL GC_set_same_obj_print_proc(GC_same_obj_print_proc_t fn) { - GC_ASSERT(NONNULL_ARG_NOT_NULL(fn)); - LOCK(); - GC_same_obj_print_proc = fn; - UNLOCK(); -} - -GC_API GC_same_obj_print_proc_t GC_CALL GC_get_same_obj_print_proc(void) { - GC_same_obj_print_proc_t fn; - - LOCK(); - fn = GC_same_obj_print_proc; - UNLOCK(); - return fn; -} - -GC_API void GC_CALL GC_set_is_valid_displacement_print_proc( - GC_valid_ptr_print_proc_t fn) { - GC_ASSERT(NONNULL_ARG_NOT_NULL(fn)); - LOCK(); - GC_is_valid_displacement_print_proc = fn; - UNLOCK(); -} - -GC_API GC_valid_ptr_print_proc_t GC_CALL -GC_get_is_valid_displacement_print_proc(void) { - GC_valid_ptr_print_proc_t fn; - - LOCK(); - fn = GC_is_valid_displacement_print_proc; - UNLOCK(); - return fn; -} - -GC_API void GC_CALL GC_set_is_visible_print_proc(GC_valid_ptr_print_proc_t fn) { - GC_ASSERT(NONNULL_ARG_NOT_NULL(fn)); - LOCK(); - GC_is_visible_print_proc = fn; - UNLOCK(); -} - -GC_API GC_valid_ptr_print_proc_t GC_CALL GC_get_is_visible_print_proc(void) { - GC_valid_ptr_print_proc_t fn; - - LOCK(); - fn = GC_is_visible_print_proc; - UNLOCK(); - return fn; -} diff --git a/vendor/bdwgc/reclaim.c b/vendor/bdwgc/reclaim.c deleted file mode 100644 index ef93a737..00000000 --- a/vendor/bdwgc/reclaim.c +++ /dev/null @@ -1,820 +0,0 @@ -/* - * Copyright 1988, 1989 Hans-J. Boehm, Alan J. Demers - * Copyright (c) 1991-1996 by Xerox Corporation. All rights reserved. - * Copyright (c) 1996-1999 by Silicon Graphics. All rights reserved. - * Copyright (c) 1999-2004 Hewlett-Packard Development Company, L.P. - * Copyright (c) 2009-2022 Ivan Maidanski - * - * THIS MATERIAL IS PROVIDED AS IS, WITH ABSOLUTELY NO WARRANTY EXPRESSED - * OR IMPLIED. ANY USE IS AT YOUR OWN RISK. - * - * Permission is hereby granted to use or copy this program - * for any purpose, provided the above notices are retained on all copies. - * Permission to modify the code and to distribute modified code is granted, - * provided the above notices are retained, and a notice that the code was - * modified is included with the above copyright notice. - */ - -#ifdef _WIN32 -#define WIN32_LEAN_AND_MEAN -#include -#endif -#include "private/gc_priv.h" - -#ifdef ENABLE_DISCLAIM -#include "gc/gc_disclaim.h" -#endif - -GC_INNER signed_word GC_bytes_found = 0; -/* Number of bytes of memory reclaimed */ -/* minus the number of bytes originally */ -/* on free lists which we had to drop. */ - -#if defined(PARALLEL_MARK) -GC_INNER signed_word GC_fl_builder_count = 0; -/* Number of threads currently building free lists without */ -/* holding GC lock. It is not safe to collect if this is */ -/* nonzero. Also, together with the mark lock, it is used as */ -/* a semaphore during marker threads startup. */ -#endif /* PARALLEL_MARK */ - -/* We defer printing of leaked objects until we're done with the GC */ -/* cycle, since the routine for printing objects needs to run outside */ -/* the collector, e.g. without the allocation lock. */ -#ifndef MAX_LEAKED -#define MAX_LEAKED 40 -#endif -STATIC ptr_t GC_leaked[MAX_LEAKED] = {NULL}; -STATIC unsigned GC_n_leaked = 0; - -#ifdef AO_HAVE_store -GC_INNER volatile AO_t GC_have_errors = 0; -#else -GC_INNER GC_bool GC_have_errors = FALSE; -#endif - -#if !defined(EAGER_SWEEP) && defined(ENABLE_DISCLAIM) -STATIC void GC_reclaim_unconditionally_marked(void); -#endif - -GC_INLINE void GC_add_leaked(ptr_t leaked) { - GC_ASSERT(I_HOLD_LOCK()); -#ifndef SHORT_DBG_HDRS - if (GC_findleak_delay_free && !GC_check_leaked(leaked)) - return; -#endif - - GC_SET_HAVE_ERRORS(); - if (GC_n_leaked < MAX_LEAKED) { - GC_leaked[GC_n_leaked++] = leaked; - /* Make sure it's not reclaimed this cycle */ - GC_set_mark_bit(leaked); - } -} - -/* Print all objects on the list after printing any smashed objects. */ -/* Clear both lists. Called without the allocation lock held. */ -GC_INNER void GC_print_all_errors(void) { - static GC_bool printing_errors = FALSE; - GC_bool have_errors; - unsigned i, n_leaked; - ptr_t leaked[MAX_LEAKED]; - - LOCK(); - if (printing_errors) { - UNLOCK(); - return; - } - have_errors = get_have_errors(); - printing_errors = TRUE; - n_leaked = GC_n_leaked; - if (n_leaked > 0) { - GC_ASSERT(n_leaked <= MAX_LEAKED); - BCOPY(GC_leaked, leaked, n_leaked * sizeof(ptr_t)); - GC_n_leaked = 0; - BZERO(GC_leaked, n_leaked * sizeof(ptr_t)); - } - UNLOCK(); - - if (GC_debugging_started) { - GC_print_all_smashed(); - } else { - have_errors = FALSE; - } - - if (n_leaked > 0) { - GC_err_printf("Found %u leaked objects:\n", n_leaked); - have_errors = TRUE; - } - for (i = 0; i < n_leaked; i++) { - ptr_t p = leaked[i]; -#ifndef SKIP_LEAKED_OBJECTS_PRINTING - GC_print_heap_obj(p); -#endif - GC_free(p); - } - - if (have_errors -#ifndef GC_ABORT_ON_LEAK - && GETENV("GC_ABORT_ON_LEAK") != NULL -#endif - ) { - ABORT("Leaked or smashed objects encountered"); - } - - LOCK(); - printing_errors = FALSE; - UNLOCK(); -} - -/* - * reclaim phase - * - */ - -/* Test whether a block is completely empty, i.e. contains no marked */ -/* objects. This does not require the block to be in physical memory. */ -GC_INNER GC_bool GC_block_empty(hdr *hhdr) { - return 0 == hhdr->hb_n_marks; -} - -STATIC GC_bool GC_block_nearly_full(hdr *hhdr, word sz) { - return hhdr->hb_n_marks > HBLK_OBJS(sz) * 7 / 8; -} - -/* TODO: This should perhaps again be specialized for USE_MARK_BYTES */ -/* and USE_MARK_BITS cases. */ - -GC_INLINE word *GC_clear_block(word *p, word sz, signed_word *count) { - word *q = (word *)((ptr_t)p + sz); - - /* Clear object, advance p to next object in the process. */ -#ifdef USE_MARK_BYTES - GC_ASSERT((sz & 1) == 0); - GC_ASSERT(((word)p & (2 * sizeof(word) - 1)) == 0); - p[1] = 0; - p += 2; - while ((word)p < (word)q) { - CLEAR_DOUBLE(p); - p += 2; - } -#else - p++; /* Skip link field */ - while ((word)p < (word)q) { - *p++ = 0; - } -#endif - *count += sz; - return p; -} - -/* - * Restore unmarked small objects in h of size sz to the object - * free list. Returns the new list. - * Clears unmarked objects. Sz is in bytes. - */ -STATIC ptr_t GC_reclaim_clear(struct hblk *hbp, hdr *hhdr, word sz, - ptr_t list, signed_word *count) { - word bit_no = 0; - ptr_t p, plim; - - GC_ASSERT(hhdr == GC_find_header((ptr_t)hbp)); -#ifndef THREADS - GC_ASSERT(sz == hhdr->hb_sz); -#else - /* Skip the assertion because of a potential race with GC_realloc. */ -#endif - GC_ASSERT((sz & (BYTES_PER_WORD - 1)) == 0); - p = hbp->hb_body; - plim = p + HBLKSIZE - sz; - - /* go through all words in block */ - while ((word)p <= (word)plim) { - if (mark_bit_from_hdr(hhdr, bit_no)) { - p += sz; - } else { - /* Object is available - put it on list. */ - obj_link(p) = list; - list = p; - - p = (ptr_t)GC_clear_block((word *)p, sz, count); - } - bit_no += MARK_BIT_OFFSET(sz); - } - return list; -} - -/* The same thing, but don't clear objects: */ -STATIC ptr_t GC_reclaim_uninit(struct hblk *hbp, hdr *hhdr, word sz, - ptr_t list, signed_word *count) { - word bit_no = 0; - word *p, *plim; - signed_word n_bytes_found = 0; - -#ifndef THREADS - GC_ASSERT(sz == hhdr->hb_sz); -#endif - p = (word *)(hbp->hb_body); - plim = (word *)((ptr_t)hbp + HBLKSIZE - sz); - - /* go through all words in block */ - while ((word)p <= (word)plim) { - if (!mark_bit_from_hdr(hhdr, bit_no)) { - n_bytes_found += sz; - /* object is available - put on list */ - obj_link(p) = list; - list = ((ptr_t)p); - } - p = (word *)((ptr_t)p + sz); - bit_no += MARK_BIT_OFFSET(sz); - } - *count += n_bytes_found; - return list; -} - -#ifdef ENABLE_DISCLAIM -/* Call reclaim notifier for block's kind on each unmarked object in */ -/* block, all within a pair of corresponding enter/leave callbacks. */ -STATIC ptr_t GC_disclaim_and_reclaim(struct hblk *hbp, hdr *hhdr, word sz, - ptr_t list, signed_word *count) { - word bit_no = 0; - ptr_t p, plim; - struct obj_kind *ok = &GC_obj_kinds[hhdr->hb_obj_kind]; - int(GC_CALLBACK * disclaim)(void *) = ok->ok_disclaim_proc; - -#ifndef THREADS - GC_ASSERT(sz == hhdr->hb_sz); -#endif - p = hbp->hb_body; - plim = p + HBLKSIZE - sz; - - for (; (word)p <= (word)plim; bit_no += MARK_BIT_OFFSET(sz)) { - if (mark_bit_from_hdr(hhdr, bit_no)) { - p += sz; - } else if (disclaim(p)) { - set_mark_bit_from_hdr(hhdr, bit_no); - hhdr->hb_n_marks++; - p += sz; - } else { - obj_link(p) = list; - list = p; - p = (ptr_t)GC_clear_block((word *)p, sz, count); - } - } - return list; -} -#endif /* ENABLE_DISCLAIM */ - -/* Don't really reclaim objects, just check for unmarked ones: */ -STATIC void GC_reclaim_check(struct hblk *hbp, hdr *hhdr, word sz) { - word bit_no; - ptr_t p, plim; - -#ifndef THREADS - GC_ASSERT(sz == hhdr->hb_sz); -#endif - /* go through all words in block */ - p = hbp->hb_body; - plim = p + HBLKSIZE - sz; - for (bit_no = 0; (word)p <= (word)plim; - p += sz, bit_no += MARK_BIT_OFFSET(sz)) { - if (!mark_bit_from_hdr(hhdr, bit_no)) { - GC_add_leaked(p); - } - } -} - -/* Is a pointer-free block? Same as IS_PTRFREE macro (in os_dep.c) but */ -/* uses unordered atomic access to avoid racing with GC_realloc. */ -#ifdef AO_HAVE_load -#define IS_PTRFREE_SAFE(hhdr) \ - (AO_load((volatile AO_t *)&(hhdr)->hb_descr) == 0) -#else -/* No race as GC_realloc holds the lock while updating hb_descr. */ -#define IS_PTRFREE_SAFE(hhdr) ((hhdr)->hb_descr == 0) -#endif - -/* - * Generic procedure to rebuild a free list in hbp. - * Also called directly from GC_malloc_many. - * Sz is now in bytes. - */ -GC_INNER ptr_t GC_reclaim_generic(struct hblk *hbp, hdr *hhdr, size_t sz, - GC_bool init, ptr_t list, - signed_word *count) { - ptr_t result; - - GC_ASSERT(GC_find_header((ptr_t)hbp) == hhdr); -#ifndef GC_DISABLE_INCREMENTAL - GC_remove_protection(hbp, 1, IS_PTRFREE_SAFE(hhdr)); -#endif -#ifdef ENABLE_DISCLAIM - if ((hhdr->hb_flags & HAS_DISCLAIM) != 0) { - result = GC_disclaim_and_reclaim(hbp, hhdr, sz, list, count); - } else -#endif - /* else */ if (init || GC_debugging_started) { - result = GC_reclaim_clear(hbp, hhdr, sz, list, count); - } else { - GC_ASSERT(IS_PTRFREE_SAFE(hhdr)); - result = GC_reclaim_uninit(hbp, hhdr, sz, list, count); - } - if (IS_UNCOLLECTABLE(hhdr->hb_obj_kind)) GC_set_hdr_marks(hhdr); - return result; -} - -/* - * Restore unmarked small objects in the block pointed to by hbp - * to the appropriate object free list. - * If entirely empty blocks are to be completely deallocated, then - * caller should perform that check. - */ -STATIC void GC_reclaim_small_nonempty_block(struct hblk *hbp, word sz, - GC_bool report_if_found) { - hdr *hhdr = HDR(hbp); - struct obj_kind *ok = &GC_obj_kinds[hhdr->hb_obj_kind]; - void **flh = &(ok->ok_freelist[BYTES_TO_GRANULES(sz)]); - - hhdr->hb_last_reclaimed = (unsigned short)GC_gc_no; - - if (report_if_found) { - GC_reclaim_check(hbp, hhdr, sz); - } else { - *flh = GC_reclaim_generic(hbp, hhdr, sz, ok->ok_init, - (ptr_t)(*flh), &GC_bytes_found); - } -} - -#ifdef ENABLE_DISCLAIM -STATIC void GC_disclaim_and_reclaim_or_free_small_block(struct hblk *hbp) { - hdr *hhdr = HDR(hbp); - word sz = hhdr->hb_sz; - struct obj_kind *ok = &GC_obj_kinds[hhdr->hb_obj_kind]; - void **flh = &(ok->ok_freelist[BYTES_TO_GRANULES(sz)]); - void *flh_next; - - hhdr->hb_last_reclaimed = (unsigned short)GC_gc_no; - flh_next = GC_reclaim_generic(hbp, hhdr, sz, ok->ok_init, - (ptr_t)(*flh), &GC_bytes_found); - if (hhdr->hb_n_marks) - *flh = flh_next; - else { - GC_bytes_found += HBLKSIZE; - GC_freehblk(hbp); - } -} -#endif /* ENABLE_DISCLAIM */ - -/* - * Restore an unmarked large object or an entirely empty blocks of small objects - * to the heap block free list. - * Otherwise enqueue the block for later processing - * by GC_reclaim_small_nonempty_block. - * If report_if_found is TRUE, then process any block immediately, and - * simply report free objects; do not actually reclaim them. - */ -STATIC void GC_CALLBACK GC_reclaim_block(struct hblk *hbp, - GC_word report_if_found) { - hdr *hhdr = HDR(hbp); - word sz; /* size of objects in current block */ - struct obj_kind *ok = &GC_obj_kinds[hhdr->hb_obj_kind]; - -#ifdef AO_HAVE_load - /* Atomic access is used to avoid racing with GC_realloc. */ - sz = (word)AO_load((volatile AO_t *)&(hhdr->hb_sz)); -#else - /* No race as GC_realloc holds the lock while updating hb_sz. */ - sz = hhdr->hb_sz; -#endif - if (sz > MAXOBJBYTES) { /* 1 big object */ - if (!mark_bit_from_hdr(hhdr, 0)) { - if (report_if_found) { - GC_add_leaked((ptr_t)hbp); - } else { -#ifdef ENABLE_DISCLAIM - if (EXPECT(hhdr->hb_flags & HAS_DISCLAIM, 0)) { - if (ok->ok_disclaim_proc(hbp)) { - /* Not disclaimed => resurrect the object. */ - set_mark_bit_from_hdr(hhdr, 0); - goto in_use; - } - } -#endif - if (sz > HBLKSIZE) { - GC_large_allocd_bytes -= HBLKSIZE * OBJ_SZ_TO_BLOCKS(sz); - } - GC_bytes_found += sz; - GC_freehblk(hbp); - } - } else { -#ifdef ENABLE_DISCLAIM - in_use: -#endif - if (IS_PTRFREE_SAFE(hhdr)) { - GC_atomic_in_use += sz; - } else { - GC_composite_in_use += sz; - } - } - } else { - GC_bool empty = GC_block_empty(hhdr); -#ifdef PARALLEL_MARK - /* Count can be low or one too high because we sometimes */ - /* have to ignore decrements. Objects can also potentially */ - /* be repeatedly marked by each marker. */ - /* Here we assume 3 markers at most, but this is extremely */ - /* unlikely to fail spuriously with more. And if it does, it */ - /* should be looked at. */ - GC_ASSERT(sz != 0 && (GC_markers_m1 > 1 ? 3 : GC_markers_m1 + 1) * (HBLKSIZE / sz + 1) + 16 >= hhdr->hb_n_marks); -#else - GC_ASSERT(sz * hhdr->hb_n_marks <= HBLKSIZE); -#endif - if (report_if_found) { - GC_reclaim_small_nonempty_block(hbp, sz, - TRUE /* report_if_found */); - } else if (empty) { -#ifdef ENABLE_DISCLAIM - if ((hhdr->hb_flags & HAS_DISCLAIM) != 0) { - GC_disclaim_and_reclaim_or_free_small_block(hbp); - } else -#endif - /* else */ { - GC_bytes_found += HBLKSIZE; - GC_freehblk(hbp); - } - } else if (GC_find_leak || !GC_block_nearly_full(hhdr, sz)) { - /* group of smaller objects, enqueue the real work */ - struct hblk **rlh = ok->ok_reclaim_list; - - if (rlh != NULL) { - rlh += BYTES_TO_GRANULES(sz); - hhdr->hb_next = *rlh; - *rlh = hbp; - } - } /* else not worth salvaging. */ - /* We used to do the nearly_full check later, but we */ - /* already have the right cache context here. Also */ - /* doing it here avoids some silly lock contention in */ - /* GC_malloc_many. */ - if (IS_PTRFREE_SAFE(hhdr)) { - GC_atomic_in_use += sz * hhdr->hb_n_marks; - } else { - GC_composite_in_use += sz * hhdr->hb_n_marks; - } - } -} - -#if !defined(NO_DEBUGGING) -/* Routines to gather and print heap block info */ -/* intended for debugging. Otherwise should be called */ -/* with lock. */ - -struct Print_stats { - size_t number_of_blocks; - size_t total_bytes; -}; - -#ifdef USE_MARK_BYTES - -/* Return the number of set mark bits in the given header. */ -/* Remains externally visible as used by GNU GCJ currently. */ -unsigned GC_n_set_marks(hdr *hhdr) { - unsigned result = 0; - word i; - word offset = MARK_BIT_OFFSET(hhdr->hb_sz); - word limit = FINAL_MARK_BIT(hhdr->hb_sz); - - for (i = 0; i < limit; i += offset) { - result += hhdr->hb_marks[i]; - } - GC_ASSERT(hhdr->hb_marks[limit]); /* the one set past the end */ - return result; -} - -#else - -/* Number of set bits in a word. Not performance critical. */ -static unsigned count_ones(word n) { - unsigned result = 0; - - for (; n > 0; n >>= 1) - if (n & 1) result++; - - return result; -} - -unsigned GC_n_set_marks(hdr *hhdr) { - unsigned result = 0; - word sz = hhdr->hb_sz; - word i; -#ifdef MARK_BIT_PER_OBJ - word n_objs = HBLK_OBJS(sz); - word n_mark_words = divWORDSZ(n_objs > 0 ? n_objs : 1); /* round down */ - - for (i = 0; i <= n_mark_words; i++) { - result += count_ones(hhdr->hb_marks[i]); - } -#else /* MARK_BIT_PER_GRANULE */ - - for (i = 0; i < MARK_BITS_SZ; i++) { - result += count_ones(hhdr->hb_marks[i]); - } -#endif - GC_ASSERT(result > 0); - result--; /* exclude the one bit set past the end */ -#ifndef MARK_BIT_PER_OBJ - if (IS_UNCOLLECTABLE(hhdr->hb_obj_kind)) { - unsigned ngranules = (unsigned)BYTES_TO_GRANULES(sz); - - /* As mentioned in GC_set_hdr_marks(), all the bits are set */ - /* instead of every n-th, thus the result should be adjusted. */ - GC_ASSERT(ngranules > 0 && result % ngranules == 0); - result /= ngranules; - } -#endif - return result; -} - -#endif /* !USE_MARK_BYTES */ - -GC_API unsigned GC_CALL GC_count_set_marks_in_hblk(const void *p) { - return GC_n_set_marks(HDR(p)); -} - -STATIC void GC_CALLBACK GC_print_block_descr(struct hblk *h, - GC_word /* struct PrintStats */ raw_ps) { - hdr *hhdr = HDR(h); - word sz = hhdr->hb_sz; - struct Print_stats *ps = (struct Print_stats *)raw_ps; - unsigned n_marks = GC_n_set_marks(hhdr); - unsigned n_objs = (unsigned)HBLK_OBJS(sz); - -#ifndef PARALLEL_MARK - GC_ASSERT(hhdr->hb_n_marks == n_marks); -#endif - GC_ASSERT((n_objs > 0 ? n_objs : 1) >= n_marks); - GC_printf("%u,%u,%u,%u\n", - hhdr->hb_obj_kind, (unsigned)sz, n_marks, n_objs); - ps->number_of_blocks++; - ps->total_bytes += (sz + (HBLKSIZE - 1)) & ~(HBLKSIZE - 1); /* round up */ -} - -void GC_print_block_list(void) { - struct Print_stats pstats; - - GC_printf( - "kind(0=ptrfree/1=normal/2=unc.)," - "obj_sz,#marks_set,#objs_in_block\n"); - BZERO(&pstats, sizeof(pstats)); - GC_apply_to_all_blocks(GC_print_block_descr, (word)&pstats); - GC_printf("blocks= %lu, total_bytes= %lu\n", - (unsigned long)pstats.number_of_blocks, - (unsigned long)pstats.total_bytes); -} - -/* Currently for debugger use only: */ -GC_API void GC_CALL GC_print_free_list(int kind, size_t sz_in_granules) { - void *flh_next; - int n; - - GC_ASSERT(kind < MAXOBJKINDS); - GC_ASSERT(sz_in_granules <= MAXOBJGRANULES); - flh_next = GC_obj_kinds[kind].ok_freelist[sz_in_granules]; - for (n = 0; flh_next; n++) { - GC_printf("Free object in heap block %p [%d]: %p\n", - (void *)HBLKPTR(flh_next), n, flh_next); - flh_next = obj_link(flh_next); - } -} - -#endif /* !NO_DEBUGGING */ - -/* - * Clear all obj_link pointers in the list of free objects *flp. - * Clear *flp. - * This must be done before dropping a list of free gcj-style objects, - * since may otherwise end up with dangling "descriptor" pointers. - * It may help for other pointer-containing objects. - */ -STATIC void GC_clear_fl_links(void **flp) { - void *next = *flp; - - while (0 != next) { - *flp = 0; - flp = &(obj_link(next)); - next = *flp; - } -} - -/* - * Perform GC_reclaim_block on the entire heap, after first clearing - * small object free lists (if we are not just looking for leaks). - */ -GC_INNER void GC_start_reclaim(GC_bool report_if_found) { - unsigned kind; - -#if defined(PARALLEL_MARK) - GC_ASSERT(0 == GC_fl_builder_count); -#endif - /* Reset in use counters. GC_reclaim_block recomputes them. */ - GC_composite_in_use = 0; - GC_atomic_in_use = 0; - /* Clear reclaim- and free-lists */ - for (kind = 0; kind < GC_n_kinds; kind++) { - struct hblk **rlist = GC_obj_kinds[kind].ok_reclaim_list; - GC_bool should_clobber = (GC_obj_kinds[kind].ok_descriptor != 0); - - if (rlist == 0) continue; /* This kind not used. */ - if (!report_if_found) { - void **fop; - void **lim = &(GC_obj_kinds[kind].ok_freelist[MAXOBJGRANULES + 1]); - - for (fop = GC_obj_kinds[kind].ok_freelist; - (word)fop < (word)lim; (*(word **)&fop)++) { - if (*fop != 0) { - if (should_clobber) { - GC_clear_fl_links(fop); - } else { - *fop = 0; - } - } - } - } /* otherwise free list objects are marked, */ - /* and it's safe to leave them. */ - BZERO(rlist, (MAXOBJGRANULES + 1) * sizeof(void *)); - } - - /* Go through all heap blocks (in hblklist) and reclaim unmarked objects */ - /* or enqueue the block for later processing. */ - GC_apply_to_all_blocks(GC_reclaim_block, (word)report_if_found); - -#ifdef EAGER_SWEEP - /* This is a very stupid thing to do. We make it possible anyway, */ - /* so that you can convince yourself that it really is very stupid. */ - GC_reclaim_all((GC_stop_func)0, FALSE); -#elif defined(ENABLE_DISCLAIM) - /* However, make sure to clear reclaimable objects of kinds with */ - /* unconditional marking enabled before we do any significant */ - /* marking work. */ - GC_reclaim_unconditionally_marked(); -#endif -#if defined(PARALLEL_MARK) - GC_ASSERT(0 == GC_fl_builder_count); -#endif -} - -/* - * Sweep blocks of the indicated object size and kind until either the - * appropriate free list is nonempty, or there are no more blocks to - * sweep. - */ -GC_INNER void GC_continue_reclaim(word sz /* granules */, int kind) { - hdr *hhdr; - struct hblk *hbp; - struct obj_kind *ok = &(GC_obj_kinds[kind]); - struct hblk **rlh = ok->ok_reclaim_list; - void **flh = &(ok->ok_freelist[sz]); - - if (NULL == rlh) - return; /* No blocks of this kind. */ - - for (rlh += sz; (hbp = *rlh) != NULL;) { - hhdr = HDR(hbp); - *rlh = hhdr->hb_next; - GC_reclaim_small_nonempty_block(hbp, hhdr->hb_sz, FALSE); - if (*flh != 0) - break; - } -} - -/* - * Reclaim all small blocks waiting to be reclaimed. - * Abort and return FALSE when/if (*stop_func)() returns TRUE. - * If this returns TRUE, then it's safe to restart the world - * with incorrectly cleared mark bits. - * If ignore_old is TRUE, then reclaim only blocks that have been - * recently reclaimed, and discard the rest. - * Stop_func may be 0. - */ -GC_INNER GC_bool GC_reclaim_all(GC_stop_func stop_func, GC_bool ignore_old) { - word sz; - unsigned kind; - hdr *hhdr; - struct hblk *hbp; - struct obj_kind *ok; - struct hblk **rlp; - struct hblk **rlh; -#ifndef NO_CLOCK - CLOCK_TYPE start_time = CLOCK_TYPE_INITIALIZER; - - if (GC_print_stats == VERBOSE) - GET_TIME(start_time); -#endif - - for (kind = 0; kind < GC_n_kinds; kind++) { - ok = &(GC_obj_kinds[kind]); - rlp = ok->ok_reclaim_list; - if (rlp == 0) continue; - for (sz = 1; sz <= MAXOBJGRANULES; sz++) { - for (rlh = rlp + sz; (hbp = *rlh) != NULL;) { - if (stop_func != (GC_stop_func)0 && (*stop_func)()) { - return FALSE; - } - hhdr = HDR(hbp); - *rlh = hhdr->hb_next; - if (!ignore_old || (word)hhdr->hb_last_reclaimed == GC_gc_no - 1) { - /* It's likely we'll need it this time, too */ - /* It's been touched recently, so this */ - /* shouldn't trigger paging. */ - GC_reclaim_small_nonempty_block(hbp, hhdr->hb_sz, FALSE); - } - } - } - } -#ifndef NO_CLOCK - if (GC_print_stats == VERBOSE) { - CLOCK_TYPE done_time; - - GET_TIME(done_time); - GC_verbose_log_printf( - "Disposing of reclaim lists took %lu ms %lu ns\n", - MS_TIME_DIFF(done_time, start_time), - NS_FRAC_TIME_DIFF(done_time, start_time)); - } -#endif - return TRUE; -} - -#if !defined(EAGER_SWEEP) && defined(ENABLE_DISCLAIM) -/* We do an eager sweep on heap blocks where unconditional marking has */ -/* been enabled, so that any reclaimable objects have been reclaimed */ -/* before we start marking. This is a simplified GC_reclaim_all */ -/* restricted to kinds where ok_mark_unconditionally is true. */ -STATIC void GC_reclaim_unconditionally_marked(void) { - word sz; - unsigned kind; - hdr *hhdr; - struct hblk *hbp; - struct obj_kind *ok; - struct hblk **rlp; - struct hblk **rlh; - - for (kind = 0; kind < GC_n_kinds; kind++) { - ok = &(GC_obj_kinds[kind]); - if (!ok->ok_mark_unconditionally) - continue; - rlp = ok->ok_reclaim_list; - if (rlp == 0) - continue; - for (sz = 1; sz <= MAXOBJGRANULES; sz++) { - rlh = rlp + sz; - while ((hbp = *rlh) != 0) { - hhdr = HDR(hbp); - *rlh = hhdr->hb_next; - GC_reclaim_small_nonempty_block(hbp, hhdr->hb_sz, FALSE); - } - } - } -} -#endif /* !EAGER_SWEEP && ENABLE_DISCLAIM */ - -struct enumerate_reachable_s { - GC_reachable_object_proc proc; - void *client_data; -}; - -STATIC void GC_CALLBACK GC_do_enumerate_reachable_objects(struct hblk *hbp, - GC_word ped) { - struct hblkhdr *hhdr = HDR(hbp); - size_t sz = (size_t)hhdr->hb_sz; - size_t bit_no; - char *p, *plim; - - if (GC_block_empty(hhdr)) { - return; - } - - p = hbp->hb_body; - if (sz > MAXOBJBYTES) { /* one big object */ - plim = p; - } else { - plim = hbp->hb_body + HBLKSIZE - sz; - } - /* Go through all words in block. */ - for (bit_no = 0; p <= plim; bit_no += MARK_BIT_OFFSET(sz), p += sz) { - if (mark_bit_from_hdr(hhdr, bit_no)) { - ((struct enumerate_reachable_s *)ped)->proc(p, sz, ((struct enumerate_reachable_s *)ped)->client_data); - } - } -} - -GC_API void GC_CALL GC_enumerate_reachable_objects_inner( - GC_reachable_object_proc proc, - void *client_data) { - struct enumerate_reachable_s ed; - - GC_ASSERT(I_HOLD_LOCK()); - ed.proc = proc; - ed.client_data = client_data; - GC_apply_to_all_blocks(GC_do_enumerate_reachable_objects, (word)&ed); -} diff --git a/vendor/c11threads/threads.h b/vendor/c11threads/threads.h deleted file mode 100644 index ead6e87b..00000000 --- a/vendor/c11threads/threads.h +++ /dev/null @@ -1,151 +0,0 @@ -/* - * C11 emulation library - * - * (C) Copyright yohhoy 2012. - * Distributed under the Boost Software License, Version 1.0. - * (See copy at http://www.boost.org/LICENSE_1_0.txt) - */ -#ifndef EMULATED_THREADS_H_INCLUDED_ -#define EMULATED_THREADS_H_INCLUDED_ - -#include - -#if defined(_WIN32) -#define WIN32_LEAN_AND_MEAN -#include - -// check configuration -#if defined(EMULATED_THREADS_USE_NATIVE_CALL_ONCE) && (_WIN32_WINNT < 0x0600) -#error EMULATED_THREADS_USE_NATIVE_CALL_ONCE requires _WIN32_WINNT>=0x0600 -#endif - -#if defined(EMULATED_THREADS_USE_NATIVE_CV) && (_WIN32_WINNT < 0x0600) -#error EMULATED_THREADS_USE_NATIVE_CV requires _WIN32_WINNT>=0x0600 -#endif - -/*---------------------------- macros ----------------------------*/ -#ifdef EMULATED_THREADS_USE_NATIVE_CALL_ONCE -#define ONCE_FLAG_INIT INIT_ONCE_STATIC_INIT -#else -#define ONCE_FLAG_INIT \ -{ 0 } -#endif -#define TSS_DTOR_ITERATIONS 1 - -#ifndef thread_local -#define thread_local _Thread_local -#endif - -/*---------------------------- types ----------------------------*/ -typedef struct cnd_t { - #ifdef EMULATED_THREADS_USE_NATIVE_CV - CONDITION_VARIABLE condvar; - #else - int blocked; - int gone; - int to_unblock; - HANDLE sem_queue; - HANDLE sem_gate; - CRITICAL_SECTION monitor; - #endif -} cnd_t; - -typedef HANDLE thrd_t; - -typedef DWORD tss_t; - -typedef struct mtx_t { - CRITICAL_SECTION cs; -} mtx_t; - -#ifdef EMULATED_THREADS_USE_NATIVE_CALL_ONCE -typedef INIT_ONCE once_flag; -#else -typedef struct once_flag_t { - volatile LONG status; -} once_flag; -#endif - -#elif defined(__unix__) || defined(__unix) || defined(__APPLE__) -#include - -/*---------------------------- macros ----------------------------*/ -#define ONCE_FLAG_INIT PTHREAD_ONCE_INIT -#ifdef INIT_ONCE_STATIC_INIT -#define TSS_DTOR_ITERATIONS PTHREAD_DESTRUCTOR_ITERATIONS -#else -#define TSS_DTOR_ITERATIONS 1 // assume TSS dtor MAY be called at least once. -#endif - -/*---------------------------- types ----------------------------*/ -typedef pthread_cond_t cnd_t; -typedef pthread_t thrd_t; -typedef pthread_key_t tss_t; -typedef pthread_mutex_t mtx_t; -typedef pthread_once_t once_flag; - -#else -#error Not supported on this platform. -#endif - -/*---------------------------- types ----------------------------*/ -typedef void (*tss_dtor_t)(void*); -typedef int (*thrd_start_t)(void*); - -struct xtime { - time_t sec; - long nsec; -}; -typedef struct xtime xtime; - -/*-------------------- enumeration constants --------------------*/ -enum { - mtx_plain = 0, - mtx_try = 1, - mtx_timed = 2, - mtx_recursive = 4 -}; - -enum { - thrd_success = 0, // succeeded - thrd_timeout, // timeout - thrd_error, // failed - thrd_busy, // resource busy - thrd_nomem // out of memory -}; - -/*-------------------------- functions --------------------------*/ -void call_once(once_flag* flag, void (*func)(void)); - -int cnd_broadcast(cnd_t* cond); -void cnd_destroy(cnd_t* cond); -int cnd_init(cnd_t* cond); -int cnd_signal(cnd_t* cond); -int cnd_timedwait(cnd_t* cond, mtx_t* mtx, const xtime* xt); -int cnd_wait(cnd_t* cond, mtx_t* mtx); - -void mtx_destroy(mtx_t* mtx); -int mtx_init(mtx_t* mtx, int type); -int mtx_lock(mtx_t* mtx); -int mtx_timedlock(mtx_t* mtx, const xtime* xt); -int mtx_trylock(mtx_t* mtx); -int mtx_unlock(mtx_t* mtx); - -int thrd_create(thrd_t* thr, thrd_start_t func, void* arg); -thrd_t thrd_current(void); -int thrd_detach(thrd_t thr); -int thrd_equal(thrd_t thr0, thrd_t thr1); -void thrd_exit(int res); -int thrd_join(thrd_t thr, int* res); -void thrd_sleep(const xtime* xt); -void thrd_yield(void); - -int tss_create(tss_t* key, tss_dtor_t dtor); -void tss_delete(tss_t key); -void* tss_get(tss_t key); -int tss_set(tss_t key, void* val); - -int xtime_get(xtime* xt, int base); -#define TIME_UTC 1 - -#endif /* EMULATED_THREADS_H_INCLUDED_ */ diff --git a/vendor/c11threads/threads_msvc.c b/vendor/c11threads/threads_msvc.c deleted file mode 100644 index 6fc621ad..00000000 --- a/vendor/c11threads/threads_msvc.c +++ /dev/null @@ -1,460 +0,0 @@ -/* - * C11 emulation library - * - * (C) Copyright yohhoy 2012. - * Distributed under the Boost Software License, Version 1.0. - * (See copy at http://www.boost.org/LICENSE_1_0.txt) - */ -#include -#include -#include -#include // MSVCRT -#include - -/* -Configuration macro: - - EMULATED_THREADS_USE_NATIVE_CALL_ONCE - Use native WindowsAPI one-time initialization function. - (requires WinVista or later) - Otherwise emulate by mtx_trylock() + *busy loop* for WinXP. - - EMULATED_THREADS_USE_NATIVE_CV - Use native WindowsAPI condition variable object. - (requires WinVista or later) - Otherwise use emulated implementation for WinXP. - - EMULATED_THREADS_TSS_DTOR_SLOTNUM - Max registerable TSS dtor number. -*/ -#if _WIN32_WINNT >= 0x0600 -// Prefer native WindowsAPI on newer environment. -#define EMULATED_THREADS_USE_NATIVE_CALL_ONCE -#define EMULATED_THREADS_USE_NATIVE_CV -#endif -#define EMULATED_THREADS_TSS_DTOR_SLOTNUM 64 // see TLS_MINIMUM_AVAILABLE - -#include "threads.h" - -/* -Implementation limits: - - Conditionally emulation for "Initialization functions" - (see EMULATED_THREADS_USE_NATIVE_CALL_ONCE macro) - - Emulated `mtx_timelock()' with mtx_trylock() + *busy loop* -*/ -static void impl_tss_dtor_invoke(); // forward decl. - -struct impl_thrd_param { - thrd_start_t func; - void* arg; -}; - -static unsigned __stdcall impl_thrd_routine(void* p) { - struct impl_thrd_param pack; - int code; - memcpy(&pack, p, sizeof(struct impl_thrd_param)); - free(p); - code = pack.func(pack.arg); - impl_tss_dtor_invoke(); - return (unsigned)code; -} - -static DWORD impl_xtime2msec(const xtime* xt) { - return (DWORD)((xt->sec * 1000u) + (xt->nsec / 1000000)); -} - -#ifdef EMULATED_THREADS_USE_NATIVE_CALL_ONCE -struct impl_call_once_param { - void (*func)(void); -}; -static BOOL CALLBACK impl_call_once_callback(PINIT_ONCE InitOnce, PVOID Parameter, PVOID* Context) { - struct impl_call_once_param* param = (struct impl_call_once_param*)Parameter; - (param->func)(); - ((void)InitOnce); - ((void)Context); // suppress warning - return TRUE; -} -#endif // ifdef EMULATED_THREADS_USE_NATIVE_CALL_ONCE - -#ifndef EMULATED_THREADS_USE_NATIVE_CV -/* -Note: - The implementation of condition variable is ported from Boost.Interprocess - See http://www.boost.org/boost/interprocess/sync/windows/condition.hpp -*/ -static void impl_cond_do_signal(cnd_t* cond, int broadcast) { - int nsignal = 0; - - EnterCriticalSection(&cond->monitor); - if (cond->to_unblock != 0) { - if (cond->blocked == 0) { - LeaveCriticalSection(&cond->monitor); - return; - } - if (broadcast) { - cond->to_unblock += nsignal = cond->blocked; - cond->blocked = 0; - } else { - nsignal = 1; - cond->to_unblock++; - cond->blocked--; - } - } else if (cond->blocked > cond->gone) { - WaitForSingleObject(cond->sem_gate, INFINITE); - if (cond->gone != 0) { - cond->blocked -= cond->gone; - cond->gone = 0; - } - if (broadcast) { - nsignal = cond->to_unblock = cond->blocked; - cond->blocked = 0; - } else { - nsignal = cond->to_unblock = 1; - cond->blocked--; - } - } - LeaveCriticalSection(&cond->monitor); - - if (0 < nsignal) - ReleaseSemaphore(cond->sem_queue, nsignal, NULL); -} - -static int impl_cond_do_wait(cnd_t* cond, mtx_t* mtx, const xtime* xt) { - int nleft = 0; - int ngone = 0; - int timeout = 0; - DWORD w; - - WaitForSingleObject(cond->sem_gate, INFINITE); - cond->blocked++; - ReleaseSemaphore(cond->sem_gate, 1, NULL); - - mtx_unlock(mtx); - - w = WaitForSingleObject(cond->sem_queue, xt ? impl_xtime2msec(xt) : INFINITE); - timeout = (w == WAIT_TIMEOUT); - - EnterCriticalSection(&cond->monitor); - if ((nleft = cond->to_unblock) != 0) { - if (timeout) { - if (cond->blocked != 0) { - cond->blocked--; - } else { - cond->gone++; - } - } - if (--cond->to_unblock == 0) { - if (cond->blocked != 0) { - ReleaseSemaphore(cond->sem_gate, 1, NULL); - nleft = 0; - } else if ((ngone = cond->gone) != 0) { - cond->gone = 0; - } - } - } else if (++cond->gone == INT_MAX / 2) { - WaitForSingleObject(cond->sem_gate, INFINITE); - cond->blocked -= cond->gone; - ReleaseSemaphore(cond->sem_gate, 1, NULL); - cond->gone = 0; - } - LeaveCriticalSection(&cond->monitor); - - if (nleft == 1) { - while (ngone--) - WaitForSingleObject(cond->sem_queue, INFINITE); - ReleaseSemaphore(cond->sem_gate, 1, NULL); - } - - mtx_lock(mtx); - return timeout ? thrd_busy : thrd_success; -} -#endif // ifndef EMULATED_THREADS_USE_NATIVE_CV - -static struct impl_tss_dtor_entry { - tss_t key; - tss_dtor_t dtor; -} impl_tss_dtor_tbl[EMULATED_THREADS_TSS_DTOR_SLOTNUM]; - -static int impl_tss_dtor_register(tss_t key, tss_dtor_t dtor) { - int i; - for (i = 0; i < EMULATED_THREADS_TSS_DTOR_SLOTNUM; i++) { - if (!impl_tss_dtor_tbl[i].dtor) - break; - } - if (i == EMULATED_THREADS_TSS_DTOR_SLOTNUM) - return 1; - impl_tss_dtor_tbl[i].key = key; - impl_tss_dtor_tbl[i].dtor = dtor; - return 0; -} - -static void impl_tss_dtor_invoke() { - int i; - for (i = 0; i < EMULATED_THREADS_TSS_DTOR_SLOTNUM; i++) { - if (impl_tss_dtor_tbl[i].dtor) { - void* val = tss_get(impl_tss_dtor_tbl[i].key); - if (val) - (impl_tss_dtor_tbl[i].dtor)(val); - } - } -} - -/*--------------- 7.25.2 Initialization functions ---------------*/ -// 7.25.2.1 -void call_once(once_flag* flag, void (*func)(void)) { - assert(flag && func); -#ifdef EMULATED_THREADS_USE_NATIVE_CALL_ONCE - { - struct impl_call_once_param param; - param.func = func; - InitOnceExecuteOnce(flag, impl_call_once_callback, (PVOID)¶m, NULL); - } -#else - if (InterlockedCompareExchange(&flag->status, 1, 0) == 0) { - (func)(); - InterlockedExchange(&flag->status, 2); - } else { - while (flag->status == 1) { - // busy loop! - thrd_yield(); - } - } -#endif -} - -/*------------- 7.25.3 Condition variable functions -------------*/ -// 7.25.3.1 -int cnd_broadcast(cnd_t* cond) { - if (!cond) return thrd_error; -#ifdef EMULATED_THREADS_USE_NATIVE_CV - WakeAllConditionVariable(&cond->condvar); -#else - impl_cond_do_signal(cond, 1); -#endif - return thrd_success; -} - -// 7.25.3.2 -void cnd_destroy(cnd_t* cond) { - assert(cond); -#ifdef EMULATED_THREADS_USE_NATIVE_CV - // do nothing -#else - CloseHandle(cond->sem_queue); - CloseHandle(cond->sem_gate); - DeleteCriticalSection(&cond->monitor); -#endif -} - -// 7.25.3.3 -int cnd_init(cnd_t* cond) { - if (!cond) return thrd_error; -#ifdef EMULATED_THREADS_USE_NATIVE_CV - InitializeConditionVariable(&cond->condvar); -#else - cond->blocked = 0; - cond->gone = 0; - cond->to_unblock = 0; - cond->sem_queue = CreateSemaphore(NULL, 0, LONG_MAX, NULL); - cond->sem_gate = CreateSemaphore(NULL, 1, 1, NULL); - InitializeCriticalSection(&cond->monitor); -#endif - return thrd_success; -} - -// 7.25.3.4 -int cnd_signal(cnd_t* cond) { - if (!cond) return thrd_error; -#ifdef EMULATED_THREADS_USE_NATIVE_CV - WakeConditionVariable(&cond->condvar); -#else - impl_cond_do_signal(cond, 0); -#endif - return thrd_success; -} - -// 7.25.3.5 -int cnd_timedwait(cnd_t* cond, mtx_t* mtx, const xtime* xt) { - if (!cond || !mtx || !xt) return thrd_error; -#ifdef EMULATED_THREADS_USE_NATIVE_CV - if (SleepConditionVariableCS(&cond->condvar, &mtx->cs, impl_xtime2msec(xt))) - return thrd_success; - return (GetLastError() == ERROR_TIMEOUT) ? thrd_busy : thrd_error; -#else - return impl_cond_do_wait(cond, mtx, xt); -#endif -} - -// 7.25.3.6 -int cnd_wait(cnd_t* cond, mtx_t* mtx) { - if (!cond || !mtx) return thrd_error; -#ifdef EMULATED_THREADS_USE_NATIVE_CV - SleepConditionVariableCS(&cond->condvar, &mtx->cs, INFINITE); -#else - impl_cond_do_wait(cond, mtx, NULL); -#endif - return thrd_success; -} - -/*-------------------- 7.25.4 Mutex functions --------------------*/ -// 7.25.4.1 -void mtx_destroy(mtx_t* mtx) { - assert(mtx); - DeleteCriticalSection(&mtx->cs); -} - -// 7.25.4.2 -int mtx_init(mtx_t* mtx, int type) { - if (!mtx) return thrd_error; - if (type != mtx_plain && type != mtx_timed && type != mtx_try && type != (mtx_plain | mtx_recursive) && type != (mtx_timed | mtx_recursive) && type != (mtx_try | mtx_recursive)) - return thrd_error; - InitializeCriticalSection(&mtx->cs); - return thrd_success; -} - -// 7.25.4.3 -int mtx_lock(mtx_t* mtx) { - if (!mtx) return thrd_error; - EnterCriticalSection(&mtx->cs); - return thrd_success; -} - -// 7.25.4.4 -int mtx_timedlock(mtx_t* mtx, const xtime* xt) { - time_t expire, now; - if (!mtx || !xt) return thrd_error; - expire = time(NULL); - expire += xt->sec; - while (mtx_trylock(mtx) != thrd_success) { - now = time(NULL); - if (expire < now) - return thrd_busy; - // busy loop! - thrd_yield(); - } - return thrd_success; -} - -// 7.25.4.5 -int mtx_trylock(mtx_t* mtx) { - if (!mtx) return thrd_error; - return TryEnterCriticalSection(&mtx->cs) ? thrd_success : thrd_busy; -} - -// 7.25.4.6 -int mtx_unlock(mtx_t* mtx) { - if (!mtx) return thrd_error; - LeaveCriticalSection(&mtx->cs); - return thrd_success; -} - -/*------------------- 7.25.5 Thread functions -------------------*/ -// 7.25.5.1 -int thrd_create(thrd_t* thr, thrd_start_t func, void* arg) { - struct impl_thrd_param* pack; - uintptr_t handle; - if (!thr) return thrd_error; - pack = malloc(sizeof(struct impl_thrd_param)); - if (!pack) return thrd_nomem; - pack->func = func; - pack->arg = arg; - handle = _beginthreadex(NULL, 0, impl_thrd_routine, pack, 0, NULL); - if (handle == 0) { - if (errno == EAGAIN || errno == EACCES) - return thrd_nomem; - return thrd_error; - } - *thr = (thrd_t)handle; - return thrd_success; -} - -// 7.25.5.2 -thrd_t thrd_current(void) { - return GetCurrentThread(); -} - -// 7.25.5.3 -int thrd_detach(thrd_t thr) { - CloseHandle(thr); - return thrd_success; -} - -// 7.25.5.4 -int thrd_equal(thrd_t thr0, thrd_t thr1) { - return (thr0 == thr1); -} - -// 7.25.5.5 -void thrd_exit(int res) { - impl_tss_dtor_invoke(); - _endthreadex((unsigned)res); -} - -// 7.25.5.6 -int thrd_join(thrd_t thr, int* res) { - DWORD w, code; - w = WaitForSingleObject(thr, INFINITE); - if (w != WAIT_OBJECT_0) - return thrd_error; - if (res) { - if (!GetExitCodeThread(thr, &code)) { - CloseHandle(thr); - return thrd_error; - } - *res = (int)code; - } - CloseHandle(thr); - return thrd_success; -} - -// 7.25.5.7 -void thrd_sleep(const xtime* xt) { - assert(xt); - Sleep(impl_xtime2msec(xt)); -} - -// 7.25.5.8 -void thrd_yield(void) { - SwitchToThread(); -} - -/*----------- 7.25.6 Thread-specific storage functions -----------*/ -// 7.25.6.1 -int tss_create(tss_t* key, tss_dtor_t dtor) { - if (!key) return thrd_error; - *key = TlsAlloc(); - if (dtor) { - if (impl_tss_dtor_register(*key, dtor)) { - TlsFree(*key); - return thrd_error; - } - } - return (*key != 0xFFFFFFFF) ? thrd_success : thrd_error; -} - -// 7.25.6.2 -void tss_delete(tss_t key) { - TlsFree(key); -} - -// 7.25.6.3 -void* tss_get(tss_t key) { - return TlsGetValue(key); -} - -// 7.25.6.4 -int tss_set(tss_t key, void* val) { - return TlsSetValue(key, val) ? thrd_success : thrd_error; -} - -/*-------------------- 7.25.7 Time functions --------------------*/ -// 7.25.6.1 -int xtime_get(xtime* xt, int base) { - if (!xt) return 0; - if (base == TIME_UTC) { - xt->sec = time(NULL); - xt->nsec = 0; - return base; - } - return 0; -} diff --git a/vendor/c11threads/threads_posix.c b/vendor/c11threads/threads_posix.c deleted file mode 100644 index c206c5cd..00000000 --- a/vendor/c11threads/threads_posix.c +++ /dev/null @@ -1,271 +0,0 @@ -/* - * C11 emulation library - * - * (C) Copyright yohhoy 2012. - * Distributed under the Boost Software License, Version 1.0. - * (See copy at http://www.boost.org/LICENSE_1_0.txt) - */ -#include -#include -#include -#include -#include -#include -#include - -#include -#include - -/* -Configuration macro: - - EMULATED_THREADS_USE_NATIVE_TIMEDLOCK - Use pthread_mutex_timedlock() for `mtx_timedlock()' - Otherwise use mtx_trylock() + *busy loop* emulation. -*/ -#if !defined(__CYGWIN__) && !defined(__APPLE__) -#define EMULATED_THREADS_USE_NATIVE_TIMEDLOCK -#endif - - -#include "threads.h" - -/* -Implementation limits: - - Conditionally emulation for "mutex with timeout" - (see EMULATED_THREADS_USE_NATIVE_TIMEDLOCK macro) -*/ -struct impl_thrd_param { - thrd_start_t func; - void* arg; -}; - -void* impl_thrd_routine(void* p) { - struct impl_thrd_param pack = *((struct impl_thrd_param*)p); - free(p); - return (void*)((size_t)pack.func(pack.arg)); -} - -/*--------------- 7.25.2 Initialization functions ---------------*/ -// 7.25.2.1 -void call_once(once_flag* flag, void (*func)(void)) { - pthread_once(flag, func); -} - -/*------------- 7.25.3 Condition variable functions -------------*/ -// 7.25.3.1 -int cnd_broadcast(cnd_t* cond) { - if (!cond) return thrd_error; - pthread_cond_broadcast(cond); - return thrd_success; -} - -// 7.25.3.2 -void cnd_destroy(cnd_t* cond) { - assert(cond); - pthread_cond_destroy(cond); -} - -// 7.25.3.3 -int cnd_init(cnd_t* cond) { - if (!cond) return thrd_error; - pthread_cond_init(cond, NULL); - return thrd_success; -} - -// 7.25.3.4 -int cnd_signal(cnd_t* cond) { - if (!cond) return thrd_error; - pthread_cond_signal(cond); - return thrd_success; -} - -// 7.25.3.5 -int cnd_timedwait(cnd_t* cond, mtx_t* mtx, const xtime* xt) { - struct timespec abs_time; - int rt; - if (!cond || !mtx || !xt) return thrd_error; - rt = pthread_cond_timedwait(cond, mtx, &abs_time); - if (rt == ETIMEDOUT) - return thrd_busy; - return (rt == 0) ? thrd_success : thrd_error; -} - -// 7.25.3.6 -int cnd_wait(cnd_t* cond, mtx_t* mtx) { - if (!cond || !mtx) return thrd_error; - pthread_cond_wait(cond, mtx); - return thrd_success; -} - -/*-------------------- 7.25.4 Mutex functions --------------------*/ -// 7.25.4.1 -void mtx_destroy(mtx_t* mtx) { - assert(mtx); - pthread_mutex_destroy(mtx); -} - -// 7.25.4.2 -int mtx_init(mtx_t* mtx, int type) { - pthread_mutexattr_t attr; - if (!mtx) return thrd_error; - if (type != mtx_plain && type != mtx_timed && type != mtx_try && type != (mtx_plain | mtx_recursive) && type != (mtx_timed | mtx_recursive) && type != (mtx_try | mtx_recursive)) - return thrd_error; - pthread_mutexattr_init(&attr); - if ((type & mtx_recursive) != 0) { -#if defined(__linux__) || defined(__linux) - pthread_mutexattr_settype(&attr, PTHREAD_MUTEX_RECURSIVE_NP); -#else - pthread_mutexattr_settype(&attr, PTHREAD_MUTEX_RECURSIVE); -#endif - } - pthread_mutex_init(mtx, &attr); - pthread_mutexattr_destroy(&attr); - return thrd_success; -} - -// 7.25.4.3 -int mtx_lock(mtx_t* mtx) { - if (!mtx) return thrd_error; - pthread_mutex_lock(mtx); - return thrd_success; -} - -// 7.25.4.4 -int mtx_timedlock(mtx_t* mtx, const xtime* xt) { - if (!mtx || !xt) return thrd_error; - { -#ifdef EMULATED_THREADS_USE_NATIVE_TIMEDLOCK - struct timespec ts; - int rt; - ts.tv_sec = xt->sec; - ts.tv_nsec = xt->nsec; - rt = pthread_mutex_timedlock(mtx, &ts); - if (rt == 0) - return thrd_success; - return (rt == ETIMEDOUT) ? thrd_busy : thrd_error; -#else - time_t expire = time(NULL); - expire += xt->sec; - while (mtx_trylock(mtx) != thrd_success) { - time_t now = time(NULL); - if (expire < now) - return thrd_busy; - // busy loop! - thrd_yield(); - } - return thrd_success; -#endif - } -} - -// 7.25.4.5 -int mtx_trylock(mtx_t* mtx) { - if (!mtx) return thrd_error; - return (pthread_mutex_trylock(mtx) == 0) ? thrd_success : thrd_busy; -} - -// 7.25.4.6 -int mtx_unlock(mtx_t* mtx) { - if (!mtx) return thrd_error; - pthread_mutex_unlock(mtx); - return thrd_success; -} - -/*------------------- 7.25.5 Thread functions -------------------*/ -// 7.25.5.1 -int thrd_create(thrd_t* thr, thrd_start_t func, void* arg) { - struct impl_thrd_param* pack; - if (!thr) return thrd_error; - pack = malloc(sizeof(struct impl_thrd_param)); - if (!pack) return thrd_nomem; - pack->func = func; - pack->arg = arg; - - pthread_attr_t attr; - pthread_attr_init(&attr); - - if (pthread_create(thr, &attr, impl_thrd_routine, pack) != 0) { - free(pack); - return thrd_error; - } - return thrd_success; -} - -// 7.25.5.2 -thrd_t thrd_current(void) { - return pthread_self(); -} - -// 7.25.5.3 -int thrd_detach(thrd_t thr) { - return (pthread_detach(thr) == 0) ? thrd_success : thrd_error; -} - -// 7.25.5.4 -int thrd_equal(thrd_t thr0, thrd_t thr1) { - return pthread_equal(thr0, thr1); -} - -// 7.25.5.5 -void thrd_exit(int res) { - pthread_exit((void*)((size_t)res)); -} - -// 7.25.5.6 -int thrd_join(thrd_t thr, int* res) { - void* code; - if (pthread_join(thr, &code) != 0) - return thrd_error; - if (res) - *res = (int)((size_t)code); - return thrd_success; -} - -// 7.25.5.7 -void thrd_sleep(const xtime* xt) { - struct timespec req; - assert(xt); - req.tv_sec = xt->sec; - req.tv_nsec = xt->nsec; - nanosleep(&req, NULL); -} - -// 7.25.5.8 -void thrd_yield(void) { - sched_yield(); -} - -/*----------- 7.25.6 Thread-specific storage functions -----------*/ -// 7.25.6.1 -int tss_create(tss_t* key, tss_dtor_t dtor) { - if (!key) return thrd_error; - return (pthread_key_create(key, dtor) == 0) ? thrd_success : thrd_error; -} - -// 7.25.6.2 -void tss_delete(tss_t key) { - pthread_key_delete(key); -} - -// 7.25.6.3 -void* tss_get(tss_t key) { - return pthread_getspecific(key); -} - -// 7.25.6.4 -int tss_set(tss_t key, void* val) { - return (pthread_setspecific(key, val) == 0) ? thrd_success : thrd_error; -} - -/*-------------------- 7.25.7 Time functions --------------------*/ -// 7.25.6.1 -int xtime_get(xtime* xt, int base) { - if (!xt) return 0; - if (base == TIME_UTC) { - xt->sec = time(NULL); - xt->nsec = 0; - return base; - } - return 0; -} diff --git a/vendor/common/README.txt b/vendor/common/README.txt deleted file mode 100644 index a98b1c21..00000000 --- a/vendor/common/README.txt +++ /dev/null @@ -1,10 +0,0 @@ -//////////////////////////////////////////////////////////////////////////////// -// Cuik repository common utilities -//////////////////////////////////////////////////////////////////////////////// - -This is used by all kinds of components within the project, almost everything -will be including it. It has nice macros and includes. - -TODO: - - * move everything to using the iterator macros \ No newline at end of file diff --git a/vendor/common/arena.h b/vendor/common/arena.h deleted file mode 100644 index b6d5f559..00000000 --- a/vendor/common/arena.h +++ /dev/null @@ -1,77 +0,0 @@ -#pragma once -#include -#include - -#ifndef TB_API -# ifdef __cplusplus -# define TB_EXTERN extern "C" -# else -# define TB_EXTERN -# endif -# ifdef TB_DLL -# ifdef TB_IMPORT_DLL -# define TB_API TB_EXTERN __declspec(dllimport) -# else -# define TB_API TB_EXTERN __declspec(dllexport) -# endif -# else -# define TB_API TB_EXTERN -# endif -#endif - -enum { - TB_ARENA_SMALL_CHUNK_SIZE = 4 * 1024, - TB_ARENA_MEDIUM_CHUNK_SIZE = 512 * 1024, - TB_ARENA_LARGE_CHUNK_SIZE = 64 * 1024 * 1024, - - TB_ARENA_ALIGNMENT = 16, -}; - -typedef struct TB_Arena TB_Arena; -struct TB_Arena { - TB_Arena* next; - - // we only care about this for the root arena chunk but whatever - TB_Arena* top; - - // top of the allocation space - char* watermark; - char* high_point; // &top->data[chunk_size] - - char data[]; -}; - -typedef struct TB_ArenaSavepoint { - TB_Arena* top; - char* watermark; -} TB_ArenaSavepoint; - -#define TB_ARENA_FOR(it, arena) for (TB_Arena* it = (arena); it != NULL; it = it->next) - -#define TB_ARENA_ALLOC(arena, T) tb_arena_alloc(arena, sizeof(T)) -#define TB_ARENA_ARR_ALLOC(arena, count, T) tb_arena_alloc(arena, (count) * sizeof(T)) - -TB_API TB_Arena* tb_arena_create(size_t chunk_size); -TB_API void tb_arena_destroy(TB_Arena* restrict arena); - -TB_API void* tb_arena_unaligned_alloc(TB_Arena* restrict arena, size_t size); -TB_API void* tb_arena_alloc(TB_Arena* restrict arena, size_t size); - -TB_API void* tb_arena_realloc(TB_Arena* restrict arena, void* old, size_t size); - -// return false on failure -TB_API bool tb_arena_free(TB_Arena* restrict arena, void* ptr, size_t size); -TB_API void tb_arena_pop(TB_Arena* restrict arena, void* ptr, size_t size); - -// in case you wanna mix unaligned and aligned arenas -TB_API void tb_arena_realign(TB_Arena* restrict arena); - -TB_API size_t tb_arena_chunk_size(TB_Arena* arena); -TB_API size_t tb_arena_current_size(TB_Arena* arena); - -// savepoints -TB_API TB_ArenaSavepoint tb_arena_save(TB_Arena* arena); -TB_API void tb_arena_restore(TB_Arena* arena, TB_ArenaSavepoint sp); - -// resets to only having one chunk -TB_API void tb_arena_clear(TB_Arena* arena); diff --git a/vendor/common/buffer.h b/vendor/common/buffer.h deleted file mode 100644 index b68e0d24..00000000 --- a/vendor/common/buffer.h +++ /dev/null @@ -1,55 +0,0 @@ -// String Buffer -#ifndef NL_BUFFER_H -#define NL_BUFFER_H - -struct nl_buffer_t; -typedef struct nl_buffer_t nl_buffer_t; - -struct nl_buffer_t { - char *buf; - int len; - int alloc; -}; - -nl_buffer_t *nl_buffer_new(void); -void nl_buffer_alloc(nl_buffer_t *buf); - -void nl_buffer_format(nl_buffer_t *buf, const char *fmt, ...); -char *nl_buffer_get(nl_buffer_t *buf); - -#ifdef NL_BUFFER_IMPL - -nl_buffer_t *nl_buffer_new(void) { - nl_buffer_t *buf = NL_MALLOC(sizeof(nl_buffer_t)); - buf->alloc = 16; - buf->buf = NL_MALLOC(sizeof(char) * buf->alloc); - buf->buf[0] = '\0'; - buf->len = 0; - return buf; -} - -void nl_buffer_format(nl_buffer_t *buf, const char *restrict fmt, ...) { - while (true) { - int avail = buf->alloc - buf->len; - va_list ap; - va_start(ap, fmt); - int written = vsnprintf(&buf->buf[buf->len], avail, fmt, ap); - va_end(ap); - if (avail <= written) { - buf->alloc = buf->alloc * 2 + 16; - buf->buf = NL_REALLOC(buf->buf, sizeof(char) * buf->alloc); - continue; - } - buf->len += written; - break; - } -} - -char *nl_buffer_get(nl_buffer_t *buf) { - return buf->buf; -} - -#endif - -#endif - diff --git a/vendor/common/chunked_array.h b/vendor/common/chunked_array.h deleted file mode 100644 index b787e0ff..00000000 --- a/vendor/common/chunked_array.h +++ /dev/null @@ -1,87 +0,0 @@ -// Arena-backed arrays without fully contiguous memory -#ifndef NL_CHUNKED_ARRAY_H -#define NL_CHUNKED_ARRAY_H - -typedef struct NL_ArrChunk NL_ArrChunk; -struct NL_ArrChunk { - NL_ArrChunk* next; - size_t cap; - size_t count; - void* elems[]; -}; - -typedef struct NL_ChunkedArr { - TB_Arena* arena; - TB_ArenaSavepoint sp; - - NL_ArrChunk* first; - NL_ArrChunk* last; -} NL_ChunkedArr; - -NL_ChunkedArr nl_chunked_arr_alloc(TB_Arena* arena); -void nl_chunked_arr_put(NL_ChunkedArr* arr, void* v); -void nl_chunked_arr_reset(NL_ChunkedArr* arr); - -// we're done with growing it, so we don't wanna take up any more reserved space -void nl_chunked_arr_trim(NL_ChunkedArr* arr); - -#endif // NL_CHUNKED_ARRAY_H - -#ifdef NL_CHUNKED_ARRAY_IMPL -#include - -NL_ChunkedArr nl_chunked_arr_alloc(TB_Arena* arena) { - TB_ArenaSavepoint sp = tb_arena_save(arena); - tb_arena_realign(arena); - - ptrdiff_t leftovers = arena->high_point - (arena->watermark + sizeof(NL_ChunkedArr)); - if (leftovers < 64) { - size_t chunk_size = tb_arena_chunk_size(arena); - leftovers = chunk_size - (sizeof(NL_ArrChunk) + sizeof(TB_Arena)); - } - - ptrdiff_t num_elems = leftovers / sizeof(void*); - NL_ArrChunk* arr = tb_arena_unaligned_alloc(arena, sizeof(NL_ArrChunk) + num_elems*sizeof(void*)); - arr->next = NULL; - arr->cap = num_elems; - arr->count = 0; - - return (NL_ChunkedArr){ arena, sp, arr, arr }; -} - -void nl_chunked_arr_put(NL_ChunkedArr* arr, void* v) { - NL_ArrChunk* last = arr->last; - if (last->cap == last->count) { - // allocate new chunk - size_t chunk_size = tb_arena_chunk_size(arr->arena); - ptrdiff_t leftovers = chunk_size - (sizeof(NL_ArrChunk) + sizeof(TB_Arena)); - ptrdiff_t num_elems = leftovers / sizeof(void*); - NL_ArrChunk* new_chk = tb_arena_alloc(arr->arena, sizeof(NL_ChunkedArr) + num_elems*sizeof(void*)); - - // append - arr->last->next = new_chk; - arr->last = new_chk; - } - last->elems[last->count++] = v; -} - -void* nl_chunked_arr_pop(NL_ChunkedArr* arr) { - NL_ArrChunk* last = arr->last; - return last->elems[--last->count]; -} - -void nl_chunked_arr_reset(NL_ChunkedArr* arr) { - tb_arena_restore(arr->arena, arr->sp); -} - -void nl_chunked_arr_trim(NL_ChunkedArr* arr) { - TB_Arena* arena = arr->arena; - - void* top = &arr->last->elems[arr->last->count]; - tb_arena_pop(arena, top, (arr->last->cap - arr->last->count) * sizeof(void*)); - arr->last->cap = arr->last->count; - - tb_arena_realign(arena); -} - -#endif // NL_CHUNKED_ARRAY_IMPL diff --git a/vendor/common/common.c b/vendor/common/common.c deleted file mode 100644 index 41579f03..00000000 --- a/vendor/common/common.c +++ /dev/null @@ -1,379 +0,0 @@ -// Common is just a bunch of crap i want accessible to all projects in the Cuik repo -#include "arena.h" -#include "futex.h" -#include - -#ifdef _WIN32 -#define WIN32_LEAN_AND_MEAN -#include -#else -#include -#endif - -#define NL_MAP_IMPL -#include - -#define NL_HASH_SET_IMPL -#include - -#define NL_CHUNKED_ARRAY_IMPL -#include - -#define LOG_USE_COLOR -#include "log.c" - -#define NL_BUFFER_IMPL -#include - -#include "perf.h" - -uint64_t cuik__page_size = 0; -uint64_t cuik__page_mask = 0; - -void cuik_init_terminal(void) { - #if _WIN32 - // Raw input mode - SetConsoleMode(GetStdHandle(STD_INPUT_HANDLE), ENABLE_PROCESSED_INPUT); - - // Enable ANSI/VT sequences on windows - HANDLE output_handle = GetStdHandle(STD_OUTPUT_HANDLE); - if (output_handle != INVALID_HANDLE_VALUE) { - DWORD old_mode; - if (GetConsoleMode(output_handle, &old_mode)) { - SetConsoleMode(output_handle, old_mode | ENABLE_VIRTUAL_TERMINAL_PROCESSING); - } - } - #endif -} - -void* cuik__valloc(size_t size) { - #ifdef _WIN32 - if (cuik__page_size == 0) { - SYSTEM_INFO si; - GetSystemInfo(&si); - - cuik__page_size = si.dwPageSize; - cuik__page_mask = si.dwPageSize - 1; - } - - // round size to page size - size = (size + cuik__page_mask) & ~cuik__page_mask; - - return VirtualAlloc(NULL, size, MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE); - #else - cuik__page_size = 4096; - cuik__page_mask = 4095; - - return mmap(NULL, size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); - #endif -} - -void cuik__vfree(void* ptr, size_t size) { - #ifdef _WIN32 - VirtualFree(ptr, 0, MEM_RELEASE); - #else - munmap(ptr, size); - #endif -} - -//////////////////////////////// -// TB_Arenas -//////////////////////////////// -TB_Arena* tb_arena_create(size_t chunk_size) { - if (chunk_size == 0) { - chunk_size = TB_ARENA_LARGE_CHUNK_SIZE; - } - - // allocate initial chunk - TB_Arena* arena = cuik__valloc(chunk_size); - arena->watermark = arena->data; - arena->high_point = &arena->data[chunk_size - sizeof(TB_Arena)]; - arena->top = arena; - return arena; -} - -void tb_arena_destroy(TB_Arena* restrict arena) { - TB_Arena* c = arena; - while (c != NULL) { - TB_Arena* next = c->next; - cuik__vfree(c, tb_arena_chunk_size(c)); - c = next; - } -} - -void* tb_arena_unaligned_alloc(TB_Arena* restrict arena, size_t size) { - TB_Arena* top = arena->top; - if (LIKELY(top->watermark + size <= top->high_point)) { - char* ptr = top->watermark; - top->watermark += size; - return ptr; - } else { - // slow path, we need to allocate more pages - size_t chunk_size = tb_arena_chunk_size(arena); - assert(size < chunk_size - sizeof(TB_Arena)); - - TB_Arena* c = cuik__valloc(chunk_size); - c->next = NULL; - c->watermark = c->data + size; - c->high_point = &c->data[chunk_size - sizeof(TB_Arena)]; - - // append to top - arena->top->next = c; - arena->top = c; - - return c->data; - } -} - -TB_API void* tb_arena_realloc(TB_Arena* restrict arena, void* old, size_t size) { - char* p = old; - if (p + size == arena->watermark) { - // try to resize - arena->watermark = old; - } - - char* dst = tb_arena_unaligned_alloc(arena, size); - if (dst != p && old) { - memcpy(dst, old, size); - } - return dst; -} - -void tb_arena_pop(TB_Arena* restrict arena, void* ptr, size_t size) { - char* p = ptr; - assert(p + size == arena->watermark); // cannot pop from arena if it's not at the top - - arena->watermark = p; -} - -bool tb_arena_free(TB_Arena* restrict arena, void* ptr, size_t size) { - size = (size + TB_ARENA_ALIGNMENT - 1) & ~(TB_ARENA_ALIGNMENT - 1); - - char* p = ptr; - if (p + size == arena->watermark) { - arena->watermark = p; - return true; - } else { - return false; - } -} - -void tb_arena_realign(TB_Arena* restrict arena) { - ptrdiff_t pos = arena->watermark - arena->top->data; - pos = (pos + TB_ARENA_ALIGNMENT - 1) & ~(TB_ARENA_ALIGNMENT - 1); - - arena->watermark = &arena->top->data[pos]; -} - -TB_ArenaSavepoint tb_arena_save(TB_Arena* arena) { - return (TB_ArenaSavepoint){ arena->top, arena->watermark }; -} - -void tb_arena_restore(TB_Arena* arena, TB_ArenaSavepoint sp) { - // kill any chunks which are ahead of the top - TB_Arena* c = sp.top->next; - while (c != NULL) { - TB_Arena* next = c->next; - cuik__vfree(c, tb_arena_chunk_size(c)); - c = next; - } - - arena->top = sp.top; - arena->top->next = NULL; - arena->watermark = sp.watermark; -} - -void* tb_arena_alloc(TB_Arena* restrict arena, size_t size) { - uintptr_t wm = (uintptr_t) arena->watermark; - assert((wm & ~0xFull) == wm); - - size = (size + TB_ARENA_ALIGNMENT - 1) & ~(TB_ARENA_ALIGNMENT - 1); - return tb_arena_unaligned_alloc(arena, size); -} - -void tb_arena_clear(TB_Arena* arena) { - if (arena != NULL) { - arena->watermark = arena->data; - arena->top = arena; - - // remove extra chunks - TB_Arena* c = arena->next; - while (c != NULL) { - TB_Arena* next = c->next; - cuik__vfree(c, tb_arena_chunk_size(c)); - c = next; - } - - arena->next = NULL; - } -} - -bool tb_arena_is_empty(TB_Arena* arena) { - return arena->top != arena || (arena->watermark - arena->data) > 0; -} - -size_t tb_arena_chunk_size(TB_Arena* arena) { - return arena->high_point - (char*) arena; -} - -size_t tb_arena_current_size(TB_Arena* arena) { - size_t total = 0; - TB_Arena* c = arena; - while (c != NULL) { - total += arena->watermark - (char*) arena; - c = c->next; - } - return total; -} - -//////////////////////////////// -// Futex functions -//////////////////////////////// -void futex_dec(Futex* f) { - if (atomic_fetch_sub(f, 1) == 1) { - futex_signal(f); - } -} - -#ifdef __linux__ -#include -#include -#include -#include - -// [https://man7.org/linux/man-pages/man2/futex.2.html] -// glibc provides no wrapper for futex() -#define futex(...) syscall(SYS_futex, __VA_ARGS__) - -void futex_signal(Futex* addr) { - int ret = futex(addr, FUTEX_WAKE | FUTEX_PRIVATE_FLAG, 1, NULL, NULL, 0); - if (ret == -1) { - __builtin_trap(); - } -} - -void futex_broadcast(Futex* addr) { - int ret = futex(addr, FUTEX_WAKE | FUTEX_PRIVATE_FLAG, INT32_MAX, NULL, NULL, 0); - if (ret == -1) { - __builtin_trap(); - } -} - -void futex_wait(Futex* addr, Futex val) { - for (;;) { - int ret = futex(addr, FUTEX_WAIT | FUTEX_PRIVATE_FLAG, val, NULL, NULL, 0); - - if (ret == -1) { - if (errno != EAGAIN) { - __builtin_trap(); - } - } else if (ret == 0) { - if (*addr != val) { - return; - } - } - } -} - -#undef futex -#elif defined(__APPLE__) - -#include - -enum { - UL_COMPARE_AND_WAIT = 0x00000001, - ULF_WAKE_ALL = 0x00000100, - ULF_NO_ERRNO = 0x01000000 -}; - -/* timeout is specified in microseconds */ -int __ulock_wait(uint32_t operation, void *addr, uint64_t value, uint32_t timeout); -int __ulock_wake(uint32_t operation, void *addr, uint64_t wake_value); - -void futex_signal(Futex* addr) { - for (;;) { - int ret = __ulock_wake(UL_COMPARE_AND_WAIT | ULF_NO_ERRNO, addr, 0); - if (ret >= 0) { - return; - } - ret = -ret; - if (ret == EINTR || ret == EFAULT) { - continue; - } - if (ret == ENOENT) { - return; - } - printf("futex wake fail?\n"); - __builtin_trap(); - } -} - -void _tpool_broadcast(Futex* addr) { - for (;;) { - int ret = __ulock_wake(UL_COMPARE_AND_WAIT | ULF_NO_ERRNO | ULF_WAKE_ALL, addr, 0); - if (ret >= 0) { - return; - } - ret = -ret; - if (ret == EINTR || ret == EFAULT) { - continue; - } - if (ret == ENOENT) { - return; - } - printf("futex wake fail?\n"); - __builtin_trap(); - } -} - -void futex_wait(Futex* addr, Futex val) { - for (;;) { - int ret = __ulock_wait(UL_COMPARE_AND_WAIT | ULF_NO_ERRNO, addr, val, 0); - if (ret >= 0) { - if (*addr != val) { - return; - } - continue; - } - ret = -ret; - if (ret == EINTR || ret == EFAULT) { - continue; - } - if (ret == ENOENT) { - return; - } - - printf("futex wait fail?\n"); - } -} - -#elif defined(_WIN32) -#define WIN32_LEAN_AND_MEAN -#include - -#ifndef __GNUC__ -#pragma comment(lib, "advapi32.lib") -#pragma comment(lib, "Synchronization.lib") -#endif - -void futex_signal(Futex* addr) { - WakeByAddressSingle((void*) addr); -} - -void futex_broadcast(Futex* addr) { - WakeByAddressAll((void*) addr); -} - -void futex_wait(Futex* addr, Futex val) { - for (;;) { - WaitOnAddress(addr, (void *)&val, sizeof(val), INFINITE); - if (*addr != val) break; - } -} -#endif - -void futex_wait_eq(Futex* addr, Futex val) { - while (*addr != val) { - futex_wait(addr, *addr); - } -} diff --git a/vendor/common/common.h b/vendor/common/common.h deleted file mode 100644 index 069bba12..00000000 --- a/vendor/common/common.h +++ /dev/null @@ -1,89 +0,0 @@ -#pragma once -#include -#include -#include -#include -#include -#include - -// Cuik currently uses mimalloc so we wrap those calls here -#ifdef CUIK_USE_MIMALLOC -#include - -#define cuik_malloc(size) mi_malloc(size) -#define cuik_calloc(count, size) mi_calloc(count, size) -#define cuik_free(ptr) mi_free(ptr) -#define cuik_realloc(ptr, size) mi_realloc(ptr, size) -#define cuik_strdup(x) mi_strdup(x) -#else -#define cuik_malloc(size) malloc(size) -#define cuik_calloc(count, size) calloc(count, size) -#define cuik_free(size) free(size) -#define cuik_realloc(ptr, size) realloc(ptr, size) - -#ifdef _WIN32 -#define cuik_strdup(x) _strdup(x) -#else -#define cuik_strdup(x) strdup(x) -#endif -#endif - -#if defined(__amd64) || defined(__amd64__) || defined(_M_AMD64) || defined(__x86_64__) || defined(__x86_64) -#define CUIK__IS_X64 1 -#elif defined(__aarch64__) -#define CUIK__IS_AARCH64 1 -#endif - -// Cuik doesn't have SIMD intrinsics yet... sadge -#if defined(__CUIK__) -#define USE_INTRIN 0 -#else -#define USE_INTRIN 1 -#endif - -#define STR2(x) #x -#define STR(x) STR2(x) - -#ifndef COUNTOF -#define COUNTOF(...) (sizeof(__VA_ARGS__) / sizeof(__VA_ARGS__[0])) -#endif - -#ifdef NDEBUG -#define ASSUME(x) ((x) ? 0 : __builtin_unreachable()) -#else -#define ASSUME(x) ((x) ? 0 : (fprintf(stderr, __FILE__ ": " STR(__LINE__) ": bad assumption: " #x "\n"))) -#endif - -#define LIKELY(x) __builtin_expect(!!(x), 1) -#define UNLIKELY(x) __builtin_expect(!!(x), 0) - -#ifdef NDEBUG -#define TODO() __builtin_unreachable() -#else -#define TODO() (assert(0 && "TODO"), __builtin_unreachable()) -#endif - -// just because we use a threads fallback layer which can include windows -// and such which is annoying... eventually need to modify that out or something -#ifndef thread_local -#define thread_local _Thread_local -#endif - -#define SWAP(T, a, b) \ -do { \ - T temp = a; \ - a = b; \ - b = temp; \ -} while (0) - -void cuik_init_terminal(void); - -void tls_init(void); -void tls_reset(void); -void* tls_push(size_t size); -void* tls_pop(size_t size); -void* tls_save(void); -void tls_restore(void* p); - -void* cuik__valloc(size_t sz); -void cuik__vfree(void* p, size_t sz); diff --git a/vendor/common/dyn_array.h b/vendor/common/dyn_array.h deleted file mode 100644 index 407d722a..00000000 --- a/vendor/common/dyn_array.h +++ /dev/null @@ -1,119 +0,0 @@ -#pragma once -#include -#include -#include - -#define INITIAL_CAP 64 - -typedef struct DynArrayHeader { - // honestly storing the type size is kinda weird - size_t size, capacity; - char data[]; -} DynArrayHeader; - -static void* dyn_array_internal_create(size_t type_size, size_t cap) { - DynArrayHeader* header = cuik_malloc(sizeof(DynArrayHeader) + (type_size * cap)); - - *header = (DynArrayHeader){ - .capacity = cap - }; - return &header->data[0]; -} - -static void dyn_array_internal_destroy(void* ptr) { - if (ptr != NULL) { - DynArrayHeader* header = ((DynArrayHeader*)ptr) - 1; - cuik_free(header); - } -} - -static void* dyn_array_internal_reserve2(void* ptr, size_t type_size, size_t min_size) { - if (ptr == NULL) { - return dyn_array_internal_create(type_size, INITIAL_CAP); - } - - DynArrayHeader* header = ((DynArrayHeader*)ptr) - 1; - if (min_size > header->capacity) { - header->capacity = min_size; - DynArrayHeader* new_ptr = cuik_realloc(header, sizeof(DynArrayHeader) + (type_size * header->capacity)); - if (!new_ptr) { - fprintf(stderr, "error: out of memory!"); - abort(); - } - - return &new_ptr->data[0]; - } - - return ptr; -} - -static void* dyn_array_internal_reserve(void* ptr, size_t type_size, size_t extra) { - if (ptr == NULL) { - return dyn_array_internal_create(type_size, INITIAL_CAP); - } - - DynArrayHeader* header = ((DynArrayHeader*)ptr) - 1; - if (header->size + extra >= header->capacity) { - header->capacity = (header->size + extra) * 2; - - DynArrayHeader* new_ptr = cuik_realloc(header, sizeof(DynArrayHeader) + (type_size * header->capacity)); - if (!new_ptr) { - fprintf(stderr, "error: out of memory!"); - abort(); - } - return &new_ptr->data[0]; - } - - return ptr; -} - -static void* dyn_array_internal_trim(void* ptr, size_t type_size) { - DynArrayHeader* header = ((DynArrayHeader*)ptr) - 1; - header->capacity = header->size; - - DynArrayHeader* new_ptr = cuik_realloc(header, sizeof(DynArrayHeader) + (type_size * header->capacity)); - assert(new_ptr != NULL); - return &new_ptr->data[0]; -} - -#define DynArray(T) T* -#define dyn_array_create(T, cap) dyn_array_internal_create(sizeof(T), cap) -#define dyn_array_destroy(arr) (dyn_array_internal_destroy(arr), (arr) = NULL) -#define dyn_array_pop(arr) ((arr)[(((DynArrayHeader*)(arr)) - 1)->size -= 1]) -#define dyn_array_peek(arr) ((arr)[(((DynArrayHeader*)(arr)) - 1)->size - 1]) - -#define dyn_array_put(arr, ...) \ -do { \ - arr = dyn_array_internal_reserve(arr, sizeof(*arr), 1); \ - DynArrayHeader* header = ((DynArrayHeader*)arr) - 1; \ - (arr)[header->size++] = __VA_ARGS__; \ -} while (0) - -#define dyn_array_insert(arr, at, ...) \ -do { \ - arr = dyn_array_internal_reserve2(arr, sizeof(*arr), at); \ - (arr)[at] = __VA_ARGS__; \ -} while (0) - -#define dyn_array_put_uninit(arr, extra) \ -do { \ - size_t extra_ = (extra); \ - arr = dyn_array_internal_reserve(arr, sizeof(*arr), extra_); \ - DynArrayHeader* header = ((DynArrayHeader*)arr) - 1; \ - header->size += extra_; \ -} while (0) - -#define dyn_array_remove(arr, index) \ -do { \ - DynArrayHeader* header = ((DynArrayHeader*) (arr)) - 1; \ - header->size -= 1; \ - if (header->size > 0) { \ - (arr)[index] = (arr)[header->size]; \ - } \ -} while (0) - -#define dyn_array_trim(arr) (arr = dyn_array_internal_trim(arr, sizeof(*arr))) -#define dyn_array_clear(arr) (arr ? (((((DynArrayHeader*)(arr)) - 1)->size) = 0) : 0) -#define dyn_array_set_length(arr, newlen) (((((DynArrayHeader*)(arr)) - 1)->size) = (newlen)) -#define dyn_array_length(arr) ((arr) ? (((DynArrayHeader*)(arr)) - 1)->size : 0) -#define dyn_array_for(it, arr) for (ptrdiff_t it = 0, _count_ = dyn_array_length(arr); it < _count_; it++) diff --git a/vendor/common/file_map.h b/vendor/common/file_map.h deleted file mode 100644 index 41dab52d..00000000 --- a/vendor/common/file_map.h +++ /dev/null @@ -1,83 +0,0 @@ -#pragma once - -#ifdef _WIN32 -#define WIN32_LEAN_AND_MEAN -#include - -typedef struct { - HANDLE file; - HANDLE mapping; - size_t size; - void* data; -} FileMap; - -static FileMap open_file_map(const char* filepath) { - HANDLE file = CreateFileA(filepath, GENERIC_READ, FILE_SHARE_READ, 0, OPEN_EXISTING, 0, 0); - if (file == INVALID_HANDLE_VALUE) { - return (FileMap){ INVALID_HANDLE_VALUE }; - } - - LARGE_INTEGER size; - if (!GetFileSizeEx(file, &size)) { - fprintf(stderr, "Could not read file size! %s", filepath); - return (FileMap){ INVALID_HANDLE_VALUE }; - } - - HANDLE mapping = CreateFileMappingA(file, NULL, PAGE_READONLY, 0, 0, 0); - if (!mapping) { - fprintf(stderr, "Could not map file! %s", filepath); - return (FileMap){ INVALID_HANDLE_VALUE }; - } - - void* memory = MapViewOfFileEx(mapping, FILE_MAP_READ, 0, 0, 0, 0); - if (!memory) { - fprintf(stderr, "Could not view mapped file! %s", filepath); - return (FileMap){ INVALID_HANDLE_VALUE }; - } - - return (FileMap){ - .file = file, - .mapping = mapping, - .size = size.QuadPart, - .data = memory - }; -} - -static void close_file_map(FileMap* file_map) { - UnmapViewOfFile(file_map->data); - CloseHandle(file_map->mapping); - CloseHandle(file_map->file); -} -#else -#include -#include -#include -#include - -typedef struct { - int fd; - size_t size; - void* data; -} FileMap; - -static FileMap open_file_map(const char* filepath) { - int fd = open(filepath, O_RDONLY); - - struct stat file_stats; - if (fstat(fd, &file_stats) == -1) { - return (FileMap){ 0 }; - } - - void* buffer = mmap(NULL, file_stats.st_size, PROT_READ, MAP_PRIVATE, fd, 0); - if (buffer == MAP_FAILED) { - return (FileMap){ 0 }; - } - - return (FileMap){ fd, file_stats.st_size, buffer }; -} - -static void close_file_map(FileMap* file_map) { - munmap(file_map->data, file_map->size); - close(file_map->fd); -} -#endif diff --git a/vendor/common/futex.h b/vendor/common/futex.h deleted file mode 100644 index 16c47b46..00000000 --- a/vendor/common/futex.h +++ /dev/null @@ -1,16 +0,0 @@ -#pragma once - -#include - -#ifdef __APPLE__ -typedef _Atomic int32_t Futex; -#else -typedef _Atomic int64_t Futex; -#endif - -void futex_inc(Futex* f); -void futex_dec(Futex* f); -void futex_signal(Futex* f); -void futex_broadcast(Futex* f); -void futex_wait(Futex* f, Futex val); // leaves if *f != val -void futex_wait_eq(Futex* f, Futex val); // leaves if *f == val diff --git a/vendor/common/hash_map.h b/vendor/common/hash_map.h deleted file mode 100644 index 7c92ab30..00000000 --- a/vendor/common/hash_map.h +++ /dev/null @@ -1,363 +0,0 @@ -#ifndef NL_HASH_MAP_H -#define NL_HASH_MAP_H - -#include -#include -#include -#include -#include -#include -#include - -#if defined(TB_USE_MIMALLOC) || defined(CUIK_USE_MIMALLOC) -#include - -#define NL_MALLOC(s) mi_malloc(s) -#define NL_CALLOC(c, s) mi_calloc(c, s) -#define NL_REALLOC(p, s) mi_realloc(p, s) -#define NL_FREE(p) mi_free(p) -#else -#define NL_MALLOC(s) malloc(s) -#define NL_CALLOC(c, s) calloc(c, s) -#define NL_REALLOC(p, s) realloc(p, s) -#define NL_FREE(p) free(p) -#endif - -#define NL_Map(K, V) struct { K k; V v; }* -#define NL_Strmap(T) struct { NL_Slice k; T v; }* - -typedef struct { - size_t length; - const uint8_t* data; -} NL_Slice; - -///////////////////////////////////////////////// -// public macros -///////////////////////////////////////////////// -#define nl_map_is_strmap(map) _Generic((map)->k, NL_Slice: true, default: false) - -#define nl_map_create(map, initial_cap) ((map) = ((void*) nl_map__alloc(initial_cap, sizeof(*map))->kv_table)) - -#define nl_map_put(map, key, value) \ -do { \ - NL_MapInsert ins__ = (nl_map_is_strmap(map) ? nl_map__inserts : nl_map__insert)((map), sizeof(*(map)), sizeof(key), &(key)); \ - (map) = ins__.new_map; \ - (map)[ins__.index].v = (value); \ -} while (0) - -#define nl_map_puti(map, key, out_index) \ -do { \ - NL_MapInsert ins__ = (nl_map_is_strmap(map) ? nl_map__inserts : nl_map__insert)((map), sizeof(*(map)), sizeof(key), &(key)); \ - (map) = ins__.new_map; \ - (out_index) = ins__.index; \ -} while (0) - -#define nl_map_put_cstr(map, key, value) \ -do { \ - NL_Slice key_ = { strlen(key), (const uint8_t*) (key) }; \ - NL_MapInsert ins__ = (nl_map_is_strmap(map) ? nl_map__inserts : nl_map__insert)((map), sizeof(*(map)), sizeof(key_), &key_); \ - (map) = ins__.new_map; \ - (map)[ins__.index].v = (value); \ -} while (0) - -#define nl_map_puti_cstr(map, key, out_index) \ -do { \ - NL_Slice key_ = { strlen(key), (const uint8_t*) (key) }; \ - NL_MapInsert ins__ = (nl_map_is_strmap(map) ? nl_map__inserts : nl_map__insert)((map), sizeof(*(map)), sizeof(key_), &key_); \ - (map) = ins__.new_map; \ - (out_index) = ins__.index; \ -} while (0) - -#define nl_map_remove(map, key) ((map) != NULL ? nl_map__remove(map, sizeof(*map), sizeof(key), &(key)) : -1) - -#define nl_map_get(map, key) ((map) != NULL ? (nl_map_is_strmap(map) ? nl_map__gets : nl_map__get)(((NL_MapHeader*)(map)) - 1, sizeof(*map), sizeof(key), &(key)) : -1) -#define nl_map_get_checked(map, key) ((map)[nl_map__check(nl_map_get(map, key))].v) -#define nl_map_get_cstr(map, key) ((map) != NULL ? nl_map__gets(((NL_MapHeader*)(map)) - 1, sizeof(*map), sizeof(NL_Slice), &(NL_Slice){ strlen(key), (const uint8_t*) (key) }) : -1) - -#define nl_map_for_str(it, map) \ -for (size_t it = 0; it < nl_map_get_capacity(map); it++) if ((map)[it].k.length != 0) - -#define nl_map_for(it, map) \ -for (size_t it = 0; it < nl_map_get_capacity(map); it++) if ((map)[it].k != 0 && (map)[it].k != (void*) (uintptr_t) -1) - -#define nl_map_free(map) \ -do { \ - if ((map) != NULL) { \ - nl_map__free(((NL_MapHeader*)(map)) - 1); \ - (map) = NULL; \ - } \ -} while (0) - -///////////////////////////////////////////////// -// internals -///////////////////////////////////////////////// -#define nl_map__get_header(map) (((NL_MapHeader*)(map)) - 1) -#define nl_map_get_capacity(map) ((map) ? 1ull << nl_map__get_header(map)->exp : 0) - -typedef struct { - void* new_map; - size_t index; -} NL_MapInsert; - -// behind the array the user manages there's some -// information about the rest of the string map -typedef struct { - size_t count; - size_t exp; - char kv_table[]; -} NL_MapHeader; - -inline static NL_Slice nl_slice__cstr(const char* key) { - return (NL_Slice){strlen(key), (const uint8_t*)key}; -} - -NL_MapHeader* nl_map__alloc(size_t cap, size_t entry_size); -NL_MapInsert nl_map__insert(void* map, size_t entry_size, size_t key_size, const void* key); -NL_MapInsert nl_map__inserts(void* map, size_t entry_size, size_t key_size, const void* key); -ptrdiff_t nl_map__get(NL_MapHeader* restrict table, size_t entry_size, size_t key_size, const void* key); -ptrdiff_t nl_map__gets(NL_MapHeader* restrict table, size_t entry_size, size_t key_size, const void* key); -void nl_map__free(NL_MapHeader* restrict table); -void nl_map__remove(void* map, size_t entry_size, size_t key_size, const void* key); - -inline static ptrdiff_t nl_map__check(ptrdiff_t x) { - assert(x >= 0 && "map entry not found!"); - return x; -} - -#endif /* NL_HASH_MAP_H */ - -#ifdef NL_MAP_IMPL - -// FNV1A -inline static uint32_t nl_map__raw_hash(size_t len, const void *key) { - const uint8_t* data = key; - uint32_t h = 0x811C9DC5; - for (size_t i = 0; i < len; i++) { - h = (data[i] ^ h) * 0x01000193; - } - - return h; -} - -void nl_map__free(NL_MapHeader* restrict table) { - NL_FREE(table); -} - -NL_MapHeader* nl_map__alloc(size_t cap, size_t entry_size) { - cap = (cap * 4) / 3; - if (cap < 4) cap = 4; - - // next power of two - #if defined(_MSC_VER) && !defined(__clang__) - size_t exp = 64 - _lzcnt_u64(cap - 1); - #else - size_t exp = 64 - __builtin_clzll(cap - 1); - #endif - - cap = (cap == 1 ? 1 : 1 << exp); - - NL_MapHeader* table = NL_CALLOC(1, sizeof(NL_MapHeader) + (cap * entry_size)); - table->exp = exp; - table->count = 0; - return table; -} - -static bool nl_map__is_zero(const char* ptr, size_t size) { - // we're almost exclusively using this code for pointer keys - if (size == sizeof(void*)) { - return *((uintptr_t*) ptr) == 0; - } else { - for (size_t i = 0; i < size; i++) { - if (ptr[i] != 0) return false; - } - - return true; - } -} - -static bool nl_map__is_one(const char* ptr, size_t size) { - // we're almost exclusively using this code for pointer keys - if (size == sizeof(void*)) { - return *((uintptr_t*) ptr) == UINTPTR_MAX; - } else { - for (size_t i = 0; i < size; i++) { - if (ptr[i] != (char)0xFF) return false; - } - - return true; - } -} - -NL_MapHeader* nl_map__rehash(NL_MapHeader* table, size_t entry_size, size_t key_size, bool is_strmap) { - size_t count = 1u << table->exp; - - // Allocate bigger hashmap - NL_MapHeader* new_table = nl_map__alloc(count * 2, entry_size); - if (is_strmap) { - for (size_t i = 0; i < count; i++) { - NL_Slice* slot_entry = (NL_Slice*) &table->kv_table[i * entry_size]; - if (slot_entry->length > 0) { - NL_MapInsert ins = nl_map__inserts(new_table->kv_table, entry_size, key_size, slot_entry); - memcpy(&new_table->kv_table[ins.index*entry_size + key_size], &slot_entry[1], entry_size - key_size); - } - } - } else { - for (size_t i = 0; i < count; i++) { - const char* slot_entry = (const char*) &table->kv_table[i * entry_size]; - if (!nl_map__is_zero(slot_entry, key_size) && !nl_map__is_one(slot_entry, key_size)) { - NL_MapInsert ins = nl_map__insert(new_table->kv_table, entry_size, key_size, slot_entry); - memcpy(&new_table->kv_table[ins.index*entry_size + key_size], &slot_entry[key_size], entry_size - key_size); - } - } - } - - nl_map__free(table); - return new_table; -} - -NL_MapInsert nl_map__insert(void* map, size_t entry_size, size_t key_size, const void* key) { - NL_MapHeader* table; - if (map == NULL) { - table = nl_map__alloc(1024, entry_size); - map = table->kv_table; - } else { - table = ((NL_MapHeader*)map) - 1; - } - - uint32_t cap = 1ull << table->exp; - if (table->count >= (cap * 3) / 4) { - // past 75% load... resize - table = nl_map__rehash(table, entry_size, key_size, false); - map = table->kv_table; - } - - uint32_t exp = table->exp; - uint32_t mask = (1 << table->exp) - 1; - uint32_t hash = nl_map__raw_hash(key_size, key); - - for (size_t i = hash;;) { - // hash table lookup - uint32_t step = (hash >> (32 - exp)) | 1; - i = (i + step) & mask; - - void* slot_entry = &table->kv_table[i * entry_size]; - if (nl_map__is_zero(slot_entry, key_size) || nl_map__is_one(slot_entry, key_size)) { - table->count++; - memcpy(slot_entry, key, key_size); - return (NL_MapInsert){ map, i }; - } else if (memcmp(slot_entry, key, key_size) == 0) { - return (NL_MapInsert){ map, i }; - } - } -} - -void nl_map__remove(void* map, size_t entry_size, size_t key_size, const void* key) { - if (map == NULL) { - return; - } - - NL_MapHeader* table = ((NL_MapHeader*)map) - 1; - - uint32_t exp = table->exp; - uint32_t mask = (1 << table->exp) - 1; - uint32_t hash = nl_map__raw_hash(key_size, key); - - for (size_t i = hash;;) { - // hash table lookup - uint32_t step = (hash >> (32 - exp)) | 1; - i = (i + step) & mask; - - void* slot_entry = &table->kv_table[i * entry_size]; - if (nl_map__is_zero(slot_entry, key_size)) { - break; - } else if (memcmp(slot_entry, key, key_size) == 0) { - table->count--; - memset(slot_entry, 0xFF, key_size); // mark key as TOMBSTONE - break; - } - } -} - -NL_MapInsert nl_map__inserts(void* map, size_t entry_size, size_t key_size, const void* key) { - NL_MapHeader* table; - if (map == NULL) { - table = nl_map__alloc(256, entry_size); - map = table->kv_table; - } else { - table = ((NL_MapHeader*)map) - 1; - } - - uint32_t cap = 1ull << table->exp; - if (table->count >= (cap * 3) / 4) { - // past 75% load... resize - table = nl_map__rehash(table, entry_size, key_size, true); - map = table->kv_table; - } - - const NL_Slice* key_entry = key; - - uint32_t exp = table->exp; - uint32_t mask = (1 << table->exp) - 1; - uint32_t hash = nl_map__raw_hash(key_entry->length, key_entry->data); - - for (size_t i = hash;;) { - // hash table lookup - uint32_t step = (hash >> (32 - exp)) | 1; - i = (i + step) & mask; - - NL_Slice* slot_entry = (NL_Slice*) &table->kv_table[i * entry_size]; - if (slot_entry->length == 0) { - table->count++; - memcpy(slot_entry, key, key_size); - return (NL_MapInsert){ map, i }; - } else if (slot_entry->length == key_entry->length && memcmp(slot_entry->data, key_entry->data, key_entry->length) == 0) { - return (NL_MapInsert){ map, i }; - } - } -} - -ptrdiff_t nl_map__get(NL_MapHeader* restrict table, size_t entry_size, size_t key_size, const void* key) { - if (table == NULL) return -1; - - uint32_t exp = table->exp; - uint32_t mask = (1 << table->exp) - 1; - uint32_t hash = nl_map__raw_hash(key_size, key); - - for (size_t i = hash;;) { - // hash table lookup - uint32_t step = (hash >> (32 - exp)) | 1; - i = (i + step) & mask; - - void* slot_entry = &table->kv_table[i * entry_size]; - if (nl_map__is_zero(slot_entry, key_size)) { - return -1; - } else if (memcmp(slot_entry, key, key_size) == 0) { - return i; - } - } -} - -ptrdiff_t nl_map__gets(NL_MapHeader* restrict table, size_t entry_size, size_t key_size, const void* key) { - if (table == NULL) return -1; - - const NL_Slice* key_entry = key; - - uint32_t exp = table->exp; - uint32_t mask = (1 << table->exp) - 1; - uint32_t hash = nl_map__raw_hash(key_entry->length, key_entry->data); - - for (size_t i = hash;;) { - // hash table lookup - uint32_t step = (hash >> (32 - exp)) | 1; - i = (i + step) & mask; - - NL_Slice* slot_entry = (NL_Slice*) &table->kv_table[i * entry_size]; - if (slot_entry->length == 0) { - return -1; - } else if (slot_entry->length == key_entry->length && memcmp(slot_entry->data, key_entry->data, key_entry->length) == 0) { - return i; - } - } -} - -#endif /* NL_MAP_IMPL */ diff --git a/vendor/common/hashes.h b/vendor/common/hashes.h deleted file mode 100644 index 8508e73b..00000000 --- a/vendor/common/hashes.h +++ /dev/null @@ -1,84 +0,0 @@ -#pragma once -#include - -// murmur3 32-bit without UB unaligned accesses -// https://github.com/demetri/scribbles/blob/master/hashing/ub_aware_hash_functions.c -static uint32_t tb__murmur3_32(const void* key, size_t len) { - const uint32_t* key32 = key; - uint32_t h = 0; - - // main body, work on 32-bit blocks at a time - for (size_t i=0;i> 17))*0x1b873593; - h = (((h^k) << 13) | ((h^k) >> 19))*5 + 0xe6546b64; - } - - // load/mix up to 3 remaining tail bytes into a tail block - uint32_t t = 0; - const uint8_t *tail = ((const uint8_t*) key) + 4*(len/4); - switch(len & 3) { - case 3: t ^= tail[2] << 16; - case 2: t ^= tail[1] << 8; - case 1: { - t ^= tail[0] << 0; - h ^= ((0xcc9e2d51*t << 15) | (0xcc9e2d51*t >> 17))*0x1b873593; - } - } - - // finalization mix, including key length - h = ((h^len) ^ ((h^len) >> 16))*0x85ebca6b; - h = (h ^ (h >> 13))*0xc2b2ae35; - return (h ^ (h >> 16)); -} - -static const uint32_t crc_table[256] = { - 0x00000000, 0x77073096, 0xee0e612c, 0x990951ba, 0x076dc419, 0x706af48f, 0xe963a535, 0x9e6495a3, - 0x0edb8832, 0x79dcb8a4, 0xe0d5e91e, 0x97d2d988, 0x09b64c2b, 0x7eb17cbd, 0xe7b82d07, 0x90bf1d91, - 0x1db71064, 0x6ab020f2, 0xf3b97148, 0x84be41de, 0x1adad47d, 0x6ddde4eb, 0xf4d4b551, 0x83d385c7, - 0x136c9856, 0x646ba8c0, 0xfd62f97a, 0x8a65c9ec, 0x14015c4f, 0x63066cd9, 0xfa0f3d63, 0x8d080df5, - 0x3b6e20c8, 0x4c69105e, 0xd56041e4, 0xa2677172, 0x3c03e4d1, 0x4b04d447, 0xd20d85fd, 0xa50ab56b, - 0x35b5a8fa, 0x42b2986c, 0xdbbbc9d6, 0xacbcf940, 0x32d86ce3, 0x45df5c75, 0xdcd60dcf, 0xabd13d59, - 0x26d930ac, 0x51de003a, 0xc8d75180, 0xbfd06116, 0x21b4f4b5, 0x56b3c423, 0xcfba9599, 0xb8bda50f, - 0x2802b89e, 0x5f058808, 0xc60cd9b2, 0xb10be924, 0x2f6f7c87, 0x58684c11, 0xc1611dab, 0xb6662d3d, - 0x76dc4190, 0x01db7106, 0x98d220bc, 0xefd5102a, 0x71b18589, 0x06b6b51f, 0x9fbfe4a5, 0xe8b8d433, - 0x7807c9a2, 0x0f00f934, 0x9609a88e, 0xe10e9818, 0x7f6a0dbb, 0x086d3d2d, 0x91646c97, 0xe6635c01, - 0x6b6b51f4, 0x1c6c6162, 0x856530d8, 0xf262004e, 0x6c0695ed, 0x1b01a57b, 0x8208f4c1, 0xf50fc457, - 0x65b0d9c6, 0x12b7e950, 0x8bbeb8ea, 0xfcb9887c, 0x62dd1ddf, 0x15da2d49, 0x8cd37cf3, 0xfbd44c65, - 0x4db26158, 0x3ab551ce, 0xa3bc0074, 0xd4bb30e2, 0x4adfa541, 0x3dd895d7, 0xa4d1c46d, 0xd3d6f4fb, - 0x4369e96a, 0x346ed9fc, 0xad678846, 0xda60b8d0, 0x44042d73, 0x33031de5, 0xaa0a4c5f, 0xdd0d7cc9, - 0x5005713c, 0x270241aa, 0xbe0b1010, 0xc90c2086, 0x5768b525, 0x206f85b3, 0xb966d409, 0xce61e49f, - 0x5edef90e, 0x29d9c998, 0xb0d09822, 0xc7d7a8b4, 0x59b33d17, 0x2eb40d81, 0xb7bd5c3b, 0xc0ba6cad, - 0xedb88320, 0x9abfb3b6, 0x03b6e20c, 0x74b1d29a, 0xead54739, 0x9dd277af, 0x04db2615, 0x73dc1683, - 0xe3630b12, 0x94643b84, 0x0d6d6a3e, 0x7a6a5aa8, 0xe40ecf0b, 0x9309ff9d, 0x0a00ae27, 0x7d079eb1, - 0xf00f9344, 0x8708a3d2, 0x1e01f268, 0x6906c2fe, 0xf762575d, 0x806567cb, 0x196c3671, 0x6e6b06e7, - 0xfed41b76, 0x89d32be0, 0x10da7a5a, 0x67dd4acc, 0xf9b9df6f, 0x8ebeeff9, 0x17b7be43, 0x60b08ed5, - 0xd6d6a3e8, 0xa1d1937e, 0x38d8c2c4, 0x4fdff252, 0xd1bb67f1, 0xa6bc5767, 0x3fb506dd, 0x48b2364b, - 0xd80d2bda, 0xaf0a1b4c, 0x36034af6, 0x41047a60, 0xdf60efc3, 0xa867df55, 0x316e8eef, 0x4669be79, - 0xcb61b38c, 0xbc66831a, 0x256fd2a0, 0x5268e236, 0xcc0c7795, 0xbb0b4703, 0x220216b9, 0x5505262f, - 0xc5ba3bbe, 0xb2bd0b28, 0x2bb45a92, 0x5cb36a04, 0xc2d7ffa7, 0xb5d0cf31, 0x2cd99e8b, 0x5bdeae1d, - 0x9b64c2b0, 0xec63f226, 0x756aa39c, 0x026d930a, 0x9c0906a9, 0xeb0e363f, 0x72076785, 0x05005713, - 0x95bf4a82, 0xe2b87a14, 0x7bb12bae, 0x0cb61b38, 0x92d28e9b, 0xe5d5be0d, 0x7cdcefb7, 0x0bdbdf21, - 0x86d3d2d4, 0xf1d4e242, 0x68ddb3f8, 0x1fda836e, 0x81be16cd, 0xf6b9265b, 0x6fb077e1, 0x18b74777, - 0x88085ae6, 0xff0f6a70, 0x66063bca, 0x11010b5c, 0x8f659eff, 0xf862ae69, 0x616bffd3, 0x166ccf45, - 0xa00ae278, 0xd70dd2ee, 0x4e048354, 0x3903b3c2, 0xa7672661, 0xd06016f7, 0x4969474d, 0x3e6e77db, - 0xaed16a4a, 0xd9d65adc, 0x40df0b66, 0x37d83bf0, 0xa9bcae53, 0xdebb9ec5, 0x47b2cf7f, 0x30b5ffe9, - 0xbdbdf21c, 0xcabac28a, 0x53b39330, 0x24b4a3a6, 0xbad03605, 0xcdd70693, 0x54de5729, 0x23d967bf, - 0xb3667a2e, 0xc4614ab8, 0x5d681b02, 0x2a6f2b94, 0xb40bbe37, 0xc30c8ea1, 0x5a05df1b, 0x2d02ef8d -}; - -// shamelessly ripped from LLVM: -// https://llvm.org/doxygen/CRC_8cpp_source.html#l00103 -static uint32_t tb__crc32(uint32_t crc, size_t length, const void* d) { - const uint8_t* data = d; - - crc ^= 0xFFFFFFFFU; - for (size_t i = 0; i < length; i++) { - int table_index = (crc ^ data[i]) & 0xff; - crc = crc_table[table_index] ^ (crc >> 8); - } - return crc ^ 0xFFFFFFFFU; -} diff --git a/vendor/common/log.c b/vendor/common/log.c deleted file mode 100644 index 5c72cc37..00000000 --- a/vendor/common/log.c +++ /dev/null @@ -1,167 +0,0 @@ -/* - * Copyright (c) 2020 rxi - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to - * deal in the Software without restriction, including without limitation the - * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or - * sell copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING - * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - * - * Modified by NeGate to enforce C11 mutexes - */ - -#include "log.h" -#include - -#ifdef _WIN32 -#ifdef _POSIX_C_SOURCE -__declspec(dllimport) unsigned int GetCurrentThreadId(void); -#else -__declspec(dllimport) unsigned long GetCurrentThreadId(void); -#endif -#else -#include -#endif - -#define MAX_CALLBACKS 32 - -typedef struct { - log_LogFn fn; - void *udata; - int level; -} Callback; - -static struct { - once_flag init; - - void *udata; - mtx_t lock; - int level; - bool quiet; - Callback callbacks[MAX_CALLBACKS]; -} L; - -static const char *level_strings[] = { - "TRACE", "DEBUG", "INFO", "WARN", "ERROR", "FATAL" -}; - -#ifdef LOG_USE_COLOR -static const char *level_colors[] = { - "\x1b[94m", "\x1b[36m", "\x1b[32m", "\x1b[33m", "\x1b[31m", "\x1b[35m" -}; -#endif - -static void stdout_callback(log_Event *ev) { - char buf[16]; - buf[strftime(buf, sizeof(buf), "%H:%M:%S", ev->time)] = '\0'; - #ifdef LOG_USE_COLOR - fprintf( - ev->udata, "Thread-%d %s %s%-5s\x1b[0m \x1b[90m%s:%d:\x1b[0m ", - ev->tid, buf, level_colors[ev->level], level_strings[ev->level], - ev->file, ev->line); - #else - fprintf( - ev->udata, "Thread-%d %s %-5s %s:%d: ", - ev->tid, buf, level_strings[ev->level], ev->file, ev->line); - #endif - vfprintf(ev->udata, ev->fmt, ev->ap); - fprintf(ev->udata, "\n"); - fflush(ev->udata); -} - -static void file_callback(log_Event *ev) { - char buf[64]; - buf[strftime(buf, sizeof(buf), "%Y-%m-%d %H:%M:%S", ev->time)] = '\0'; - fprintf( - ev->udata, "Thread-%d %s %-5s %s:%d: ", - ev->tid, buf, level_strings[ev->level], ev->file, ev->line); - vfprintf(ev->udata, ev->fmt, ev->ap); - fprintf(ev->udata, "\n"); - fflush(ev->udata); -} - -const char* log_level_string(int level) { - return level_strings[level]; -} - -void log_set_level(int level) { - L.level = level; -} - -void log_set_quiet(bool enable) { - L.quiet = enable; -} - -int log_add_callback(log_LogFn fn, void *udata, int level) { - for (int i = 0; i < MAX_CALLBACKS; i++) { - if (!L.callbacks[i].fn) { - L.callbacks[i] = (Callback) { fn, udata, level }; - return 0; - } - } - return -1; -} - -int log_add_fp(FILE *fp, int level) { - return log_add_callback(file_callback, fp, level); -} - -static void init_event(log_Event *ev, void *udata) { - if (!ev->time) { - time_t t = time(NULL); - ev->time = localtime(&t); - } - ev->udata = udata; -} - -static void log_init(void) { - mtx_init(&L.lock, mtx_plain); -} - -void log_log(int level, const char *file, int line, const char *fmt, ...) { - log_Event ev = { - .fmt = fmt, - .file = file, - .line = line, - .level = level, - #if _WIN32 - .tid = GetCurrentThreadId(), - #else - .tid = getpid(), - #endif - }; - - call_once(&L.init, log_init); - mtx_lock(&L.lock); - - if (!L.quiet && level >= L.level) { - init_event(&ev, stderr); - va_start(ev.ap, fmt); - stdout_callback(&ev); - va_end(ev.ap); - } - - for (int i = 0; i < MAX_CALLBACKS && L.callbacks[i].fn; i++) { - Callback *cb = &L.callbacks[i]; - if (level >= cb->level) { - init_event(&ev, cb->udata); - va_start(ev.ap, fmt); - cb->fn(&ev); - va_end(ev.ap); - } - } - - mtx_unlock(&L.lock); -} diff --git a/vendor/common/log.h b/vendor/common/log.h deleted file mode 100644 index db9c1446..00000000 --- a/vendor/common/log.h +++ /dev/null @@ -1,58 +0,0 @@ -/** -* Copyright (c) 2020 rxi -* -* This library is free software; you can redistribute it and/or modify it -* under the terms of the MIT license. See `log.c` for details. -* -* Modified by NeGate to enforce C11 mutexes -*/ -#pragma once - -#include -#include -#include -#include - -#define LOG_VERSION "0.1.0" - -typedef struct { - va_list ap; - const char *fmt; - const char *file; - struct tm *time; - void *udata; - int tid; - int line; - int level; -} log_Event; - -typedef void (*log_LogFn)(log_Event *ev); -typedef void (*log_LockFn)(bool lock, void *udata); - -enum { LOG_TRACE, LOG_DEBUG, LOG_INFO, LOG_WARN, LOG_ERROR, LOG_FATAL }; - -#if 0 -#define log_trace(...) log_log(LOG_TRACE, __FILE__, __LINE__, __VA_ARGS__) -#define log_debug(...) log_log(LOG_DEBUG, __FILE__, __LINE__, __VA_ARGS__) -#define log_info(...) log_log(LOG_INFO, __FILE__, __LINE__, __VA_ARGS__) -#define log_warn(...) log_log(LOG_WARN, __FILE__, __LINE__, __VA_ARGS__) -#define log_error(...) log_log(LOG_ERROR, __FILE__, __LINE__, __VA_ARGS__) -#define log_fatal(...) log_log(LOG_FATAL, __FILE__, __LINE__, __VA_ARGS__) -#define log_watch(fmt, var) log_log(LOG_DEBUG, __FILE__, __LINE__, #var " = " fmt, (uint64_t)var) -#else -#define log_trace(...) (__VA_ARGS__, 0) -#define log_debug(...) (__VA_ARGS__, 0) -#define log_info(...) (__VA_ARGS__, 0) -#define log_warn(...) (__VA_ARGS__, 0) -#define log_error(...) (__VA_ARGS__, 0) -#define log_fatal(...) (__VA_ARGS__, 0) -#define log_watch(fmt, var) (var, 0) -#endif - -const char* log_level_string(int level); -void log_set_level(int level); -void log_set_quiet(bool enable); -int log_add_callback(log_LogFn fn, void *udata, int level); -int log_add_fp(FILE *fp, int level); - -void log_log(int level, const char *file, int line, const char *fmt, ...); diff --git a/vendor/common/new_hash_map.h b/vendor/common/new_hash_map.h deleted file mode 100644 index 2a9e9260..00000000 --- a/vendor/common/new_hash_map.h +++ /dev/null @@ -1,357 +0,0 @@ -// This is built specifically for pointer hash sets -#ifndef NL_HASH_SET_H -#define NL_HASH_SET_H - -#define NL_HASHSET_TOMB ((void*) UINTPTR_MAX) - -#define NL_HASHSET_HIGH_BIT (~(SIZE_MAX >> ((size_t) 1))) -#define NL_HASHSET_INDEX_BITS (SIZE_MAX >> ((size_t) 1)) - -//////////////////////////////// -// Hashset -//////////////////////////////// -typedef struct NL_HashSet { - TB_Arena* allocator; - - size_t exp, count; - void** data; -} NL_HashSet; - -typedef uint32_t (*NL_HashFunc)(void* a); -typedef bool (*NL_CompareFunc)(void* a, void* b); - -NL_HashSet nl_hashset_alloc(size_t cap); -NL_HashSet nl_hashset_arena_alloc(TB_Arena* arena, size_t cap); -void nl_hashset_free(NL_HashSet hs); - -void nl_hashset_clear(NL_HashSet* restrict hs); -bool nl_hashset_remove(NL_HashSet* restrict hs, void* ptr); -void* nl_hashset_put(NL_HashSet* restrict hs, void* ptr); - -void* nl_hashset_get2(NL_HashSet* restrict hs, void* ptr, NL_HashFunc hash, NL_CompareFunc cmp); - -// this one takes a custom hash function -void* nl_hashset_put2(NL_HashSet* restrict hs, void* ptr, NL_HashFunc hash, NL_CompareFunc cmp); -void nl_hashset_remove2(NL_HashSet* restrict hs, void* ptr, NL_HashFunc hash, NL_CompareFunc cmp); - -#define nl_hashset_capacity(hs) (1ull << (hs)->exp) -#define nl_hashset_for(it, hs) for (void **it = (hs)->data, **_end_ = &it[nl_hashset_capacity(hs)]; it != _end_; it++) if (*it != NULL && *it != NL_HASHSET_TOMB) - -//////////////////////////////// -// Hashmap -//////////////////////////////// -typedef struct { - void *k, *v; -} NL_TableEntry; - -typedef struct { - TB_Arena* allocator; - - size_t exp, count; - NL_TableEntry* data; -} NL_Table; - -NL_Table nl_table_alloc(size_t cap); -NL_Table nl_table_arena_alloc(TB_Arena* arena, size_t cap); -void nl_table_free(NL_Table tbl); - -void* nl_table_put(NL_Table* restrict tbl, void* k, void* v); -void* nl_table_get(NL_Table* restrict tbl, void* k); -size_t nl_table_lookup(NL_Table* restrict tbl, void* k); - -#define nl_table_capacity(tbl) (1ull << (tbl)->exp) -#define nl_table_for(it, tbl) for (NL_TableEntry *it = (tbl)->data, *_end_ = &it[nl_table_capacity(tbl)]; it != _end_; it++) if (it->k != NULL && it->k != NL_HASHSET_TOMB) - -#endif /* NL_HASH_SET_H */ - -#ifdef NL_HASH_SET_IMPL -#include - -#define NL_HASHSET_HASH(ptr) ((((uintptr_t) ptr) * 11400714819323198485ull) >> 32ull) - -NL_HashSet nl_hashset_alloc_exp(size_t exp) { - return (NL_HashSet){ .exp = exp, .data = cuik_calloc(1u << exp, sizeof(void*)) }; -} - -NL_HashSet nl_hashset_alloc(size_t cap) { - cap = (cap * 4) / 3; - if (cap < 4) cap = 4; - - // next power of two - #if defined(_MSC_VER) && !defined(__clang__) - size_t exp = 64 - _lzcnt_u64(cap - 1); - #else - size_t exp = 64 - __builtin_clzll(cap - 1); - #endif - - return (NL_HashSet){ .exp = exp, .data = cuik_calloc(1u << exp, sizeof(void*)) }; -} - -NL_HashSet nl_hashset_arena_alloc(TB_Arena* arena, size_t cap) { - cap = (cap * 4) / 3; - if (cap < 4) cap = 4; - - // next power of two - #if defined(_MSC_VER) && !defined(__clang__) - size_t exp = 64 - _lzcnt_u64(cap - 1); - #else - size_t exp = 64 - __builtin_clzll(cap - 1); - #endif - - cap = (cap == 1 ? 1 : 1 << exp); - - void* data = tb_arena_alloc(arena, cap * sizeof(void*)); - memset(data, 0, cap * sizeof(void*)); - return (NL_HashSet){ .allocator = arena, .exp = exp, .data = data }; -} - -void nl_hashset_free(NL_HashSet hs) { - if (hs.allocator == NULL) { - cuik_free(hs.data); - } else { - tb_arena_pop(hs.allocator, hs.data, (1ull << hs.exp) * sizeof(void*)); - } -} - -void* nl_hashset_put(NL_HashSet* restrict hs, void* ptr) { - // rehash - uint32_t threshold = ((1 << hs->exp) * 3) / 4; - if (hs->count >= threshold) { - assert(hs->allocator == NULL && "arena hashsets can't be resized!"); - NL_HashSet new_hs = nl_hashset_alloc(nl_hashset_capacity(hs)); - nl_hashset_for(p, hs) { - nl_hashset_put(&new_hs, *p); - } - nl_hashset_free(*hs); - *hs = new_hs; - } - - assert(ptr); - uint32_t h = NL_HASHSET_HASH(ptr); - - size_t mask = (1 << hs->exp) - 1; - size_t first = h & mask, i = first; - do { - if (hs->data[i] == NULL) { - hs->count++; - hs->data[i] = ptr; - return NULL; - } else if (hs->data[i] == ptr) { - return ptr; - } - - i = (i + 1) & mask; - } while (i != first); - - abort(); -} - -bool nl_hashset_remove(NL_HashSet* restrict hs, void* ptr) { - assert(ptr); - uint32_t h = NL_HASHSET_HASH(ptr); - - size_t mask = (1 << hs->exp) - 1; - size_t first = h & mask, i = first; - do { - if (hs->data[i] == NULL) { - return false; - } else if (hs->data[i] == ptr) { - hs->count--; - hs->data[i] = NL_HASHSET_TOMB; - return true; - } - - i = (i + 1) & mask; - } while (i != first); - - abort(); -} - -void nl_hashset_remove2(NL_HashSet* restrict hs, void* ptr, NL_HashFunc hash, NL_CompareFunc cmp) { - uint32_t h = hash(ptr); - - size_t mask = (1 << hs->exp) - 1; - size_t first = h & mask, i = first; - - do { - if (hs->data[i] == NULL) { - break; - } else if (hs->data[i] == ptr) { - hs->count--; - hs->data[i] = NL_HASHSET_TOMB; - break; - } - - i = (i + 1) & mask; - } while (i != first); -} - -void* nl_hashset_get2(NL_HashSet* restrict hs, void* ptr, NL_HashFunc hash, NL_CompareFunc cmp) { - uint32_t h = hash(ptr); - - size_t mask = (1 << hs->exp) - 1; - size_t first = h & mask, i = first; - - do { - if (hs->data[i] == NULL) { - return NULL; - } else if (hs->data[i] == NL_HASHSET_TOMB) { - // go past it - } else if (hs->data[i] == ptr || cmp(hs->data[i], ptr)) { - return hs->data[i]; - } - - i = (i + 1) & mask; - } while (i != first); - - return NULL; -} - -// returns old value -void* nl_hashset_put2(NL_HashSet* restrict hs, void* ptr, NL_HashFunc hash, NL_CompareFunc cmp) { - uint32_t h = hash(ptr); - - uint32_t threshold = ((1 << hs->exp) * 3) / 4; - if (hs->count >= threshold) { - NL_HashSet new_hs = nl_hashset_alloc(nl_hashset_capacity(hs) + 16); - nl_hashset_for(p, hs) { - nl_hashset_put2(&new_hs, *p, hash, cmp); - } - nl_hashset_free(*hs); - *hs = new_hs; - } - - size_t mask = (1 << hs->exp) - 1; - size_t first = h & mask, i = first; - - do { - if (hs->data[i] == NULL) { - hs->count++; - hs->data[i] = ptr; - return NULL; - } else if (hs->data[i] == NL_HASHSET_TOMB) { - // go past it - } else if (hs->data[i] == ptr || cmp(hs->data[i], ptr)) { - return hs->data[i]; - } - - i = (i + 1) & mask; - } while (i != first); - - NL_HashSet new_hs = nl_hashset_alloc_exp(hs->exp + 2); - nl_hashset_for(p, hs) { - nl_hashset_put2(&new_hs, *p, hash, cmp); - } - nl_hashset_free(*hs); - *hs = new_hs; - return NULL; -} - -void nl_hashset_clear(NL_HashSet* restrict hs) { - memset(hs->data, 0, nl_hashset_capacity(hs) * sizeof(void*)); - hs->count = 0; -} - -//////////////////////////////// -// Hashmap -//////////////////////////////// -NL_Table nl_table_alloc(size_t cap) { - cap = (cap * 4) / 3; - if (cap < 4) cap = 4; - - // next power of two - #if defined(_MSC_VER) && !defined(__clang__) - size_t exp = 64 - _lzcnt_u64(cap - 1); - #else - size_t exp = 64 - __builtin_clzll(cap - 1); - #endif - - return (NL_Table){ .exp = exp, .data = cuik_calloc(1u << exp, sizeof(NL_TableEntry)) }; -} - -NL_Table nl_table_arena_alloc(TB_Arena* arena, size_t cap) { - cap = (cap * 4) / 3; - if (cap < 4) cap = 4; - - // next power of two - #if defined(_MSC_VER) && !defined(__clang__) - size_t exp = 64 - _lzcnt_u64(cap - 1); - #else - size_t exp = 64 - __builtin_clzll(cap - 1); - #endif - - cap = 1ull << exp; - - void* data = tb_arena_alloc(arena, cap * sizeof(NL_TableEntry)); - memset(data, 0, cap * sizeof(NL_TableEntry)); - return (NL_Table){ .exp = exp, .data = data }; -} - -void nl_table_free(NL_Table tbl) { - if (tbl.allocator == NULL) { - cuik_free(tbl.data); - } else { - tb_arena_pop(tbl.allocator, tbl.data, (1ull << tbl.exp) * sizeof(NL_TableEntry)); - } -} - -void* nl_table_put(NL_Table* restrict tbl, void* k, void* v) { - uint32_t post_load_factor = ((1ull << tbl->exp) * 3) / 4; - if (tbl->count >= post_load_factor) { - // rehash - assert(tbl->allocator == NULL && "arena hashsets can't be resized!"); - NL_Table new_tbl = nl_table_alloc(nl_table_capacity(tbl)); - nl_table_for(p, tbl) { - nl_table_put(&new_tbl, p->k, p->v); - } - nl_table_free(*tbl); - *tbl = new_tbl; - } - - uint32_t h = NL_HASHSET_HASH(k); - size_t mask = (1 << tbl->exp) - 1; - size_t first = h & mask, i = first; - - do { - if (tbl->data[i].k == NULL) { - // insert - tbl->count++; - tbl->data[i].k = k; - tbl->data[i].v = v; - return NULL; - } else if (tbl->data[i].k == NL_HASHSET_TOMB) { - // recycle tombstone - tbl->data[i].k = k; - tbl->data[i].v = v; - return NULL; - } else if (tbl->data[i].k == k) { - void* old = tbl->data[i].v; - tbl->data[i].v = v; - return old; - } - - i = (i + 1) & mask; - } while (i != first); - - abort(); -} - -void* nl_table_get(NL_Table* restrict tbl, void* k) { - uint32_t h = NL_HASHSET_HASH(k); - size_t mask = (1 << tbl->exp) - 1; - size_t first = h & mask, i = first; - - do { - if (tbl->data[i].k == NULL) { - return NULL; - } else if (tbl->data[i].k == k) { - return tbl->data[i].v; - } - - i = (i + 1) & mask; - } while (i != first); - - return NULL; -} - -#endif /* NL_HASH_SET_IMPL */ diff --git a/vendor/common/perf.c b/vendor/common/perf.c deleted file mode 100644 index e8ecadb1..00000000 --- a/vendor/common/perf.c +++ /dev/null @@ -1,226 +0,0 @@ -#include -#include -#include -#include -#include - -#ifdef CUIK_USE_SPALL_AUTO -#define SPALL_BUFFER_PROFILING -#define SPALL_BUFFER_PROFILING_GET_TIME() __rdtsc() -#define SPALL_AUTO_IMPLEMENTATION -#include "spall_native_auto.h" -#else -#define SPALL_BUFFER_PROFILING -#define SPALL_BUFFER_PROFILING_GET_TIME() cuik_time_in_nanos() -#include "spall.h" -#endif - -#if defined(_AMD64_) || defined(__amd64__) -static double rdtsc_freq; -static uint64_t timer_start; -#endif - -#ifdef _WIN32 -#define WIN32_LEAN_AND_MEAN -#include -#else -#include -#include -#include -#endif - -static _Atomic bool profiling; -static SpallProfile ctx; -static _Thread_local SpallBuffer muh_buffer; - -#ifdef CUIK__IS_X64 -#ifdef _WIN32 -static uint64_t get_rdtsc_freq(void) { - // Get time before sleep - uint64_t qpc_begin = 0; QueryPerformanceCounter((LARGE_INTEGER *)&qpc_begin); - uint64_t tsc_begin = __rdtsc(); - - Sleep(2); - - // Get time after sleep - uint64_t qpc_end = qpc_begin + 1; QueryPerformanceCounter((LARGE_INTEGER *)&qpc_end); - uint64_t tsc_end = __rdtsc(); - - // Do the math to extrapolate the RDTSC ticks elapsed in 1 second - uint64_t qpc_freq = 0; QueryPerformanceFrequency((LARGE_INTEGER *)&qpc_freq); - uint64_t tsc_freq = (tsc_end - tsc_begin) * qpc_freq / (qpc_end - qpc_begin); - - // Failure case - if (!tsc_freq) { - tsc_freq = 1000000000; - } - - return tsc_freq; -} -#else -#include -#include -#include -#include -#include - -static uint64_t get_rdtsc_freq(void) { - // Fast path: Load kernel-mapped memory page - struct perf_event_attr pe = {0}; - pe.type = PERF_TYPE_HARDWARE; - pe.size = sizeof(struct perf_event_attr); - pe.config = PERF_COUNT_HW_INSTRUCTIONS; - pe.disabled = 1; - pe.exclude_kernel = 1; - pe.exclude_hv = 1; - - uint64_t tsc_freq = 0; - - // __NR_perf_event_open == 298 (on x86_64) - int fd = syscall(298, &pe, 0, -1, -1, 0); - if (fd != -1) { - - struct perf_event_mmap_page *pc = (struct perf_event_mmap_page *)mmap(NULL, 4096, PROT_READ, MAP_SHARED, fd, 0); - if (pc) { - - // success - if (pc->cap_user_time == 1) { - // docs say nanoseconds = (tsc * time_mult) >> time_shift - // set nanoseconds = 1000000000 = 1 second in nanoseconds, solve for tsc - // => tsc = 1000000000 / (time_mult >> time_shift) - tsc_freq = (1000000000ull << (pc->time_shift / 2)) / (pc->time_mult >> (pc->time_shift - pc->time_shift / 2)); - // If your build configuration supports 128 bit arithmetic, do this: - // tsc_freq = ((__uint128_t)1000000000ull << (__uint128_t)pc->time_shift) / pc->time_mult; - } - munmap(pc, 4096); - } - close(fd); - } - - // Slow path - if (!tsc_freq) { - - // Get time before sleep - uint64_t nsc_begin = 0; { struct timespec t; if (!clock_gettime(CLOCK_MONOTONIC_RAW, &t)) nsc_begin = (uint64_t)t.tv_sec * 1000000000ull + t.tv_nsec; } - uint64_t tsc_begin = __rdtsc(); - - usleep(10000); // 10ms gives ~4.5 digits of precision - the longer you sleep, the more precise you get - - // Get time after sleep - uint64_t nsc_end = nsc_begin + 1; { struct timespec t; if (!clock_gettime(CLOCK_MONOTONIC_RAW, &t)) nsc_end = (uint64_t)t.tv_sec * 1000000000ull + t.tv_nsec; } - uint64_t tsc_end = __rdtsc(); - - // Do the math to extrapolate the RDTSC ticks elapsed in 1 second - tsc_freq = (tsc_end - tsc_begin) * 1000000000 / (nsc_end - nsc_begin); - } - - // Failure case - if (!tsc_freq) { - tsc_freq = 1000000000; - } - - return tsc_freq; -} -#endif -#endif - -void cuik_init_timer_system(void) { - #if defined(_AMD64_) || defined(__amd64__) - rdtsc_freq = 1000000000.0 / get_rdtsc_freq(); - timer_start = __rdtsc(); - #endif -} - -void cuikperf_start(const char* path) { - #ifndef CUIK_USE_SPALL_AUTO - profiling = true; - ctx = spall_init_file(path, 1.0 / 1000.0); - cuikperf_thread_start(); - #endif -} - -void cuikperf_stop(void) { - #ifndef CUIK_USE_SPALL_AUTO - cuikperf_thread_stop(); - spall_quit(&ctx); - profiling = false; - #endif -} - -bool cuikperf_is_active(void) { - return profiling; -} - -uint64_t cuik_time_in_nanos(void) { - #if defined(_AMD64_) || defined(__amd64__) - return (__rdtsc() - timer_start) * rdtsc_freq; - #else - struct timespec ts; - clock_gettime(CLOCK_MONOTONIC, &ts); - return ((long long)ts.tv_sec * 1000000000LL) + ts.tv_nsec; - #endif -} - -void cuikperf_thread_start(void) { - #if _WIN32 - uint32_t tid = GetCurrentThreadId(); - #else - uint32_t tid = pthread_self(); - #endif - - if (profiling) { - #ifdef CUIK_USE_SPALL_AUTO - spall_auto_thread_init(tid, SPALL_DEFAULT_BUFFER_SIZE); - #else - if (cuikperf_is_active()) { - size_t size = 4 * 1024 * 1024; - muh_buffer = (SpallBuffer){ cuik_malloc(size), size }; - spall_buffer_init(&ctx, &muh_buffer); - } - #endif - } -} - -void cuikperf_thread_stop(void) { - if (profiling) { - #ifdef CUIK_USE_SPALL_AUTO - spall_auto_thread_quit(); - #else - if (cuikperf_is_active()) { - spall_buffer_quit(&ctx, &muh_buffer); - } - #endif - } -} - -void cuikperf_region_start(const char* label, const char* extra) { - if (profiling) { - uint64_t nanos = cuik_time_in_nanos(); - - #ifndef CUIK_USE_SPALL_AUTO - #if _WIN32 - uint32_t tid = GetCurrentThreadId(); - #else - uint32_t tid = pthread_self(); - #endif - - spall_buffer_begin_args(&ctx, &muh_buffer, label, strlen(label), extra, extra ? strlen(extra) : 0, nanos, tid, 0); - #endif - } -} - -void cuikperf_region_end(void) { - if (profiling) { - uint64_t nanos = cuik_time_in_nanos(); - - #ifndef CUIK_USE_SPALL_AUTO - #if _WIN32 - uint32_t tid = GetCurrentThreadId(); - #else - uint32_t tid = pthread_self(); - #endif - - spall_buffer_end_ex(&ctx, &muh_buffer, nanos, tid, 0); - #endif - } -} diff --git a/vendor/common/perf.h b/vendor/common/perf.h deleted file mode 100644 index d428bebd..00000000 --- a/vendor/common/perf.h +++ /dev/null @@ -1,36 +0,0 @@ -//////////////////////////////////////////// -// Profiler -//////////////////////////////////////////// -// The callbacks are global and a user may even hook in using the cuikperf_start -// and cuikperf_stop, or by using CUIK_TIMED_BLOCK -#pragma once - -#include -#include -#include -#include - -// DONT USE THIS :((( -void cuik_init_timer_system(void); - -void cuikperf_start(const char* path); -void cuikperf_stop(void); -bool cuikperf_is_active(void); - -// the absolute values here don't have to mean anything, it's just about being able -// to measure between two points. -uint64_t cuik_time_in_nanos(void); - -void cuikperf_thread_start(void); -void cuikperf_thread_stop(void); - -// Reports a region of time to the profiler callback -void cuikperf_region_start(const char* fmt, const char* extra); -void cuikperf_region_end(void); - -// Usage: -// CUIK_TIMED_BLOCK("Beans %d", 5) { -// ... -// } -#define CUIK_TIMED_BLOCK(label) for (uint64_t __i = (cuikperf_region_start(label, NULL), 0); __i < 1; __i++, cuikperf_region_end()) -#define CUIK_TIMED_BLOCK_ARGS(label, extra) for (uint64_t __i = (cuikperf_region_start(label, extra), 0); __i < 1; __i++, cuikperf_region_end()) diff --git a/vendor/common/spall.h b/vendor/common/spall.h deleted file mode 100644 index e47b231c..00000000 --- a/vendor/common/spall.h +++ /dev/null @@ -1,454 +0,0 @@ -// SPDX-FileCopyrightText: © 2022 Phillip Trudeau-Tavara -// SPDX-License-Identifier: 0BSD - -/* - -TODO: Optional Helper APIs: - - - Compression API: would require a mutexed lockable context (yuck...) - - Either using a ZIP library, a name cache + TIDPID cache, or both (but ZIP is likely more than enough!!!) - - begin()/end() writes compressed chunks to a caller-determined destination - - The destination can be the buffered-writing API or a custom user destination - - Ultimately need to take a lock with some granularity... can that be the caller's responsibility? - - - Counter Event: should allow tracking arbitrary named values with a single event, for memory and frame profiling - - - Ring-buffer API - spall_ring_init - spall_ring_emit_begin - spall_ring_emit_end - spall_ring_flush -*/ - -#ifndef SPALL_H -#define SPALL_H - -#if !defined(_MSC_VER) || defined(__clang__) -#define SPALL_NOINSTRUMENT __attribute__((no_instrument_function)) -#define SPALL_FORCEINLINE __attribute__((always_inline)) -#else -#define _CRT_SECURE_NO_WARNINGS -#define SPALL_NOINSTRUMENT // Can't noinstrument on MSVC! -#define SPALL_FORCEINLINE __forceinline -#endif - -#include -#include -#include -#include - -#define SPALL_FN static inline SPALL_NOINSTRUMENT - -#define SPALL_MIN(a, b) (((a) < (b)) ? (a) : (b)) - -#pragma pack(push, 1) - -typedef struct SpallHeader { - uint64_t magic_header; // = 0x0BADF00D - uint64_t version; // = 1 - double timestamp_unit; - uint64_t must_be_0; -} SpallHeader; - -enum { - SpallEventType_Invalid = 0, - SpallEventType_Custom_Data = 1, // Basic readers can skip this. - SpallEventType_StreamOver = 2, - - SpallEventType_Begin = 3, - SpallEventType_End = 4, - SpallEventType_Instant = 5, - - SpallEventType_Overwrite_Timestamp = 6, // Retroactively change timestamp units - useful for incrementally improving RDTSC frequency. - SpallEventType_Pad_Skip = 7, -}; - -typedef struct SpallBeginEvent { - uint8_t type; // = SpallEventType_Begin - uint8_t category; - - uint32_t pid; - uint32_t tid; - double when; - - uint8_t name_length; - uint8_t args_length; -} SpallBeginEvent; - -typedef struct SpallBeginEventMax { - SpallBeginEvent event; - char name_bytes[255]; - char args_bytes[255]; -} SpallBeginEventMax; - -typedef struct SpallEndEvent { - uint8_t type; // = SpallEventType_End - uint32_t pid; - uint32_t tid; - double when; -} SpallEndEvent; - -typedef struct SpallPadSkipEvent { - uint8_t type; // = SpallEventType_Pad_Skip - uint32_t size; -} SpallPadSkipEvent; - -#pragma pack(pop) - -typedef struct SpallProfile SpallProfile; - -// Important!: If you define your own callbacks, mark them SPALL_NOINSTRUMENT! -typedef bool (*SpallWriteCallback)(SpallProfile *self, const void *data, size_t length); -typedef bool (*SpallFlushCallback)(SpallProfile *self); -typedef void (*SpallCloseCallback)(SpallProfile *self); - -struct SpallProfile { - double timestamp_unit; - bool is_json; - SpallWriteCallback write; - SpallFlushCallback flush; - SpallCloseCallback close; - void *data; -}; - -// Important!: If you are writing Begin/End events, then do NOT write -// events for the same PID + TID pair on different buffers!!! -typedef struct SpallBuffer { - void *data; - size_t length; - - // Internal data - don't assign this - size_t head; - SpallProfile *ctx; -} SpallBuffer; - -#ifdef __cplusplus -extern "C" { -#endif - -#if defined(SPALL_BUFFER_PROFILING) && !defined(SPALL_BUFFER_PROFILING_GET_TIME) -#error "You must #define SPALL_BUFFER_PROFILING_GET_TIME() to profile buffer flushes." -#endif - -SPALL_FN SPALL_FORCEINLINE void spall__buffer_profile(SpallProfile *ctx, SpallBuffer *wb, double spall_time_begin, double spall_time_end, const char *name, int name_len); -#ifdef SPALL_BUFFER_PROFILING -#define SPALL_BUFFER_PROFILE_BEGIN() double spall_time_begin = (SPALL_BUFFER_PROFILING_GET_TIME()) -// Don't call this with anything other than a string literal -#define SPALL_BUFFER_PROFILE_END(name) spall__buffer_profile(ctx, wb, spall_time_begin, (SPALL_BUFFER_PROFILING_GET_TIME()), "" name "", sizeof("" name "") - 1) -#else -#define SPALL_BUFFER_PROFILE_BEGIN() -#define SPALL_BUFFER_PROFILE_END(name) -#endif - -SPALL_FN SPALL_FORCEINLINE bool spall__file_write(SpallProfile *ctx, const void *p, size_t n) { - if (!ctx->data) return false; -#ifdef SPALL_DEBUG - if (feof((FILE *)ctx->data)) return false; - if (ferror((FILE *)ctx->data)) return false; -#endif - - if (fwrite(p, n, 1, (FILE *)ctx->data) != 1) return false; - return true; -} -SPALL_FN bool spall__file_flush(SpallProfile *ctx) { - if (!ctx->data) return false; - if (fflush((FILE *)ctx->data)) return false; - return true; -} -SPALL_FN void spall__file_close(SpallProfile *ctx) { - if (!ctx->data) return; - - if (ctx->is_json) { -#ifdef SPALL_DEBUG - if (!feof((FILE *)ctx->data) && !ferror((FILE *)ctx->data)) -#endif - { - fseek((FILE *)ctx->data, -2, SEEK_CUR); // seek back to overwrite trailing comma - fwrite("\n]}\n", sizeof("\n]}\n") - 1, 1, (FILE *)ctx->data); - } - } - fflush((FILE *)ctx->data); - fclose((FILE *)ctx->data); - ctx->data = NULL; -} - -SPALL_FN SPALL_FORCEINLINE bool spall__buffer_flush(SpallProfile *ctx, SpallBuffer *wb) { - // precon: wb - // precon: wb->data - // precon: wb->head <= wb->length - // precon: !ctx || ctx->write -#ifdef SPALL_DEBUG - if (wb->ctx != ctx) return false; // Buffer must be bound to this context (or to NULL) -#endif - - if (wb->head && ctx) { - SPALL_BUFFER_PROFILE_BEGIN(); - if (!ctx->write) return false; - if (ctx->write == spall__file_write) { - if (!spall__file_write(ctx, wb->data, wb->head)) return false; - } else { - if (!ctx->write(ctx, wb->data, wb->head)) return false; - } - SPALL_BUFFER_PROFILE_END("Buffer Flush"); - } - wb->head = 0; - return true; -} - -SPALL_FN SPALL_FORCEINLINE bool spall__buffer_write(SpallProfile *ctx, SpallBuffer *wb, void *p, size_t n) { - // precon: !wb || wb->head < wb->length - // precon: !ctx || ctx->write - if (!wb) return ctx->write && ctx->write(ctx, p, n); -#ifdef SPALL_DEBUG - if (wb->ctx != ctx) return false; // Buffer must be bound to this context (or to NULL) -#endif - if (wb->head + n > wb->length && !spall__buffer_flush(ctx, wb)) return false; - if (n > wb->length) { - SPALL_BUFFER_PROFILE_BEGIN(); - if (!ctx->write || !ctx->write(ctx, p, n)) return false; - SPALL_BUFFER_PROFILE_END("Unbuffered Write"); - return true; - } - memcpy((char *)wb->data + wb->head, p, n); - wb->head += n; - return true; -} - -SPALL_FN bool spall_buffer_flush(SpallProfile *ctx, SpallBuffer *wb) { -#ifdef SPALL_DEBUG - if (!wb) return false; - if (!wb->data) return false; -#endif - - if (!spall__buffer_flush(ctx, wb)) return false; - return true; -} - -SPALL_FN bool spall_buffer_init(SpallProfile *ctx, SpallBuffer *wb) { - if (!spall_buffer_flush(NULL, wb)) return false; - wb->ctx = ctx; - return true; -} -SPALL_FN bool spall_buffer_quit(SpallProfile *ctx, SpallBuffer *wb) { - if (!spall_buffer_flush(ctx, wb)) return false; - wb->ctx = NULL; - return true; -} - -SPALL_FN bool spall_buffer_abort(SpallBuffer *wb) { - if (!wb) return false; - wb->ctx = NULL; - if (!spall__buffer_flush(NULL, wb)) return false; - return true; -} - -SPALL_FN size_t spall_build_header(void *buffer, size_t rem_size, double timestamp_unit) { - size_t header_size = sizeof(SpallHeader); - if (header_size > rem_size) { - return 0; - } - - SpallHeader *header = (SpallHeader *)buffer; - header->magic_header = 0x0BADF00D; - header->version = 1; - header->timestamp_unit = timestamp_unit; - header->must_be_0 = 0; - return header_size; -} -SPALL_FN SPALL_FORCEINLINE size_t spall_build_begin(void *buffer, size_t rem_size, const char *name, signed long name_len, const char *args, signed long args_len, double when, uint32_t tid, uint32_t pid) { - SpallBeginEventMax *ev = (SpallBeginEventMax *)buffer; - uint8_t trunc_name_len = (uint8_t)SPALL_MIN(name_len, 255); // will be interpreted as truncated in the app (?) - uint8_t trunc_args_len = (uint8_t)SPALL_MIN(args_len, 255); // will be interpreted as truncated in the app (?) - - size_t ev_size = sizeof(SpallBeginEvent) + trunc_name_len + trunc_args_len; - if (ev_size > rem_size) { - return 0; - } - - ev->event.type = SpallEventType_Begin; - ev->event.category = 0; - ev->event.pid = pid; - ev->event.tid = tid; - ev->event.when = when; - ev->event.name_length = trunc_name_len; - ev->event.args_length = trunc_args_len; - memcpy(ev->name_bytes, name, trunc_name_len); - memcpy(ev->name_bytes + name_len, args, trunc_args_len); - - return ev_size; -} -SPALL_FN SPALL_FORCEINLINE size_t spall_build_end(void *buffer, size_t rem_size, double when, uint32_t tid, uint32_t pid) { - size_t ev_size = sizeof(SpallEndEvent); - if (ev_size > rem_size) { - return 0; - } - - SpallEndEvent *ev = (SpallEndEvent *)buffer; - ev->type = SpallEventType_End; - ev->pid = pid; - ev->tid = tid; - ev->when = when; - - return ev_size; -} - -SPALL_FN void spall_quit(SpallProfile *ctx) { - if (!ctx) return; - if (ctx->close) ctx->close(ctx); - - memset(ctx, 0, sizeof(*ctx)); -} - -SPALL_FN SpallProfile spall_init_callbacks(double timestamp_unit, - SpallWriteCallback write, - SpallFlushCallback flush, - SpallCloseCallback close, - void *userdata, - bool is_json) { - SpallProfile ctx; - memset(&ctx, 0, sizeof(ctx)); - if (timestamp_unit < 0) return ctx; - ctx.timestamp_unit = timestamp_unit; - ctx.is_json = is_json; - ctx.data = userdata; - ctx.write = write; - ctx.flush = flush; - ctx.close = close; - - if (ctx.is_json) { - if (!ctx.write(&ctx, "{\"traceEvents\":[\n", sizeof("{\"traceEvents\":[\n") - 1)) { spall_quit(&ctx); return ctx; } - } else { - SpallHeader header; - size_t len = spall_build_header(&header, sizeof(header), timestamp_unit); - if (!ctx.write(&ctx, &header, len)) { spall_quit(&ctx); return ctx; } - } - - return ctx; -} - -SPALL_FN SpallProfile spall_init_file_ex(const char *filename, double timestamp_unit, bool is_json) { - SpallProfile ctx; - memset(&ctx, 0, sizeof(ctx)); - if (!filename) return ctx; - ctx.data = fopen(filename, "wb"); // TODO: handle utf8 and long paths on windows - if (ctx.data) { // basically freopen() but we don't want to force users to lug along another macro define - fclose((FILE *)ctx.data); - ctx.data = fopen(filename, "ab"); - } - if (!ctx.data) { spall_quit(&ctx); return ctx; } - ctx = spall_init_callbacks(timestamp_unit, spall__file_write, spall__file_flush, spall__file_close, ctx.data, is_json); - return ctx; -} - -SPALL_FN SpallProfile spall_init_file (const char* filename, double timestamp_unit) { return spall_init_file_ex(filename, timestamp_unit, false); } -SPALL_FN SpallProfile spall_init_file_json(const char* filename, double timestamp_unit) { return spall_init_file_ex(filename, timestamp_unit, true); } - -SPALL_FN bool spall_flush(SpallProfile *ctx) { -#ifdef SPALL_DEBUG - if (!ctx) return false; -#endif - - if (!ctx->flush || !ctx->flush(ctx)) return false; - return true; -} - -SPALL_FN SPALL_FORCEINLINE bool spall_buffer_begin_args(SpallProfile *ctx, SpallBuffer *wb, const char *name, signed long name_len, const char *args, signed long args_len, double when, uint32_t tid, uint32_t pid) { -#ifdef SPALL_DEBUG - if (!ctx) return false; - if (!name) return false; - if (name_len <= 0) return false; - if (!wb) return false; -#endif - - if (ctx->is_json) { - char buf[1024]; - int buf_len = snprintf(buf, sizeof(buf), - "{\"ph\":\"B\",\"ts\":%f,\"pid\":%u,\"tid\":%u,\"name\":\"%.*s\",\"args\":\"%.*s\"},\n", - when * ctx->timestamp_unit, pid, tid, (int)(uint8_t)name_len, name, (int)(uint8_t)args_len, args); - if (buf_len <= 0) return false; - if (buf_len >= sizeof(buf)) return false; - if (!spall__buffer_write(ctx, wb, buf, buf_len)) return false; - } else { - if ((wb->head + sizeof(SpallBeginEventMax)) > wb->length) { - if (!spall__buffer_flush(ctx, wb)) { - return false; - } - } - - wb->head += spall_build_begin((char *)wb->data + wb->head, wb->length - wb->head, name, name_len, args, args_len, when, tid, pid); - } - - return true; -} - -SPALL_FN bool spall_buffer_begin_ex(SpallProfile *ctx, SpallBuffer *wb, const char *name, signed long name_len, double when, uint32_t tid, uint32_t pid) { - return spall_buffer_begin_args(ctx, wb, name, name_len, "", 0, when, tid, pid); -} - -SPALL_FN bool spall_buffer_begin(SpallProfile *ctx, SpallBuffer *wb, const char *name, signed long name_len, double when) { - return spall_buffer_begin_args(ctx, wb, name, name_len, "", 0, when, 0, 0); -} - -SPALL_FN bool spall_buffer_end_ex(SpallProfile *ctx, SpallBuffer *wb, double when, uint32_t tid, uint32_t pid) { -#ifdef SPALL_DEBUG - if (!ctx) return false; - if (!wb) return false; -#endif - - if (ctx->is_json) { - char buf[512]; - int buf_len = snprintf(buf, sizeof(buf), - "{\"ph\":\"E\",\"ts\":%f,\"pid\":%u,\"tid\":%u},\n", - when * ctx->timestamp_unit, pid, tid); - if (buf_len <= 0) return false; - if (buf_len >= sizeof(buf)) return false; - if (!spall__buffer_write(ctx, wb, buf, buf_len)) return false; - } else { - if ((wb->head + sizeof(SpallEndEvent)) > wb->length) { - if (!spall__buffer_flush(ctx, wb)) { - return false; - } - } - - wb->head += spall_build_end((char *)wb->data + wb->head, wb->length - wb->head, when, tid, pid); - } - - return true; -} - -SPALL_FN bool spall_buffer_end(SpallProfile *ctx, SpallBuffer *wb, double when) { return spall_buffer_end_ex(ctx, wb, when, 0, 0); } - -SPALL_FN SPALL_FORCEINLINE void spall__buffer_profile(SpallProfile *ctx, SpallBuffer *wb, double spall_time_begin, double spall_time_end, const char *name, int name_len) { - // precon: ctx - // precon: ctx->write - char temp_buffer_data[2048]; - SpallBuffer temp_buffer = { temp_buffer_data, sizeof(temp_buffer_data) }; - if (!spall_buffer_begin_ex(ctx, &temp_buffer, name, name_len, spall_time_begin, (uint32_t)(uintptr_t)wb->data, 4222222222)) return; - if (!spall_buffer_end_ex(ctx, &temp_buffer, spall_time_end, (uint32_t)(uintptr_t)wb->data, 4222222222)) return; - if (ctx->write) ctx->write(ctx, temp_buffer_data, temp_buffer.head); -} - -#ifdef __cplusplus -} -#endif - -#endif // SPALL_H - -/* -Zero-Clause BSD (0BSD) - -Copyright (c) 2022, Phillip Trudeau-Tavara -All rights reserved. - -Permission to use, copy, modify, and/or distribute this software -for any purpose with or without fee is hereby granted. - -THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL -WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED -WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL -THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR -CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM -LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, -NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION -WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. -*/ diff --git a/vendor/common/spall_native_auto.h b/vendor/common/spall_native_auto.h deleted file mode 100644 index 8e9b10f9..00000000 --- a/vendor/common/spall_native_auto.h +++ /dev/null @@ -1,803 +0,0 @@ -#ifndef SPALL_AUTO_H -#define SPALL_AUTO_H - -// THIS IS EXPERIMENTAL, BUT VERY HANDY -// *should* work on clang and gcc on Windows, Mac, and Linux - -#define SPALL_IS_WINDOWS 0 -#define SPALL_IS_DARWIN 0 -#define SPALL_IS_LINUX 0 -#define SPALL_IS_GCC 0 -#define SPALL_IS_CLANG 0 -#define SPALL_IS_CPP 0 -#define SPALL_IS_X64 0 -#define SPALL_IS_ARM64 0 - -#ifdef __cplusplus - #undef SPALL_IS_CPP - #define SPALL_IS_CPP 1 -#endif - -#if defined(__clang__) - #undef SPALL_IS_CLANG - #define SPALL_IS_CLANG 1 -#endif -#if defined(_WIN32) - #undef SPALL_IS_WINDOWS - #define SPALL_IS_WINDOWS 1 -#elif defined(__APPLE__) - #undef SPALL_IS_DARWIN - #define SPALL_IS_DARWIN 1 -#elif defined(__linux__) - #undef SPALL_IS_LINUX - #define SPALL_IS_LINUX 1 -#endif -#ifdef __GNUC__ - #undef SPALL_IS_GCC - #define SPALL_IS_GCC 1 -#endif -#if defined(__x86_64__) || defined(_M_AMD64) - #undef SPALL_IS_X64 - #define SPALL_IS_X64 1 -#elif defined(__aarch64__) - #undef SPALL_IS_ARM64 - #define SPALL_IS_ARM64 1 -#endif - -#if (!SPALL_IS_CLANG && !SPALL_IS_GCC) -#error "Compiler not supported!" -#endif - -#ifdef __cplusplus -extern "C" { -#endif - -#include -#include -#include - -bool spall_auto_init(char *filename); -void spall_auto_quit(void); -bool spall_auto_thread_init(uint32_t thread_id, size_t buffer_size); -void spall_auto_thread_quit(void); - -bool spall_auto_buffer_begin(const char *name, signed long name_len, const char *args, signed long args_len); -bool spall_auto_buffer_end(void); -bool spall_auto_buffer_flush(void); - -void spall_auto_set_thread_instrumenting(bool on); - -#if SPALL_IS_GCC && SPALL_IS_CPP - #define _Thread_local thread_local -#endif - -#define SPALL_DEFAULT_BUFFER_SIZE (32 * 1024 * 1024) -#define SPALL_MIN(a, b) (((a) < (b)) ? (a) : (b)) -#define SPALL_MAX(a, b) (((a) > (b)) ? (a) : (b)) - -#ifdef __cplusplus -} -#endif -#endif // endif SPALL_AUTO_H - -#ifdef SPALL_AUTO_IMPLEMENTATION -#ifndef SPALL_AUTO_IMPLEMENTED_H -#define SPALL_AUTO_IMPLEMENTED_H - -#if !SPALL_IS_WINDOWS - #if SPALL_IS_CPP - #include - #else - #include - #endif -#endif - -#ifdef __cplusplus -extern "C" { -#endif - -#include -#include -#include -#include - - -#if !SPALL_IS_WINDOWS - #include - #include - #include - #include -#endif - -#if SPALL_IS_WINDOWS - #include - #include - - typedef ptrdiff_t ssize_t; - typedef HANDLE Spall_ThreadHandle; - - #define spall_thread_start(t) ((t)->writer.thread = (HANDLE) _beginthread(spall_writer, 0, t)) - #define spall_thread_end(t) WaitForSingleObject((t)->writer.thread, INFINITE) -#else - typedef pthread_t Spall_ThreadHandle; - #define spall_thread_start(t) pthread_create(&(t)->writer.thread, NULL, spall_writer, (void *) (t)) - #define spall_thread_end(t) pthread_join((t)->writer.thread, NULL) -#endif - -#define SPALL_NOINSTRUMENT __attribute__((no_instrument_function)) -#define SPALL_FORCEINLINE __attribute__((always_inline)) -#define __debugbreak() __builtin_trap() - -#if SPALL_IS_CPP - #define Spall_Atomic(X) std::atomic -#else - #define Spall_Atomic(X) _Atomic (X) -#endif - -#define SPALL_FN static SPALL_NOINSTRUMENT - -#if SPALL_IS_X64 -#include -SPALL_FN uint64_t spall_get_clock(void) { - return __rdtsc(); -} -SPALL_FN void spall_pause(void) { - _mm_pause(); -} -#elif SPALL_IS_ARM64 -SPALL_FN uint64_t spall_get_clock(void) { - int64_t timer_val; - asm volatile("mrs %0, cntvct_el0" : "=r"(timer_val)); - return (uint64_t)timer_val; -} -SPALL_FN void spall_pause(void) { - asm volatile("yield"); -} -#endif - -#pragma pack(push, 1) - -typedef struct SpallHeader { - uint64_t magic_header; // = 0xABADF00D - uint64_t version; // = 2 - double timestamp_unit; - uint64_t known_address; // Address for spall_auto_init, for skew-correction - uint16_t program_path_len; -} SpallHeader; - -enum { - SpallAutoEventType_Invalid = 0, - SpallAutoEventType_Begin = 1, -}; - -typedef struct SpallMicroBeginEventMax { - uint8_t type; - uint64_t ts; - uint64_t caller; -} SpallMicroBeginEventMax; - -typedef struct SpallMicroEndEventMax { - uint8_t type; - uint64_t ts; -} SpallMicroEndEventMax; - -typedef struct SpallAutoBeginEvent { - uint8_t type; - uint64_t when; -} SpallAutoBeginEvent; - -typedef struct SpallBufferHeader { - uint32_t size; - uint32_t tid; - uint64_t first_ts; - uint32_t max_depth; -} SpallBufferHeader; - -#pragma pack(pop) - -uint8_t spall_squash_1[] = {1, 1, 2, 4, 4, 8, 8, 8, 8}; -uint8_t spall_squash_2[] = {0, 0, 1, 2, 2, 3, 3, 3, 3}; - -SPALL_FN SPALL_FORCEINLINE uint64_t spall_delta_to_size(uint64_t dt) { - dt = SPALL_MAX(1, dt); - uint64_t set_bits = 64 - __builtin_clzll(dt); - uint64_t set_bytes = (set_bits + 7) >> 3; - return spall_squash_1[set_bytes]; -} - -typedef struct SpallProfile { - double stamp_scale; - FILE *file; -} SpallProfile; - -typedef Spall_Atomic(uint64_t) Spall_Futex; -typedef struct SpallBuffer { - uint8_t *data; - size_t length; - - // if true, write to upper-half, else lower-half - size_t sub_length; - bool write_half; - - struct { - Spall_Atomic(bool) is_running; - Spall_ThreadHandle thread; - Spall_Atomic(uint64_t) ptr; - Spall_Atomic(size_t) size; - } writer; - - size_t head; - uint32_t thread_id; - - uint64_t previous_ts; - uint64_t first_ts; - - uint64_t previous_addr; - uint64_t previous_caller; - - uint32_t current_depth; - uint32_t max_depth; -} SpallBuffer; - - -// Cross-platform wrappers -#if SPALL_IS_LINUX -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -SPALL_FN bool get_program_path(char **out_path) { - char path[PATH_MAX] = {0}; - uint32_t size = sizeof(path); - - ssize_t buff_len = (ssize_t)readlink("/proc/self/exe", path, size - 1); - if (buff_len == -1) { - *out_path = NULL; - return false; - } - - char *post_path = (char *)calloc(PATH_MAX, 1); - if (realpath(path, post_path) == NULL) { - free(post_path); - *out_path = NULL; - return false; - } - - *out_path = post_path; - return true; -} - -SPALL_FN uint64_t mul_u64_u32_shr(uint64_t cyc, uint32_t mult, uint32_t shift) { - __uint128_t x = cyc; - x *= mult; - x >>= shift; - return (uint64_t)x; -} - -SPALL_FN long perf_event_open(struct perf_event_attr *hw_event, pid_t pid, int cpu, int group_fd, unsigned long flags) { - return syscall(__NR_perf_event_open, hw_event, pid, cpu, group_fd, flags); -} - -#if SPALL_IS_X64 -SPALL_FN double spall_get_clock_multiplier(void) { - struct perf_event_attr pe = { - .type = PERF_TYPE_HARDWARE, - .size = sizeof(struct perf_event_attr), - .config = PERF_COUNT_HW_INSTRUCTIONS, - .disabled = 1, - .exclude_kernel = 1, - .exclude_hv = 1 - }; - - int fd = (int)perf_event_open(&pe, 0, -1, -1, 0); - if (fd == -1) { - perror("perf_event_open failed"); - return 1; - } - void *addr = mmap(NULL, 4*1024, PROT_READ, MAP_SHARED, fd, 0); - if (!addr) { - perror("mmap failed"); - return 1; - } - struct perf_event_mmap_page *pc = (struct perf_event_mmap_page *)addr; - if (pc->cap_user_time != 1) { - fprintf(stderr, "Perf system doesn't support user time\n"); - return 1; - } - double nanos = (double)mul_u64_u32_shr(1000000000000000ull, pc->time_mult, pc->time_shift); - double multiplier = nanos / 1000000000000000.0; - return multiplier; -} -#endif - - -SPALL_FN SPALL_FORCEINLINE void spall_signal(Spall_Futex *addr) { - long ret = syscall(SYS_futex, addr, FUTEX_WAKE | FUTEX_PRIVATE_FLAG, 1, NULL, NULL, 0); - if (ret == -1) { - perror("Futex wake"); - __debugbreak(); - } -} - -SPALL_FN SPALL_FORCEINLINE void spall_wait(Spall_Futex *addr, uint64_t val) { - for (;;) { - long ret = syscall(SYS_futex, addr, FUTEX_WAIT | FUTEX_PRIVATE_FLAG, val, NULL, NULL, 0); - if (ret == -1) { - if (errno != EAGAIN) { - perror("Futex wait"); - __debugbreak(); - } else { - return; - } - } else if (ret == 0) { - return; - } - } -} - -#elif SPALL_IS_DARWIN - -#include -#include -#include - -#if SPALL_IS_X64 -SPALL_FN double spall_get_clock_multiplier(void) { - uint64_t freq; - size_t size = sizeof(freq); - - sysctlbyname("machdep.tsc.frequency", &freq, &size, NULL, 0); - return 1000000000.0 / (double)freq; -} -#elif SPALL_IS_ARM64 -SPALL_FN double spall_get_clock_multiplier(void) { - uint64_t freq_val; - asm volatile("mrs %0, cntfrq_el0" : "=r"(freq_val)); - - double multiplier = 1000000000.0 / (double)freq_val; - return multiplier; -} -#endif - -SPALL_FN bool get_program_path(char **out_path) { - char pre_path[1025]; - uint32_t size = sizeof(pre_path); - if (_NSGetExecutablePath(pre_path, &size) == -1) { - *out_path = NULL; - return false; - } - - char *post_path = (char *)malloc(1025); - if (realpath(pre_path, post_path) == NULL) { - free(post_path); - *out_path = NULL; - return false; - } - - *out_path = post_path; - return true; -} - -#define UL_COMPARE_AND_WAIT 0x00000001 -#define ULF_WAKE_ALL 0x00000100 -#define ULF_NO_ERRNO 0x01000000 - -/* timeout is specified in microseconds */ -int __ulock_wait(uint32_t operation, void *addr, uint64_t value, uint32_t timeout); -int __ulock_wake(uint32_t operation, void *addr, uint64_t wake_value); - -SPALL_FN SPALL_FORCEINLINE void spall_signal(Spall_Futex *addr) { - for (;;) { - int ret = __ulock_wake(UL_COMPARE_AND_WAIT | ULF_NO_ERRNO, addr, 0); - if (ret >= 0) { - return; - } - ret = -ret; - if (ret == EINTR || ret == EFAULT) { - continue; - } - if (ret == ENOENT) { - return; - } - printf("futex signal fail?\n"); - __debugbreak(); - } -} - -SPALL_FN SPALL_FORCEINLINE void spall_wait(Spall_Futex *addr, uint64_t val) { - for (;;) { - int ret = __ulock_wait(UL_COMPARE_AND_WAIT | ULF_NO_ERRNO, addr, val, 0); - if (ret >= 0) { - return; - } - ret = -ret; - if (ret == EINTR || ret == EFAULT) { - continue; - } - if (ret == ENOENT) { - return; - } - - printf("futex wait fail? %d\n", ret); - __debugbreak(); - } -} - -#elif SPALL_IS_WINDOWS - -SPALL_FN bool get_program_path(char **out_path) { - char *post_path = (char *)calloc(MAX_PATH, 1); - if (GetModuleFileNameA(NULL, post_path, MAX_PATH) == 0) { - *out_path = NULL; - return false; - } - - *out_path = post_path; - return true; -} - -SPALL_FN SPALL_FORCEINLINE double spall_get_clock_multiplier(void) { - - // Cache the answer so that multiple calls never take the slow path more than once - static double multiplier = 0; - if (multiplier) { - return multiplier; - } - - uint64_t tsc_freq = 0; - - // Get time before sleep - uint64_t qpc_begin = 0; QueryPerformanceCounter((LARGE_INTEGER *)&qpc_begin); - uint64_t tsc_begin = spall_get_clock(); - - Sleep(2); - - // Get time after sleep - uint64_t qpc_end = qpc_begin + 1; QueryPerformanceCounter((LARGE_INTEGER *)&qpc_end); - uint64_t tsc_end = spall_get_clock(); - - // Do the math to extrapolate the RDTSC ticks elapsed in 1 second - uint64_t qpc_freq = 0; QueryPerformanceFrequency((LARGE_INTEGER *)&qpc_freq); - tsc_freq = (tsc_end - tsc_begin) * qpc_freq / (qpc_end - qpc_begin); - - multiplier = 1000000000.0 / (double)tsc_freq; - return multiplier; -} - -SPALL_FN SPALL_FORCEINLINE void spall_signal(Spall_Futex *addr) { - WakeByAddressSingle((void *)addr); -} - -SPALL_FN SPALL_FORCEINLINE void spall_wait(Spall_Futex *addr, uint64_t val) { - WaitOnAddress(addr, (void *)&val, sizeof(val), INFINITE); -} - -#endif - -// Auto-tracing impl -static SpallProfile spall_ctx; -static _Thread_local SpallBuffer *spall_buffer = NULL; -static _Thread_local bool spall_thread_running = false; - -SPALL_NOINSTRUMENT void spall_auto_set_thread_instrumenting(bool on) { - spall_thread_running = on; -} - -#if SPALL_IS_WINDOWS -SPALL_FN void spall_writer(void *arg) { -#else -SPALL_FN void *spall_writer(void *arg) { -#endif - - SpallBuffer *buffer = (SpallBuffer *)arg; - while (buffer->writer.is_running) { - spall_wait(&buffer->writer.ptr, 0); - if (!buffer->writer.is_running) { break; } - if (buffer->writer.ptr == 0) { continue; } - - size_t size = buffer->writer.size; - void *buffer_ptr = (void *)atomic_load(&buffer->writer.ptr); - buffer->writer.ptr = 0; - - fwrite(buffer_ptr, size, 1, spall_ctx.file); - } - -#if !SPALL_IS_WINDOWS - return NULL; -#endif -} - -SPALL_FN SPALL_FORCEINLINE bool spall__file_write(void *p, size_t n) { - spall_buffer->writer.size = n; - atomic_store(&spall_buffer->writer.ptr, (uint64_t)p); - spall_signal(&spall_buffer->writer.ptr); - - while (spall_buffer->writer.ptr != 0) { spall_pause(); } - - return true; -} - -SPALL_NOINSTRUMENT SPALL_FORCEINLINE bool spall_auto_buffer_flush(void) { - if (!spall_buffer) return false; - - size_t data_start = spall_buffer->write_half ? spall_buffer->sub_length : 0; - - SpallBufferHeader *sbp = (SpallBufferHeader *)(spall_buffer->data + data_start); - if (spall_buffer->head > 0) { - sbp->size = (uint32_t)(spall_buffer->head - sizeof(SpallBufferHeader)); - sbp->first_ts = spall_buffer->first_ts; - sbp->max_depth = spall_buffer->max_depth; - if (!spall__file_write(spall_buffer->data + data_start, spall_buffer->head)) return false; - - spall_buffer->write_half = !spall_buffer->write_half; - } - - data_start = spall_buffer->write_half ? spall_buffer->sub_length : 0; - sbp = (SpallBufferHeader *)(spall_buffer->data + data_start); - sbp->size = 0; - sbp->first_ts = 0; - sbp->tid = spall_buffer->thread_id; - - spall_buffer->head = sizeof(SpallBufferHeader); - spall_buffer->first_ts = 0; - spall_buffer->previous_ts = 0; - spall_buffer->previous_addr = 0; - spall_buffer->previous_caller = 0; - return true; -} - -SPALL_FN SPALL_FORCEINLINE bool spall_buffer_micro_begin(uint64_t addr, uint64_t caller) { - spall_buffer->current_depth += 1; - spall_buffer->max_depth = SPALL_MAX(spall_buffer->max_depth, spall_buffer->current_depth); - - size_t ev_size = sizeof(SpallMicroBeginEventMax); - if ((spall_buffer->head + ev_size) > spall_buffer->sub_length) { - if (!spall_auto_buffer_flush()) { - return false; - } - } - - size_t data_start = spall_buffer->write_half ? spall_buffer->sub_length : 0; - uint8_t *ev_buffer = (spall_buffer->data + data_start) + spall_buffer->head; - - uint64_t now = spall_get_clock(); - if (spall_buffer->first_ts == 0) { - spall_buffer->first_ts = now; - spall_buffer->previous_ts = now; - } - - uint64_t dt = now - spall_buffer->previous_ts; - uint64_t d_addr = addr ^ spall_buffer->previous_addr; - uint64_t d_caller = caller ^ spall_buffer->previous_caller; - - uint64_t dt_size = spall_delta_to_size(dt); - uint64_t addr_size = spall_delta_to_size(d_addr); - uint64_t caller_size = spall_delta_to_size(d_caller); - - // [begin event tag | size of ts | size of addr | size of caller] - uint8_t type_byte = (0 << 6) | (spall_squash_2[dt_size] << 4) | (spall_squash_2[addr_size] << 2) | spall_squash_2[caller_size]; - - int i = 0; - *(ev_buffer + i) = type_byte; i += 1; - memcpy(ev_buffer + i, &dt, 8); i += dt_size; - memcpy(ev_buffer + i, &d_addr, 8); i += addr_size; - memcpy(ev_buffer + i, &d_caller, 8); i += caller_size; - - spall_buffer->previous_ts = now; - spall_buffer->previous_addr = addr; - spall_buffer->previous_caller = caller; - spall_buffer->head += i; - - return true; -} - -SPALL_FN SPALL_FORCEINLINE bool spall_buffer_micro_end(void) { - uint64_t now = spall_get_clock(); - spall_buffer->current_depth -= 1; - - size_t ev_size = sizeof(SpallMicroEndEventMax); - if ((spall_buffer->head + ev_size) > spall_buffer->sub_length) { - if (!spall_auto_buffer_flush()) { - return false; - } - } - if (spall_buffer->first_ts == 0) { - spall_buffer->first_ts = now; - spall_buffer->previous_ts = now; - } - - size_t data_start = spall_buffer->write_half ? spall_buffer->sub_length : 0; - uint8_t *ev_buffer = (spall_buffer->data + data_start) + spall_buffer->head; - - uint64_t dt = now - spall_buffer->previous_ts; - uint64_t dt_size = spall_delta_to_size(dt); - - // [end event tag | size of ts] - uint8_t type_byte = (1 << 6) | (spall_squash_2[dt_size] << 4); - - int i = 0; - *(ev_buffer + i) = type_byte; i += 1; - memcpy(ev_buffer + i, &dt, 8); i += dt_size; - - spall_buffer->previous_ts = now; - spall_buffer->head += i; - return true; -} - -SPALL_NOINSTRUMENT SPALL_FORCEINLINE bool spall_auto_buffer_begin(const char *name, signed long name_len, const char *args, signed long args_len) { - - spall_buffer->current_depth += 1; - spall_buffer->max_depth = SPALL_MAX(spall_buffer->max_depth, spall_buffer->current_depth); - - uint16_t trunc_name_len = (uint16_t)SPALL_MIN(name_len, UINT16_MAX); - uint16_t trunc_args_len = (uint16_t)SPALL_MIN(args_len, UINT16_MAX); - uint64_t name_len_size = (trunc_name_len > 255) ? 2 : 1; - uint64_t args_len_size = (trunc_args_len > 255) ? 2 : 1; - - uint64_t event_tail = trunc_name_len + name_len_size + trunc_args_len + args_len_size; - if ((spall_buffer->head + sizeof(SpallAutoBeginEvent) + event_tail) > spall_buffer->sub_length) { - if (!spall_auto_buffer_flush()) { - return false; - } - } - - size_t data_start = spall_buffer->write_half ? spall_buffer->sub_length : 0; - uint8_t *ev_buffer = (spall_buffer->data + data_start) + spall_buffer->head; - - uint64_t now = spall_get_clock(); - if (spall_buffer->first_ts == 0) { - spall_buffer->first_ts = now; - spall_buffer->previous_ts = now; - } - uint64_t dt = now - spall_buffer->previous_ts; - uint64_t dt_size = spall_delta_to_size(dt); - - // [extended tag | begin type | delta size | field lengths] - uint8_t name_args_lens = ((name_len_size >> 1) << 1) | (args_len_size >> 1); - uint8_t type_byte = (2 << 6) | (SpallAutoEventType_Begin << 4) | (spall_squash_2[dt_size] << 2) | name_args_lens; - - int i = 0; - *(ev_buffer + i) = type_byte; i += 1; - memcpy(ev_buffer + i, &dt, 8); i += dt_size; - memcpy(ev_buffer + i, &trunc_name_len, name_len_size); i += name_len_size; - memcpy(ev_buffer + i, &trunc_args_len, args_len_size); i += args_len_size; - memcpy(ev_buffer + i, name, trunc_name_len); i += trunc_name_len; - memcpy(ev_buffer + i, args, trunc_args_len); i += trunc_args_len; - - spall_buffer->previous_ts = now; - spall_buffer->head += i; - - return true; -} - -SPALL_NOINSTRUMENT SPALL_FORCEINLINE bool spall_auto_buffer_end(void) { - return spall_buffer_micro_end(); -} - -SPALL_NOINSTRUMENT SPALL_FORCEINLINE bool (spall_auto_thread_init)(uint32_t thread_id, size_t buffer_size) { - if (buffer_size < 512) { return false; } - if (spall_buffer != NULL) { return false; } - - spall_buffer = (SpallBuffer *)calloc(sizeof(SpallBuffer), 1); - spall_buffer->data = (uint8_t *)malloc(buffer_size); - spall_buffer->length = buffer_size; - spall_buffer->thread_id = thread_id; - spall_buffer->sub_length = buffer_size / 2; - - // removing initial page-fault bubbles to make the data a little more accurate, at the cost of thread spin-up time - memset(spall_buffer->data, 1, spall_buffer->length); - - spall_buffer->writer.is_running = true; - spall_thread_start(spall_buffer); - - spall_auto_buffer_flush(); - spall_thread_running = true; - return true; -} - -void (spall_auto_thread_quit)(void) { - spall_thread_running = false; - spall_auto_buffer_flush(); - - spall_buffer->writer.is_running = false; - spall_buffer->writer.ptr = 1; - spall_signal(&spall_buffer->writer.ptr); - spall_thread_end(spall_buffer); - - free(spall_buffer->data); - free(spall_buffer); - spall_buffer = NULL; -} - -SPALL_FN void *spall_canonical_addr(void* fn) { - // sometimes the pointer we get back is to a jump table; walk past that first layer. - - void *ret = fn; -#if SPALL_IS_X64 - unsigned char *fn_data = (unsigned char *)fn; - if (fn_data[0] == 0xE9) { - // JMP rel32 - int32_t target = *(int32_t*) &fn_data[1]; - - int jump_inst_size = 5; - ret = (void *)(fn_data + jump_inst_size + target); - } -#endif - - return ret; -} - - -SPALL_NOINSTRUMENT bool spall_auto_init(char *filename) { - if (!filename) return false; - memset(&spall_ctx, 0, sizeof(spall_ctx)); - - spall_ctx.file = fopen(filename, "wb"); // TODO: handle utf8 and long paths on windows - if (spall_ctx.file) { // basically freopen() but we don't want to force users to lug along another macro define - fclose(spall_ctx.file); - spall_ctx.file = fopen(filename, "ab"); - } - if (!spall_ctx.file) { return false; } - - spall_ctx.stamp_scale = spall_get_clock_multiplier(); - SpallHeader header = {0}; - header.magic_header = 0xABADF00D; - header.version = 2; - header.timestamp_unit = spall_ctx.stamp_scale; - header.known_address = (uint64_t)spall_canonical_addr((void *)spall_auto_init); - - char *program_path; - if (!get_program_path(&program_path)) { return false; } - uint16_t program_path_len = (uint16_t)strlen(program_path); - - header.program_path_len = program_path_len; - - size_t full_header_size = sizeof(SpallHeader) + (size_t)program_path_len; - uint8_t *full_header = (uint8_t *)malloc(full_header_size); - memcpy(full_header, &header, sizeof(SpallHeader)); - memcpy(full_header + sizeof(SpallHeader), program_path, program_path_len); - - size_t write_ret = fwrite(full_header, 1, full_header_size, spall_ctx.file); - if (write_ret < full_header_size) { return false; } - - free(full_header); - return true; -} - -SPALL_NOINSTRUMENT void spall_auto_quit(void) {} - -SPALL_NOINSTRUMENT void __cyg_profile_func_enter(void *fn, void *caller) { - if (!spall_thread_running) { - return; - } - fn = spall_canonical_addr(fn); - - spall_thread_running = false; - spall_buffer_micro_begin((uint64_t)fn, (uint64_t)caller); - spall_thread_running = true; -} - -SPALL_NOINSTRUMENT void __cyg_profile_func_exit(void *fn, void *caller) { - if (!spall_thread_running) { - return; - } - - spall_thread_running = false; - spall_buffer_micro_end(); - spall_thread_running = true; -} - -#ifdef __cplusplus -} -#endif - -#endif -#endif diff --git a/vendor/cuik b/vendor/cuik new file mode 160000 index 00000000..eb8f46ce --- /dev/null +++ b/vendor/cuik @@ -0,0 +1 @@ +Subproject commit eb8f46ce2cca3e3809443a5fa719eb95c4e91d76 diff --git a/vendor/isocline b/vendor/isocline new file mode 160000 index 00000000..c9310ae5 --- /dev/null +++ b/vendor/isocline @@ -0,0 +1 @@ +Subproject commit c9310ae58941559d761fe5d2dd2713d245f18da6 diff --git a/vendor/isocline/attr.c b/vendor/isocline/attr.c deleted file mode 100644 index b5ad78f8..00000000 --- a/vendor/isocline/attr.c +++ /dev/null @@ -1,294 +0,0 @@ -/* ---------------------------------------------------------------------------- - Copyright (c) 2021, Daan Leijen - This is free software; you can redistribute it and/or modify it - under the terms of the MIT License. A copy of the license can be - found in the "LICENSE" file at the root of this distribution. ------------------------------------------------------------------------------*/ -#include - -#include "common.h" -#include "stringbuf.h" // str_next_ofs -#include "attr.h" -#include "term.h" // color_from_ansi256 - -//------------------------------------------------------------- -// Attributes -//------------------------------------------------------------- - -ic_private attr_t attr_none(void) { - attr_t attr; - attr.value = 0; - return attr; -} - -ic_private attr_t attr_default(void) { - attr_t attr = attr_none(); - attr.x.color = IC_ANSI_DEFAULT; - attr.x.bgcolor = IC_ANSI_DEFAULT; - attr.x.bold = IC_OFF; - attr.x.underline = IC_OFF; - attr.x.reverse = IC_OFF; - attr.x.italic = IC_OFF; - return attr; -} - -ic_private bool attr_is_none(attr_t attr) { - return (attr.value == 0); -} - -ic_private bool attr_is_eq(attr_t attr1, attr_t attr2) { - return (attr1.value == attr2.value); -} - -ic_private attr_t attr_from_color( ic_color_t color ) { - attr_t attr = attr_none(); - attr.x.color = color; - return attr; -} - - -ic_private attr_t attr_update_with( attr_t oldattr, attr_t newattr ) { - attr_t attr = oldattr; - if (newattr.x.color != IC_COLOR_NONE) { attr.x.color = newattr.x.color; } - if (newattr.x.bgcolor != IC_COLOR_NONE) { attr.x.bgcolor = newattr.x.bgcolor; } - if (newattr.x.bold != IC_NONE) { attr.x.bold = newattr.x.bold; } - if (newattr.x.italic != IC_NONE) { attr.x.italic = newattr.x.italic; } - if (newattr.x.reverse != IC_NONE) { attr.x.reverse = newattr.x.reverse; } - if (newattr.x.underline != IC_NONE) { attr.x.underline = newattr.x.underline; } - return attr; -} - -static bool sgr_is_digit(char c) { - return (c >= '0' && c <= '9'); -} - -static bool sgr_is_sep( char c ) { - return (c==';' || c==':'); -} - -static bool sgr_next_par(const char* s, ssize_t* pi, ssize_t* par) { - const ssize_t i = *pi; - ssize_t n = 0; - while( sgr_is_digit(s[i+n])) { - n++; - } - if (n==0) { - *par = 0; - return true; - } - else { - *pi = i+n; - return ic_atoz(s+i, par); - } -} - -static bool sgr_next_par3(const char* s, ssize_t* pi, ssize_t* p1, ssize_t* p2, ssize_t* p3) { - bool ok = false; - ssize_t i = *pi; - if (sgr_next_par(s,&i,p1) && sgr_is_sep(s[i])) { - i++; - if (sgr_next_par(s,&i,p2) && sgr_is_sep(s[i])) { - i++; - if (sgr_next_par(s,&i,p3)) { - ok = true; - }; - } - } - *pi = i; - return ok; -} - -ic_private attr_t attr_from_sgr( const char* s, ssize_t len) { - attr_t attr = attr_none(); - for( ssize_t i = 0; i < len && s[i] != 0; i++) { - ssize_t cmd = 0; - if (!sgr_next_par(s,&i,&cmd)) continue; - switch(cmd) { - case 0: attr = attr_default(); break; - case 1: attr.x.bold = IC_ON; break; - case 3: attr.x.italic = IC_ON; break; - case 4: attr.x.underline = IC_ON; break; - case 7: attr.x.reverse = IC_ON; break; - case 22: attr.x.bold = IC_OFF; break; - case 23: attr.x.italic = IC_OFF; break; - case 24: attr.x.underline = IC_OFF; break; - case 27: attr.x.reverse = IC_OFF; break; - case 39: attr.x.color = IC_ANSI_DEFAULT; break; - case 49: attr.x.bgcolor = IC_ANSI_DEFAULT; break; - default: { - if (cmd >= 30 && cmd <= 37) { - attr.x.color = IC_ANSI_BLACK + (unsigned)(cmd - 30); - } - else if (cmd >= 40 && cmd <= 47) { - attr.x.bgcolor = IC_ANSI_BLACK + (unsigned)(cmd - 40); - } - else if (cmd >= 90 && cmd <= 97) { - attr.x.color = IC_ANSI_DARKGRAY + (unsigned)(cmd - 90); - } - else if (cmd >= 100 && cmd <= 107) { - attr.x.bgcolor = IC_ANSI_DARKGRAY + (unsigned)(cmd - 100); - } - else if ((cmd == 38 || cmd == 48) && sgr_is_sep(s[i])) { - // non-associative SGR :-( - ssize_t par = 0; - i++; - if (sgr_next_par(s, &i, &par)) { - if (par==5 && sgr_is_sep(s[i])) { - // ansi 256 index - i++; - if (sgr_next_par(s, &i, &par) && par >= 0 && par <= 0xFF) { - ic_color_t color = color_from_ansi256(par); - if (cmd==38) { attr.x.color = color; } - else { attr.x.bgcolor = color; } - } - } - else if (par == 2 && sgr_is_sep(s[i])) { - // rgb value - i++; - ssize_t r,g,b; - if (sgr_next_par3(s, &i, &r,&g,&b)) { - ic_color_t color = ic_rgbx(r,g,b); - if (cmd==38) { attr.x.color = color; } - else { attr.x.bgcolor = color; } - } - } - } - } - else { - debug_msg("attr: unknow ANSI SGR code: %zd\n", cmd ); - } - } - } - } - return attr; -} - -ic_private attr_t attr_from_esc_sgr( const char* s, ssize_t len) { - if (len <= 2 || s[0] != '\x1B' || s[1] != '[' || s[len-1] != 'm') return attr_none(); - return attr_from_sgr(s+2, len-2); -} - - -//------------------------------------------------------------- -// Attribute buffer -//------------------------------------------------------------- -struct attrbuf_s { - attr_t* attrs; - ssize_t capacity; - ssize_t count; - alloc_t* mem; -}; - -static bool attrbuf_ensure_capacity( attrbuf_t* ab, ssize_t needed ) { - if (needed <= ab->capacity) return true; - ssize_t newcap = (ab->capacity <= 0 ? 240 : (ab->capacity > 1000 ? ab->capacity + 1000 : 2*ab->capacity)); - if (needed > newcap) { newcap = needed; } - attr_t* newattrs = mem_realloc_tp( ab->mem, attr_t, ab->attrs, newcap ); - if (newattrs == NULL) return false; - ab->attrs = newattrs; - ab->capacity = newcap; - assert(needed <= ab->capacity); - return true; -} - -static bool attrbuf_ensure_extra( attrbuf_t* ab, ssize_t extra ) { - const ssize_t needed = ab->count + extra; - return attrbuf_ensure_capacity( ab, needed ); -} - - -ic_private attrbuf_t* attrbuf_new( alloc_t* mem ) { - attrbuf_t* ab = mem_zalloc_tp(mem,attrbuf_t); - if (ab == NULL) return NULL; - ab->mem = mem; - attrbuf_ensure_extra(ab,1); - return ab; -} - -ic_private void attrbuf_free( attrbuf_t* ab ) { - if (ab==NULL) return; - mem_free(ab->mem, ab->attrs); - mem_free(ab->mem, ab); -} - -ic_private void attrbuf_clear(attrbuf_t* ab) { - if (ab == NULL) return; - ab->count = 0; -} - -ic_private ssize_t attrbuf_len( attrbuf_t* ab ) { - return (ab==NULL ? 0 : ab->count); -} - -ic_private const attr_t* attrbuf_attrs( attrbuf_t* ab, ssize_t expected_len ) { - assert(expected_len <= ab->count ); - // expand if needed - if (ab->count < expected_len) { - if (!attrbuf_ensure_capacity(ab,expected_len)) return NULL; - for(ssize_t i = ab->count; i < expected_len; i++) { - ab->attrs[i] = attr_none(); - } - ab->count = expected_len; - } - return ab->attrs; -} - - - -static void attrbuf_update_set_at( attrbuf_t* ab, ssize_t pos, ssize_t count, attr_t attr, bool update ) { - const ssize_t end = pos + count; - if (!attrbuf_ensure_capacity(ab, end)) return; - ssize_t i; - // initialize if end is beyond the count (todo: avoid duplicate init and set if update==false?) - if (ab->count < end) { - for(i = ab->count; i < end; i++) { - ab->attrs[i] = attr_none(); - } - ab->count = end; - } - // fill pos to end with attr - for(i = pos; i < end; i++) { - ab->attrs[i] = (update ? attr_update_with(ab->attrs[i],attr) : attr); - } -} - -ic_private void attrbuf_set_at( attrbuf_t* ab, ssize_t pos, ssize_t count, attr_t attr ) { - attrbuf_update_set_at(ab, pos, count, attr, false); -} - -ic_private void attrbuf_update_at( attrbuf_t* ab, ssize_t pos, ssize_t count, attr_t attr ) { - attrbuf_update_set_at(ab, pos, count, attr, true); -} - -ic_private void attrbuf_insert_at( attrbuf_t* ab, ssize_t pos, ssize_t count, attr_t attr ) { - if (pos < 0 || pos > ab->count || count <= 0) return; - if (!attrbuf_ensure_extra(ab,count)) return; - ic_memmove( ab->attrs + pos + count, ab->attrs + pos, (ab->count - pos)*ssizeof(attr_t) ); - ab->count += count; - attrbuf_set_at( ab, pos, count, attr ); -} - - -// note: must allow ab == NULL! -ic_private ssize_t attrbuf_append_n( stringbuf_t* sb, attrbuf_t* ab, const char* s, ssize_t len, attr_t attr ) { - if (s == NULL || len == 0) return sbuf_len(sb); - if (ab != NULL) { - if (!attrbuf_ensure_extra(ab,len)) return sbuf_len(sb); - attrbuf_set_at(ab, ab->count, len, attr); - } - return sbuf_append_n(sb,s,len); -} - -ic_private attr_t attrbuf_attr_at( attrbuf_t* ab, ssize_t pos ) { - if (ab==NULL || pos < 0 || pos > ab->count) return attr_none(); - return ab->attrs[pos]; -} - -ic_private void attrbuf_delete_at( attrbuf_t* ab, ssize_t pos, ssize_t count ) { - if (ab==NULL || pos < 0 || pos > ab->count) return; - if (pos + count > ab->count) { count = ab->count - pos; } - if (count == 0) return; - assert(pos + count <= ab->count); - ic_memmove( ab->attrs + pos, ab->attrs + pos + count, ab->count - (pos + count) ); - ab->count -= count; -} diff --git a/vendor/isocline/attr.h b/vendor/isocline/attr.h deleted file mode 100644 index 8f37d050..00000000 --- a/vendor/isocline/attr.h +++ /dev/null @@ -1,70 +0,0 @@ -/* ---------------------------------------------------------------------------- - Copyright (c) 2021, Daan Leijen - This is free software; you can redistribute it and/or modify it - under the terms of the MIT License. A copy of the license can be - found in the "LICENSE" file at the root of this distribution. ------------------------------------------------------------------------------*/ -#pragma once -#ifndef IC_ATTR_H -#define IC_ATTR_H - -#include "common.h" -#include "stringbuf.h" - -//------------------------------------------------------------- -// text attributes -//------------------------------------------------------------- - -#define IC_ON (1) -#define IC_OFF (-1) -#define IC_NONE (0) - -// try to fit in 64 bits -// note: order is important for some compilers -// note: each color can actually be 25 bits -typedef union attr_s { - struct { - unsigned int color:28; - signed int bold:2; - signed int reverse:2; - unsigned int bgcolor:28; - signed int underline:2; - signed int italic:2; - } x; - uint64_t value; -} attr_t; - -ic_private attr_t attr_none(void); -ic_private attr_t attr_default(void); -ic_private attr_t attr_from_color( ic_color_t color ); - -ic_private bool attr_is_none(attr_t attr); -ic_private bool attr_is_eq(attr_t attr1, attr_t attr2); - -ic_private attr_t attr_update_with( attr_t attr, attr_t newattr ); - -ic_private attr_t attr_from_sgr( const char* s, ssize_t len); -ic_private attr_t attr_from_esc_sgr( const char* s, ssize_t len); - -//------------------------------------------------------------- -// attribute buffer used for rich rendering -//------------------------------------------------------------- - -struct attrbuf_s; -typedef struct attrbuf_s attrbuf_t; - -ic_private attrbuf_t* attrbuf_new( alloc_t* mem ); -ic_private void attrbuf_free( attrbuf_t* ab ); // ab can be NULL -ic_private void attrbuf_clear( attrbuf_t* ab ); // ab can be NULL -ic_private ssize_t attrbuf_len( attrbuf_t* ab); // ab can be NULL -ic_private const attr_t* attrbuf_attrs( attrbuf_t* ab, ssize_t expected_len ); -ic_private ssize_t attrbuf_append_n( stringbuf_t* sb, attrbuf_t* ab, const char* s, ssize_t len, attr_t attr ); - -ic_private void attrbuf_set_at( attrbuf_t* ab, ssize_t pos, ssize_t count, attr_t attr ); -ic_private void attrbuf_update_at( attrbuf_t* ab, ssize_t pos, ssize_t count, attr_t attr ); -ic_private void attrbuf_insert_at( attrbuf_t* ab, ssize_t pos, ssize_t count, attr_t attr ); - -ic_private attr_t attrbuf_attr_at( attrbuf_t* ab, ssize_t pos ); -ic_private void attrbuf_delete_at( attrbuf_t* ab, ssize_t pos, ssize_t count ); - -#endif // IC_ATTR_H diff --git a/vendor/isocline/bbcode.c b/vendor/isocline/bbcode.c deleted file mode 100644 index 4d11ac38..00000000 --- a/vendor/isocline/bbcode.c +++ /dev/null @@ -1,842 +0,0 @@ -/* ---------------------------------------------------------------------------- - Copyright (c) 2021, Daan Leijen - This is free software; you can redistribute it and/or modify it - under the terms of the MIT License. A copy of the license can be - found in the "LICENSE" file at the root of this distribution. ------------------------------------------------------------------------------*/ -#include -#include -#include -#include - -#include "common.h" -#include "attr.h" -#include "term.h" -#include "bbcode.h" - -//------------------------------------------------------------- -// HTML color table -//------------------------------------------------------------- - -#include "bbcode_colors.c" - -//------------------------------------------------------------- -// Types -//------------------------------------------------------------- - -typedef struct style_s { - const char* name; // name of the style - attr_t attr; // attribute to apply -} style_t; - -typedef enum align_e { - IC_ALIGN_LEFT, - IC_ALIGN_CENTER, - IC_ALIGN_RIGHT -} align_t; - -typedef struct width_s { - ssize_t w; // > 0 - align_t align; - bool dots; // "..." (e.g. "sentence...") - char fill; // " " (e.g. "hello ") -} width_t; - -typedef struct tag_s { - const char* name; // tag open name - attr_t attr; // the saved attribute before applying the style - width_t width; // start sequence of at most "width" columns - ssize_t pos; // start position in the output (used for width restriction) -} tag_t; - - -static void tag_init(tag_t* tag) { - memset(tag,0,sizeof(*tag)); -} - -struct bbcode_s { - tag_t* tags; // stack of tags; one entry for each open tag - ssize_t tags_capacity; - ssize_t tags_nesting; - style_t* styles; // list of used defined styles - ssize_t styles_capacity; - ssize_t styles_count; - term_t* term; // terminal - alloc_t* mem; // allocator - // caches - stringbuf_t* out; // print buffer - attrbuf_t* out_attrs; - stringbuf_t* vout; // vprintf buffer -}; - - -//------------------------------------------------------------- -// Create, helpers -//------------------------------------------------------------- - -ic_private bbcode_t* bbcode_new( alloc_t* mem, term_t* term ) { - bbcode_t* bb = mem_zalloc_tp(mem,bbcode_t); - if (bb==NULL) return NULL; - bb->mem = mem; - bb->term = term; - bb->out = sbuf_new(mem); - bb->out_attrs = attrbuf_new(mem); - bb->vout = sbuf_new(mem); - return bb; -} - -ic_private void bbcode_free( bbcode_t* bb ) { - for(ssize_t i = 0; i < bb->styles_count; i++) { - mem_free(bb->mem, bb->styles[i].name); - } - mem_free(bb->mem, bb->tags); - mem_free(bb->mem, bb->styles); - sbuf_free(bb->vout); - sbuf_free(bb->out); - attrbuf_free(bb->out_attrs); - mem_free(bb->mem, bb); -} - -ic_private void bbcode_style_add( bbcode_t* bb, const char* style_name, attr_t attr ) { - if (bb->styles_count >= bb->styles_capacity) { - ssize_t newlen = bb->styles_capacity + 32; - style_t* p = mem_realloc_tp( bb->mem, style_t, bb->styles, newlen ); - if (p == NULL) return; - bb->styles = p; - bb->styles_capacity = newlen; - } - assert(bb->styles_count < bb->styles_capacity); - bb->styles[bb->styles_count].name = mem_strdup( bb->mem, style_name ); - bb->styles[bb->styles_count].attr = attr; - bb->styles_count++; -} - -static ssize_t bbcode_tag_push( bbcode_t* bb, const tag_t* tag ) { - if (bb->tags_nesting >= bb->tags_capacity) { - ssize_t newcap = bb->tags_capacity + 32; - tag_t* p = mem_realloc_tp( bb->mem, tag_t, bb->tags, newcap ); - if (p == NULL) return -1; - bb->tags = p; - bb->tags_capacity = newcap; - } - assert(bb->tags_nesting < bb->tags_capacity); - bb->tags[bb->tags_nesting] = *tag; - bb->tags_nesting++; - return (bb->tags_nesting-1); -} - -static void bbcode_tag_pop( bbcode_t* bb, tag_t* tag ) { - if (bb->tags_nesting <= 0) { - if (tag != NULL) { tag_init(tag); } - } - else { - bb->tags_nesting--; - if (tag != NULL) { - *tag = bb->tags[bb->tags_nesting]; - } - } -} - -//------------------------------------------------------------- -// Invalid parse/values/balance -//------------------------------------------------------------- - -static void bbcode_invalid(const char* fmt, ... ) { - if (getenv("ISOCLINE_BBCODE_DEBUG") != NULL) { - va_list args; - va_start(args,fmt); - vfprintf(stderr,fmt,args); - va_end(args); - } -} - -//------------------------------------------------------------- -// Set attributes -//------------------------------------------------------------- - - -static attr_t bbcode_open( bbcode_t* bb, ssize_t out_pos, const tag_t* tag, attr_t current ) { - // save current and set - tag_t cur; - tag_init(&cur); - cur.name = tag->name; - cur.attr = current; - cur.width = tag->width; - cur.pos = out_pos; - bbcode_tag_push(bb,&cur); - return attr_update_with( current, tag->attr ); -} - -static bool bbcode_close( bbcode_t* bb, ssize_t base, const char* name, tag_t* pprev ) { - // pop until match - while (bb->tags_nesting > base) { - tag_t prev; - bbcode_tag_pop(bb,&prev); - if (name==NULL || prev.name==NULL || ic_stricmp(prev.name,name) == 0) { - // matched - if (pprev != NULL) { *pprev = prev; } - return true; - } - else { - // unbalanced: we either continue popping or restore the tags depending if there is a matching open tag in our tags. - bool has_open_tag = false; - if (name != NULL) { - for( ssize_t i = bb->tags_nesting - 1; i > base; i--) { - if (bb->tags[i].name != NULL && ic_stricmp(bb->tags[i].name, name) == 0) { - has_open_tag = true; - break; - } - } - } - bbcode_invalid("bbcode: unbalanced tags: open [%s], close [/%s]\n", prev.name, name); - if (!has_open_tag) { - bbcode_tag_push( bb, &prev ); // restore the tags and ignore this close tag - break; - } - else { - // continue until we hit our open tag - } - } - } - if (pprev != NULL) { memset(pprev,0,sizeof(*pprev)); } - return false; -} - -//------------------------------------------------------------- -// Update attributes -//------------------------------------------------------------- - -static const char* attr_update_bool( const char* fname, signed int* field, const char* value ) { - if (value == NULL || value[0] == 0 || strcmp(value,"on") || strcmp(value,"true") || strcmp(value,"1")) { - *field = IC_ON; - } - else if (strcmp(value,"off") || strcmp(value,"false") || strcmp(value,"0")) { - *field = IC_OFF; - } - else { - bbcode_invalid("bbcode: invalid %s value: %s\n", fname, value ); - } - return fname; -} - -static const char* attr_update_color( const char* fname, ic_color_t* field, const char* value ) { - if (value == NULL || value[0] == 0 || strcmp(value,"none") == 0) { - *field = IC_COLOR_NONE; - return fname; - } - - // hex value - if (value[0] == '#') { - uint32_t rgb = 0; - if (sscanf(value,"#%x",&rgb) == 1) { - *field = ic_rgb(rgb); - } - else { - bbcode_invalid("bbcode: invalid color code: %s\n", value); - } - return fname; - } - - // search color names - ssize_t lo = 0; - ssize_t hi = IC_HTML_COLOR_COUNT-1; - while( lo <= hi ) { - ssize_t mid = (lo + hi) / 2; - style_color_t* info = &html_colors[mid]; - int cmp = strcmp(info->name,value); - if (cmp < 0) { - lo = mid+1; - } - else if (cmp > 0) { - hi = mid-1; - } - else { - *field = info->color; - return fname; - } - } - bbcode_invalid("bbcode: unknown %s: %s\n", fname, value); - *field = IC_COLOR_NONE; - return fname; -} - -static const char* attr_update_sgr( const char* fname, attr_t* attr, const char* value ) { - *attr = attr_update_with(*attr, attr_from_sgr(value, ic_strlen(value))); - return fname; -} - -static void attr_update_width( width_t* pwidth, char default_fill, const char* value ) { - // parse width value: ;;; - width_t width; - memset(&width, 0, sizeof(width)); - width.fill = default_fill; // use 0 for no-fill (as for max-width) - if (ic_atoz(value, &width.w)) { - ssize_t i = 0; - while( value[i] != ';' && value[i] != 0 ) { i++; } - if (value[i] == ';') { - i++; - ssize_t len = 0; - while( value[i+len] != ';' && value[i+len] != 0 ) { len++; } - if (len == 4 && ic_istarts_with(value+i,"left")) { - width.align = IC_ALIGN_LEFT; - } - else if (len == 5 && ic_istarts_with(value+i,"right")) { - width.align = IC_ALIGN_RIGHT; - } - else if (len == 6 && ic_istarts_with(value+i,"center")) { - width.align = IC_ALIGN_CENTER; - } - i += len; - if (value[i] == ';') { - i++; len = 0; - while( value[i+len] != ';' && value[i+len] != 0 ) { len++; } - if (len == 1) { width.fill = value[i]; } - i+= len; - if (value[i] == ';') { - i++; len = 0; - while( value[i+len] != ';' && value[i+len] != 0 ) { len++; } - if ((len == 2 && ic_istarts_with(value+i,"on")) || (len == 1 && value[i] == '1')) { width.dots = true; } - i += len; - } - } - } - } - else { - bbcode_invalid("bbcode: illegal width: %s\n", value); - } - *pwidth = width; -} - -static const char* attr_update_ansi_color( const char* fname, ic_color_t* color, const char* value ) { - ssize_t num = 0; - if (ic_atoz(value, &num) && num >= 0 && num <= 256) { - *color = color_from_ansi256(num); - } - return fname; -} - - -static const char* attr_update_property( tag_t* tag, const char* attr_name, const char* value ) { - const char* fname = NULL; - fname = "bold"; - if (strcmp(attr_name,fname) == 0) { - signed int b = IC_NONE; - attr_update_bool(fname,&b, value); - if (b != IC_NONE) { tag->attr.x.bold = b; } - return fname; - } - fname = "italic"; - if (strcmp(attr_name,fname) == 0) { - signed int b = IC_NONE; - attr_update_bool(fname,&b, value); - if (b != IC_NONE) { tag->attr.x.italic = b; } - return fname; - } - fname = "underline"; - if (strcmp(attr_name,fname) == 0) { - signed int b = IC_NONE; - attr_update_bool(fname,&b, value); - if (b != IC_NONE) { tag->attr.x.underline = b; } - return fname; - } - fname = "reverse"; - if (strcmp(attr_name,fname) == 0) { - signed int b = IC_NONE; - attr_update_bool(fname,&b, value); - if (b != IC_NONE) { tag->attr.x.reverse = b; } - return fname; - } - fname = "color"; - if (strcmp(attr_name,fname) == 0) { - unsigned int color = IC_COLOR_NONE; - attr_update_color(fname, &color, value); - if (color != IC_COLOR_NONE) { tag->attr.x.color = color; } - return fname; - } - fname = "bgcolor"; - if (strcmp(attr_name,fname) == 0) { - unsigned int color = IC_COLOR_NONE; - attr_update_color(fname, &color, value); - if (color != IC_COLOR_NONE) { tag->attr.x.bgcolor = color; } - return fname; - } - fname = "ansi-sgr"; - if (strcmp(attr_name,fname) == 0) { - attr_update_sgr(fname, &tag->attr, value); - return fname; - } - fname = "ansi-color"; - if (strcmp(attr_name,fname) == 0) { - ic_color_t color = IC_COLOR_NONE;; - attr_update_ansi_color(fname, &color, value); - if (color != IC_COLOR_NONE) { tag->attr.x.color = color; } - return fname; - } - fname = "ansi-bgcolor"; - if (strcmp(attr_name,fname) == 0) { - ic_color_t color = IC_COLOR_NONE;; - attr_update_ansi_color(fname, &color, value); - if (color != IC_COLOR_NONE) { tag->attr.x.bgcolor = color; } - return fname; - } - fname = "width"; - if (strcmp(attr_name,fname) == 0) { - attr_update_width(&tag->width, ' ', value); - return fname; - } - fname = "max-width"; - if (strcmp(attr_name,fname) == 0) { - attr_update_width(&tag->width, 0, value); - return "width"; - } - else { - return NULL; - } -} - -static const style_t builtin_styles[] = { - { "b", { { IC_COLOR_NONE, IC_ON , IC_NONE, IC_COLOR_NONE, IC_NONE, IC_NONE } } }, - { "r", { { IC_COLOR_NONE, IC_NONE, IC_ON , IC_COLOR_NONE, IC_NONE, IC_NONE } } }, - { "u", { { IC_COLOR_NONE, IC_NONE, IC_NONE, IC_COLOR_NONE, IC_ON , IC_NONE } } }, - { "i", { { IC_COLOR_NONE, IC_NONE, IC_NONE, IC_COLOR_NONE, IC_NONE, IC_ON } } }, - { "em", { { IC_COLOR_NONE, IC_ON , IC_NONE, IC_COLOR_NONE, IC_NONE, IC_NONE } } }, // bold - { "url",{ { IC_COLOR_NONE, IC_NONE, IC_NONE, IC_COLOR_NONE, IC_ON, IC_NONE } } }, // underline - { NULL, { { IC_COLOR_NONE, IC_NONE, IC_NONE, IC_COLOR_NONE, IC_NONE, IC_NONE } } } -}; - -static void attr_update_with_styles( tag_t* tag, const char* attr_name, const char* value, - bool usebgcolor, const style_t* styles, ssize_t count ) -{ - // direct hex color? - if (attr_name[0] == '#' && (value == NULL || value[0]==0)) { - value = attr_name; - attr_name = (usebgcolor ? "bgcolor" : "color"); - } - // first try if it is a builtin property - const char* name; - if ((name = attr_update_property(tag,attr_name,value)) != NULL) { - if (tag->name != NULL) tag->name = name; - return; - } - // then check all styles - while( count-- > 0 ) { - const style_t* style = styles + count; - if (strcmp(style->name,attr_name) == 0) { - tag->attr = attr_update_with(tag->attr,style->attr); - if (tag->name != NULL) tag->name = style->name; - return; - } - } - // check builtin styles; todo: binary search? - for( const style_t* style = builtin_styles; style->name != NULL; style++) { - if (strcmp(style->name,attr_name) == 0) { - tag->attr = attr_update_with(tag->attr,style->attr); - if (tag->name != NULL) tag->name = style->name; - return; - } - } - // check colors as a style - ssize_t lo = 0; - ssize_t hi = IC_HTML_COLOR_COUNT-1; - while( lo <= hi ) { - ssize_t mid = (lo + hi) / 2; - style_color_t* info = &html_colors[mid]; - int cmp = strcmp(info->name,attr_name); - if (cmp < 0) { - lo = mid+1; - } - else if (cmp > 0) { - hi = mid-1; - } - else { - attr_t cattr = attr_none(); - if (usebgcolor) { cattr.x.bgcolor = info->color; } - else { cattr.x.color = info->color; } - tag->attr = attr_update_with(tag->attr,cattr); - if (tag->name != NULL) tag->name = info->name; - return; - } - } - // not found - bbcode_invalid("bbcode: unknown style: %s\n", attr_name); -} - - -ic_private attr_t bbcode_style( bbcode_t* bb, const char* style_name ) { - tag_t tag; - tag_init(&tag); - attr_update_with_styles( &tag, style_name, NULL, false, bb->styles, bb->styles_count ); - return tag.attr; -} - -//------------------------------------------------------------- -// Parse tags -//------------------------------------------------------------- - -ic_private const char* parse_skip_white(const char* s) { - while( *s != 0 && *s != ']') { - if (!(*s == ' ' || *s == '\t' || *s == '\n' || *s == '\r')) break; - s++; - } - return s; -} - -ic_private const char* parse_skip_to_white(const char* s) { - while( *s != 0 && *s != ']') { - if (*s == ' ' || *s == '\t' || *s == '\n' || *s == '\r') break; - s++; - } - return parse_skip_white(s); -} - -ic_private const char* parse_skip_to_end(const char* s) { - while( *s != 0 && *s != ']' ) { s++; } - return s; -} - -ic_private const char* parse_attr_name(const char* s) { - if (*s == '#') { - s++; // hex rgb color as id - while( *s != 0 && *s != ']') { - if (!((*s >= 'a' && *s <= 'f') || (*s >= 'A' && *s <= 'Z') || (*s >= '0' && *s <= '9'))) break; - s++; - } - } - else { - while( *s != 0 && *s != ']') { - if (!((*s >= 'a' && *s <= 'z') || (*s >= 'A' && *s <= 'Z') || - (*s >= '0' && *s <= '9') || *s == '_' || *s == '-')) break; - s++; - } - } - return s; -} - -ic_private const char* parse_value(const char* s, const char** start, const char** end) { - if (*s == '"') { - s++; - *start = s; - while( *s != 0 ) { - if (*s == '"') break; - s++; - } - *end = s; - if (*s == '"') { s++; } - } - else if (*s == '#') { - *start = s; - s++; - while( *s != 0 ) { - if (!((*s >= 'a' && *s <= 'f') || (*s >= 'A' && *s <= 'Z') || (*s >= '0' && *s <= '9'))) break; - s++; - } - *end = s; - } - else { - *start = s; - while( *s != 0 ) { - if (!((*s >= 'a' && *s <= 'z') || (*s >= 'A' && *s <= 'F') || (*s >= '0' && *s <= '9') || *s == '-' || *s == '_')) break; - s++; - } - *end = s; - } - return s; -} - -ic_private const char* parse_tag_value( tag_t* tag, char* idbuf, const char* s, const style_t* styles, ssize_t scount ) { - // parse: \s*[\w-]+\s*(=\s*) - bool usebgcolor = false; - const char* id = s; - const char* idend = parse_attr_name(id); - const char* val = NULL; - const char* valend = NULL; - if (id == idend) { - bbcode_invalid("bbcode: empty identifier? %.10s...\n", id ); - return parse_skip_to_white(id); - } - // "on" bgcolor? - s = parse_skip_white(idend); - if (idend - id == 2 && ic_strnicmp(id,"on",2) == 0 && *s != '=') { - usebgcolor = true; - id = s; - idend = parse_attr_name(id); - if (id == idend) { - bbcode_invalid("bbcode: empty identifier follows 'on'? %.10s...\n", id ); - return parse_skip_to_white(id); - } - s = parse_skip_white(idend); - } - // value - if (*s == '=') { - s++; - s = parse_skip_white(s); - s = parse_value(s, &val, &valend); - s = parse_skip_white(s); - } - // limit name and attr to 128 bytes - char valbuf[128]; - ic_strncpy( idbuf, 128, id, idend - id); - ic_strncpy( valbuf, 128, val, valend - val); - ic_str_tolower(idbuf); - ic_str_tolower(valbuf); - attr_update_with_styles( tag, idbuf, valbuf, usebgcolor, styles, scount ); - return s; -} - -static const char* parse_tag_values( tag_t* tag, char* idbuf, const char* s, const style_t* styles, ssize_t scount ) { - s = parse_skip_white(s); - idbuf[0] = 0; - ssize_t count = 0; - while( *s != 0 && *s != ']') { - char idbuf_next[128]; - s = parse_tag_value(tag, (count==0 ? idbuf : idbuf_next), s, styles, scount); - count++; - } - if (*s == ']') { s++; } - return s; -} - -static const char* parse_tag( tag_t* tag, char* idbuf, bool* open, bool* pre, const char* s, const style_t* styles, ssize_t scount ) { - *open = true; - *pre = false; - if (*s != '[') return s; - s = parse_skip_white(s+1); - if (*s == '!') { // pre - *pre = true; - s = parse_skip_white(s+1); - } - else if (*s == '/') { - *open = false; - s = parse_skip_white(s+1); - }; - s = parse_tag_values( tag, idbuf, s, styles, scount); - return s; -} - - -//--------------------------------------------------------- -// Styles -//--------------------------------------------------------- - -static void bbcode_parse_tag_content( bbcode_t* bb, const char* s, tag_t* tag ) { - tag_init(tag); - if (s != NULL) { - char idbuf[128]; - parse_tag_values(tag, idbuf, s, bb->styles, bb->styles_count); - } -} - -ic_private void bbcode_style_def( bbcode_t* bb, const char* style_name, const char* s ) { - tag_t tag; - bbcode_parse_tag_content( bb, s, &tag); - bbcode_style_add(bb, style_name, tag.attr); -} - -ic_private void bbcode_style_open( bbcode_t* bb, const char* fmt ) { - tag_t tag; - bbcode_parse_tag_content(bb, fmt, &tag); - term_set_attr( bb->term, bbcode_open(bb, 0, &tag, term_get_attr(bb->term)) ); -} - -ic_private void bbcode_style_close( bbcode_t* bb, const char* fmt ) { - const ssize_t base = bb->tags_nesting - 1; // as we end a style - tag_t tag; - bbcode_parse_tag_content(bb, fmt, &tag); - tag_t prev; - if (bbcode_close(bb, base, tag.name, &prev)) { - term_set_attr( bb->term, prev.attr ); - } -} - -//--------------------------------------------------------- -// Restrict to width -//--------------------------------------------------------- - -static void bbcode_restrict_width( ssize_t start, width_t width, stringbuf_t* out, attrbuf_t* attr_out ) { - if (width.w <= 0) return; - assert(start <= sbuf_len(out)); - assert(attr_out == NULL || sbuf_len(out) == attrbuf_len(attr_out)); - const char* s = sbuf_string(out) + start; - const ssize_t len = sbuf_len(out) - start; - const ssize_t w = str_column_width(s); - if (w == width.w) return; // fits exactly - if (w > width.w) { - // too large - ssize_t innerw = (width.dots && width.w > 3 ? width.w-3 : width.w); - if (width.align == IC_ALIGN_RIGHT) { - // right align - const ssize_t ndel = str_skip_until_fit( s, innerw ); - sbuf_delete_at( out, start, ndel ); - attrbuf_delete_at( attr_out, start, ndel ); - if (innerw < width.w) { - // add dots - sbuf_insert_at( out, "...", start ); - attr_t attr = attrbuf_attr_at(attr_out, start); - attrbuf_insert_at( attr_out, start, 3, attr); - } - } - else { - // left or center align - ssize_t count = str_take_while_fit( s, innerw ); - sbuf_delete_at( out, start + count, len - count ); - attrbuf_delete_at( attr_out, start + count, len - count ); - if (innerw < width.w) { - // add dots - attr_t attr = attrbuf_attr_at(attr_out,start); - attrbuf_append_n( out, attr_out, "...", 3, attr ); - } - } - } - else { - // too short, pad to width - const ssize_t diff = (width.w - w); - const ssize_t pad_left = (width.align == IC_ALIGN_RIGHT ? diff : (width.align == IC_ALIGN_LEFT ? 0 : diff / 2)); - const ssize_t pad_right = (width.align == IC_ALIGN_LEFT ? diff : (width.align == IC_ALIGN_RIGHT ? 0 : diff - pad_left)); - if (width.fill != 0 && pad_left > 0) { - const attr_t attr = attrbuf_attr_at(attr_out,start); - for( ssize_t i = 0; i < pad_left; i++) { // todo: optimize - sbuf_insert_char_at(out, width.fill, start); - } - attrbuf_insert_at( attr_out, start, pad_left, attr ); - } - if (width.fill != 0 && pad_right > 0) { - const attr_t attr = attrbuf_attr_at(attr_out,sbuf_len(out) - 1); - char buf[2]; - buf[0] = width.fill; - buf[1] = 0; - for( ssize_t i = 0; i < pad_right; i++) { // todo: optimize - attrbuf_append_n( out, attr_out, buf, 1, attr ); - } - } - } -} - -//--------------------------------------------------------- -// Print -//--------------------------------------------------------- - -ic_private ssize_t bbcode_process_tag( bbcode_t* bb, const char* s, const ssize_t nesting_base, - stringbuf_t* out, attrbuf_t* attr_out, attr_t* cur_attr ) { - assert(*s == '['); - tag_t tag; - tag_init(&tag); - bool open = true; - bool ispre = false; - char idbuf[128]; - const char* end = parse_tag( &tag, idbuf, &open, &ispre, s, bb->styles, bb->styles_count ); // todo: styles - assert(end > s); - if (open) { - if (!ispre) { - // open tag - *cur_attr = bbcode_open( bb, sbuf_len(out), &tag, *cur_attr ); - } - else { - // scan pre to end tag - attr_t attr = attr_update_with(*cur_attr, tag.attr); - char pre[132]; - if (snprintf(pre, 132, "[/%s]", idbuf) < ssizeof(pre)) { - const char* etag = strstr(end,pre); - if (etag == NULL) { - const ssize_t len = ic_strlen(end); - attrbuf_append_n(out, attr_out, end, len, attr); - end += len; - } - else { - attrbuf_append_n(out, attr_out, end, (etag - end), attr); - end = etag + ic_strlen(pre); - } - } - } - } - else { - // pop the tag - tag_t prev; - if (bbcode_close( bb, nesting_base, tag.name, &prev)) { - *cur_attr = prev.attr; - if (prev.width.w > 0) { - // closed a width tag; restrict the output to width - bbcode_restrict_width( prev.pos, prev.width, out, attr_out); - } - } - } - return (end - s); -} - -ic_private void bbcode_append( bbcode_t* bb, const char* s, stringbuf_t* out, attrbuf_t* attr_out ) { - if (bb == NULL || s == NULL) return; - attr_t attr = attr_none(); - const ssize_t base = bb->tags_nesting; // base; will not be popped - ssize_t i = 0; - while( s[i] != 0 ) { - // handle no tags in bulk - ssize_t nobb = 0; - char c; - while( (c = s[i+nobb]) != 0) { - if (c == '[' || c == '\\') { break; } - if (c == '\x1B' && s[i+nobb+1] == '[') { - nobb++; // don't count 'ESC[' as a tag opener - } - nobb++; - } - if (nobb > 0) { attrbuf_append_n(out, attr_out, s+i, nobb, attr); } - i += nobb; - // tag - if (s[i] == '[') { - i += bbcode_process_tag(bb, s+i, base, out, attr_out, &attr); - } - else if (s[i] == '\\') { - if (s[i+1] == '\\' || s[i+1] == '[') { - attrbuf_append_n(out, attr_out, s+i+1, 1, attr); // escape '\[' and '\\' - i += 2; - } - else { - attrbuf_append_n(out, attr_out, s+i, 1, attr); // pass '\\' as is - i++; - } - } - } - // pop unclosed openings - assert(bb->tags_nesting >= base); - while( bb->tags_nesting > base ) { - bbcode_tag_pop(bb,NULL); - }; -} - -ic_private void bbcode_print( bbcode_t* bb, const char* s ) { - if (bb->out == NULL || bb->out_attrs == NULL || s == NULL) return; - assert(sbuf_len(bb->out) == 0 && attrbuf_len(bb->out_attrs) == 0); - bbcode_append( bb, s, bb->out, bb->out_attrs ); - term_write_formatted( bb->term, sbuf_string(bb->out), attrbuf_attrs(bb->out_attrs,sbuf_len(bb->out)) ); - attrbuf_clear(bb->out_attrs); - sbuf_clear(bb->out); -} - -ic_private void bbcode_println( bbcode_t* bb, const char* s ) { - bbcode_print(bb,s); - term_writeln(bb->term, ""); -} - -ic_private void bbcode_vprintf( bbcode_t* bb, const char* fmt, va_list args ) { - if (bb->vout == NULL || fmt == NULL) return; - assert(sbuf_len(bb->vout) == 0); - sbuf_append_vprintf(bb->vout,fmt,args); - bbcode_print(bb, sbuf_string(bb->vout)); - sbuf_clear(bb->vout); -} - -ic_private void bbcode_printf( bbcode_t* bb, const char* fmt, ... ) { - va_list args; - va_start(args,fmt); - bbcode_vprintf(bb,fmt,args); - va_end(args); -} - -ic_private ssize_t bbcode_column_width( bbcode_t* bb, const char* s ) { - if (s==NULL || s[0] == 0) return 0; - if (bb->vout == NULL) { return str_column_width(s); } - assert(sbuf_len(bb->vout) == 0); - bbcode_append( bb, s, bb->vout, NULL); - const ssize_t w = str_column_width(sbuf_string(bb->vout)); - sbuf_clear(bb->vout); - return w; -} diff --git a/vendor/isocline/bbcode.h b/vendor/isocline/bbcode.h deleted file mode 100644 index be96bfe2..00000000 --- a/vendor/isocline/bbcode.h +++ /dev/null @@ -1,37 +0,0 @@ -/* ---------------------------------------------------------------------------- - Copyright (c) 2021, Daan Leijen - This is free software; you can redistribute it and/or modify it - under the terms of the MIT License. A copy of the license can be - found in the "LICENSE" file at the root of this distribution. ------------------------------------------------------------------------------*/ -#pragma once -#ifndef IC_BBCODE_H -#define IC_BBCODE_H - -#include -#include "common.h" -#include "term.h" - -struct bbcode_s; -typedef struct bbcode_s bbcode_t; - -ic_private bbcode_t* bbcode_new( alloc_t* mem, term_t* term ); -ic_private void bbcode_free( bbcode_t* bb ); - -ic_private void bbcode_style_add( bbcode_t* bb, const char* style_name, attr_t attr ); -ic_private void bbcode_style_def( bbcode_t* bb, const char* style_name, const char* s ); -ic_private void bbcode_style_open( bbcode_t* bb, const char* fmt ); -ic_private void bbcode_style_close( bbcode_t* bb, const char* fmt ); -ic_private attr_t bbcode_style( bbcode_t* bb, const char* style_name ); - -ic_private void bbcode_print( bbcode_t* bb, const char* s ); -ic_private void bbcode_println( bbcode_t* bb, const char* s ); -ic_private void bbcode_printf( bbcode_t* bb, const char* fmt, ... ); -ic_private void bbcode_vprintf( bbcode_t* bb, const char* fmt, va_list args ); - -ic_private ssize_t bbcode_column_width( bbcode_t* bb, const char* s ); - -// allows `attr_out == NULL`. -ic_private void bbcode_append( bbcode_t* bb, const char* s, stringbuf_t* out, attrbuf_t* attr_out ); - -#endif // IC_BBCODE_H diff --git a/vendor/isocline/bbcode_colors.c b/vendor/isocline/bbcode_colors.c deleted file mode 100644 index 245cd3de..00000000 --- a/vendor/isocline/bbcode_colors.c +++ /dev/null @@ -1,194 +0,0 @@ -/* ---------------------------------------------------------------------------- - Copyright (c) 2021, Daan Leijen - This is free software; you can redistribute it and/or modify it - under the terms of the MIT License. A copy of the license can be - found in the "LICENSE" file at the root of this distribution. ------------------------------------------------------------------------------*/ - -// This file is included from "bbcode.c" and contains html color names - -#include "common.h" - -typedef struct style_color_s { - const char* name; - ic_color_t color; -} style_color_t; - -#define IC_HTML_COLOR_COUNT (172) - -// ordered list of HTML color names (so we can use binary search) -static style_color_t html_colors[IC_HTML_COLOR_COUNT+1] = { - { "aliceblue", IC_RGB(0xf0f8ff) }, - { "ansi-aqua", IC_ANSI_AQUA }, - { "ansi-black", IC_ANSI_BLACK }, - { "ansi-blue", IC_ANSI_BLUE }, - { "ansi-cyan", IC_ANSI_CYAN }, - { "ansi-darkgray", IC_ANSI_DARKGRAY }, - { "ansi-darkgrey", IC_ANSI_DARKGRAY }, - { "ansi-default", IC_ANSI_DEFAULT }, - { "ansi-fuchsia", IC_ANSI_FUCHSIA }, - { "ansi-gray", IC_ANSI_GRAY }, - { "ansi-green", IC_ANSI_GREEN }, - { "ansi-grey", IC_ANSI_GRAY }, - { "ansi-lightgray", IC_ANSI_LIGHTGRAY }, - { "ansi-lightgrey", IC_ANSI_LIGHTGRAY }, - { "ansi-lime" , IC_ANSI_LIME }, - { "ansi-magenta", IC_ANSI_MAGENTA }, - { "ansi-maroon", IC_ANSI_MAROON }, - { "ansi-navy", IC_ANSI_NAVY }, - { "ansi-olive", IC_ANSI_OLIVE }, - { "ansi-purple", IC_ANSI_PURPLE }, - { "ansi-red", IC_ANSI_RED }, - { "ansi-silver", IC_ANSI_SILVER }, - { "ansi-teal", IC_ANSI_TEAL }, - { "ansi-white", IC_ANSI_WHITE }, - { "ansi-yellow", IC_ANSI_YELLOW }, - { "antiquewhite", IC_RGB(0xfaebd7) }, - { "aqua", IC_RGB(0x00ffff) }, - { "aquamarine", IC_RGB(0x7fffd4) }, - { "azure", IC_RGB(0xf0ffff) }, - { "beige", IC_RGB(0xf5f5dc) }, - { "bisque", IC_RGB(0xffe4c4) }, - { "black", IC_RGB(0x000000) }, - { "blanchedalmond", IC_RGB(0xffebcd) }, - { "blue", IC_RGB(0x0000ff) }, - { "blueviolet", IC_RGB(0x8a2be2) }, - { "brown", IC_RGB(0xa52a2a) }, - { "burlywood", IC_RGB(0xdeb887) }, - { "cadetblue", IC_RGB(0x5f9ea0) }, - { "chartreuse", IC_RGB(0x7fff00) }, - { "chocolate", IC_RGB(0xd2691e) }, - { "coral", IC_RGB(0xff7f50) }, - { "cornflowerblue", IC_RGB(0x6495ed) }, - { "cornsilk", IC_RGB(0xfff8dc) }, - { "crimson", IC_RGB(0xdc143c) }, - { "cyan", IC_RGB(0x00ffff) }, - { "darkblue", IC_RGB(0x00008b) }, - { "darkcyan", IC_RGB(0x008b8b) }, - { "darkgoldenrod", IC_RGB(0xb8860b) }, - { "darkgray", IC_RGB(0xa9a9a9) }, - { "darkgreen", IC_RGB(0x006400) }, - { "darkgrey", IC_RGB(0xa9a9a9) }, - { "darkkhaki", IC_RGB(0xbdb76b) }, - { "darkmagenta", IC_RGB(0x8b008b) }, - { "darkolivegreen", IC_RGB(0x556b2f) }, - { "darkorange", IC_RGB(0xff8c00) }, - { "darkorchid", IC_RGB(0x9932cc) }, - { "darkred", IC_RGB(0x8b0000) }, - { "darksalmon", IC_RGB(0xe9967a) }, - { "darkseagreen", IC_RGB(0x8fbc8f) }, - { "darkslateblue", IC_RGB(0x483d8b) }, - { "darkslategray", IC_RGB(0x2f4f4f) }, - { "darkslategrey", IC_RGB(0x2f4f4f) }, - { "darkturquoise", IC_RGB(0x00ced1) }, - { "darkviolet", IC_RGB(0x9400d3) }, - { "deeppink", IC_RGB(0xff1493) }, - { "deepskyblue", IC_RGB(0x00bfff) }, - { "dimgray", IC_RGB(0x696969) }, - { "dimgrey", IC_RGB(0x696969) }, - { "dodgerblue", IC_RGB(0x1e90ff) }, - { "firebrick", IC_RGB(0xb22222) }, - { "floralwhite", IC_RGB(0xfffaf0) }, - { "forestgreen", IC_RGB(0x228b22) }, - { "fuchsia", IC_RGB(0xff00ff) }, - { "gainsboro", IC_RGB(0xdcdcdc) }, - { "ghostwhite", IC_RGB(0xf8f8ff) }, - { "gold", IC_RGB(0xffd700) }, - { "goldenrod", IC_RGB(0xdaa520) }, - { "gray", IC_RGB(0x808080) }, - { "green", IC_RGB(0x008000) }, - { "greenyellow", IC_RGB(0xadff2f) }, - { "grey", IC_RGB(0x808080) }, - { "honeydew", IC_RGB(0xf0fff0) }, - { "hotpink", IC_RGB(0xff69b4) }, - { "indianred", IC_RGB(0xcd5c5c) }, - { "indigo", IC_RGB(0x4b0082) }, - { "ivory", IC_RGB(0xfffff0) }, - { "khaki", IC_RGB(0xf0e68c) }, - { "lavender", IC_RGB(0xe6e6fa) }, - { "lavenderblush", IC_RGB(0xfff0f5) }, - { "lawngreen", IC_RGB(0x7cfc00) }, - { "lemonchiffon", IC_RGB(0xfffacd) }, - { "lightblue", IC_RGB(0xadd8e6) }, - { "lightcoral", IC_RGB(0xf08080) }, - { "lightcyan", IC_RGB(0xe0ffff) }, - { "lightgoldenrodyellow", IC_RGB(0xfafad2) }, - { "lightgray", IC_RGB(0xd3d3d3) }, - { "lightgreen", IC_RGB(0x90ee90) }, - { "lightgrey", IC_RGB(0xd3d3d3) }, - { "lightpink", IC_RGB(0xffb6c1) }, - { "lightsalmon", IC_RGB(0xffa07a) }, - { "lightseagreen", IC_RGB(0x20b2aa) }, - { "lightskyblue", IC_RGB(0x87cefa) }, - { "lightslategray", IC_RGB(0x778899) }, - { "lightslategrey", IC_RGB(0x778899) }, - { "lightsteelblue", IC_RGB(0xb0c4de) }, - { "lightyellow", IC_RGB(0xffffe0) }, - { "lime", IC_RGB(0x00ff00) }, - { "limegreen", IC_RGB(0x32cd32) }, - { "linen", IC_RGB(0xfaf0e6) }, - { "magenta", IC_RGB(0xff00ff) }, - { "maroon", IC_RGB(0x800000) }, - { "mediumaquamarine", IC_RGB(0x66cdaa) }, - { "mediumblue", IC_RGB(0x0000cd) }, - { "mediumorchid", IC_RGB(0xba55d3) }, - { "mediumpurple", IC_RGB(0x9370db) }, - { "mediumseagreen", IC_RGB(0x3cb371) }, - { "mediumslateblue", IC_RGB(0x7b68ee) }, - { "mediumspringgreen", IC_RGB(0x00fa9a) }, - { "mediumturquoise", IC_RGB(0x48d1cc) }, - { "mediumvioletred", IC_RGB(0xc71585) }, - { "midnightblue", IC_RGB(0x191970) }, - { "mintcream", IC_RGB(0xf5fffa) }, - { "mistyrose", IC_RGB(0xffe4e1) }, - { "moccasin", IC_RGB(0xffe4b5) }, - { "navajowhite", IC_RGB(0xffdead) }, - { "navy", IC_RGB(0x000080) }, - { "oldlace", IC_RGB(0xfdf5e6) }, - { "olive", IC_RGB(0x808000) }, - { "olivedrab", IC_RGB(0x6b8e23) }, - { "orange", IC_RGB(0xffa500) }, - { "orangered", IC_RGB(0xff4500) }, - { "orchid", IC_RGB(0xda70d6) }, - { "palegoldenrod", IC_RGB(0xeee8aa) }, - { "palegreen", IC_RGB(0x98fb98) }, - { "paleturquoise", IC_RGB(0xafeeee) }, - { "palevioletred", IC_RGB(0xdb7093) }, - { "papayawhip", IC_RGB(0xffefd5) }, - { "peachpuff", IC_RGB(0xffdab9) }, - { "peru", IC_RGB(0xcd853f) }, - { "pink", IC_RGB(0xffc0cb) }, - { "plum", IC_RGB(0xdda0dd) }, - { "powderblue", IC_RGB(0xb0e0e6) }, - { "purple", IC_RGB(0x800080) }, - { "rebeccapurple", IC_RGB(0x663399) }, - { "red", IC_RGB(0xff0000) }, - { "rosybrown", IC_RGB(0xbc8f8f) }, - { "royalblue", IC_RGB(0x4169e1) }, - { "saddlebrown", IC_RGB(0x8b4513) }, - { "salmon", IC_RGB(0xfa8072) }, - { "sandybrown", IC_RGB(0xf4a460) }, - { "seagreen", IC_RGB(0x2e8b57) }, - { "seashell", IC_RGB(0xfff5ee) }, - { "sienna", IC_RGB(0xa0522d) }, - { "silver", IC_RGB(0xc0c0c0) }, - { "skyblue", IC_RGB(0x87ceeb) }, - { "slateblue", IC_RGB(0x6a5acd) }, - { "slategray", IC_RGB(0x708090) }, - { "slategrey", IC_RGB(0x708090) }, - { "snow", IC_RGB(0xfffafa) }, - { "springgreen", IC_RGB(0x00ff7f) }, - { "steelblue", IC_RGB(0x4682b4) }, - { "tan", IC_RGB(0xd2b48c) }, - { "teal", IC_RGB(0x008080) }, - { "thistle", IC_RGB(0xd8bfd8) }, - { "tomato", IC_RGB(0xff6347) }, - { "turquoise", IC_RGB(0x40e0d0) }, - { "violet", IC_RGB(0xee82ee) }, - { "wheat", IC_RGB(0xf5deb3) }, - { "white", IC_RGB(0xffffff) }, - { "whitesmoke", IC_RGB(0xf5f5f5) }, - { "yellow", IC_RGB(0xffff00) }, - { "yellowgreen", IC_RGB(0x9acd32) }, - {NULL, 0} -}; diff --git a/vendor/isocline/common.c b/vendor/isocline/common.c deleted file mode 100644 index 1d9fb566..00000000 --- a/vendor/isocline/common.c +++ /dev/null @@ -1,347 +0,0 @@ -/* ---------------------------------------------------------------------------- - Copyright (c) 2021, Daan Leijen - This is free software; you can redistribute it and/or modify it - under the terms of the MIT License. A copy of the license can be - found in the "LICENSE" file at the root of this distribution. ------------------------------------------------------------------------------*/ - -#include -#include -#include -#include -#include "common.h" - - -//------------------------------------------------------------- -// String wrappers for ssize_t -//------------------------------------------------------------- - -ic_private ssize_t ic_strlen( const char* s ) { - if (s==NULL) return 0; - return to_ssize_t(strlen(s)); -} - -ic_private void ic_memmove( void* dest, const void* src, ssize_t n ) { - assert(dest!=NULL && src != NULL); - if (n <= 0) return; - memmove(dest,src,to_size_t(n)); -} - - -ic_private void ic_memcpy( void* dest, const void* src, ssize_t n ) { - assert(dest!=NULL && src != NULL); - if (dest == NULL || src == NULL || n <= 0) return; - memcpy(dest,src,to_size_t(n)); -} - -ic_private void ic_memset(void* dest, uint8_t value, ssize_t n) { - assert(dest!=NULL); - if (dest == NULL || n <= 0) return; - memset(dest,(int8_t)value,to_size_t(n)); -} - -ic_private bool ic_memnmove( void* dest, ssize_t dest_size, const void* src, ssize_t n ) { - assert(dest!=NULL && src != NULL); - if (n <= 0) return true; - if (dest_size < n) { assert(false); return false; } - memmove(dest,src,to_size_t(n)); - return true; -} - -ic_private bool ic_strcpy( char* dest, ssize_t dest_size /* including 0 */, const char* src) { - assert(dest!=NULL && src != NULL); - if (dest == NULL || dest_size <= 0) return false; - ssize_t slen = ic_strlen(src); - if (slen >= dest_size) return false; - strcpy(dest,src); - assert(dest[slen] == 0); - return true; -} - - -ic_private bool ic_strncpy( char* dest, ssize_t dest_size /* including 0 */, const char* src, ssize_t n) { - assert(dest!=NULL && n < dest_size); - if (dest == NULL || dest_size <= 0) return false; - if (n >= dest_size) return false; - if (src==NULL || n <= 0) { - dest[0] = 0; - } - else { - strncpy(dest,src,to_size_t(n)); - dest[n] = 0; - } - return true; -} - -//------------------------------------------------------------- -// String matching -//------------------------------------------------------------- - -ic_public bool ic_starts_with( const char* s, const char* prefix ) { - if (s==prefix) return true; - if (prefix==NULL) return true; - if (s==NULL) return false; - - ssize_t i; - for( i = 0; s[i] != 0 && prefix[i] != 0; i++) { - if (s[i] != prefix[i]) return false; - } - return (prefix[i] == 0); -} - -ic_private char ic_tolower( char c ) { - return (c >= 'A' && c <= 'Z' ? c - 'A' + 'a' : c); -} - -ic_private void ic_str_tolower(char* s) { - while(*s != 0) { - *s = ic_tolower(*s); - s++; - } -} - -ic_public bool ic_istarts_with( const char* s, const char* prefix ) { - if (s==prefix) return true; - if (prefix==NULL) return true; - if (s==NULL) return false; - - ssize_t i; - for( i = 0; s[i] != 0 && prefix[i] != 0; i++) { - if (ic_tolower(s[i]) != ic_tolower(prefix[i])) return false; - } - return (prefix[i] == 0); -} - - -ic_private int ic_strnicmp(const char* s1, const char* s2, ssize_t n) { - if (s1 == NULL && s2 == NULL) return 0; - if (s1 == NULL) return -1; - if (s2 == NULL) return 1; - ssize_t i; - for (i = 0; s1[i] != 0 && i < n; i++) { // note: if s2[i] == 0 the loop will stop as c1 != c2 - char c1 = ic_tolower(s1[i]); - char c2 = ic_tolower(s2[i]); - if (c1 < c2) return -1; - if (c1 > c2) return 1; - } - return ((i >= n || s2[i] == 0) ? 0 : -1); -} - -ic_private int ic_stricmp(const char* s1, const char* s2) { - ssize_t len1 = ic_strlen(s1); - ssize_t len2 = ic_strlen(s2); - if (len1 < len2) return -1; - if (len1 > len2) return 1; - return (ic_strnicmp(s1, s2, (len1 >= len2 ? len1 : len2))); -} - - -static const char* ic_stristr(const char* s, const char* pat) { - if (s==NULL) return NULL; - if (pat==NULL || pat[0] == 0) return s; - ssize_t patlen = ic_strlen(pat); - for (ssize_t i = 0; s[i] != 0; i++) { - if (ic_strnicmp(s + i, pat, patlen) == 0) return (s+i); - } - return NULL; -} - -ic_private bool ic_contains(const char* big, const char* s) { - if (big == NULL) return false; - if (s == NULL) return true; - return (strstr(big,s) != NULL); -} - -ic_private bool ic_icontains(const char* big, const char* s) { - if (big == NULL) return false; - if (s == NULL) return true; - return (ic_stristr(big,s) != NULL); -} - - -//------------------------------------------------------------- -// Unicode -// QUTF-8: See -// Raw bytes are code points 0xEE000 - 0xEE0FF -//------------------------------------------------------------- -#define IC_UNICODE_RAW ((unicode_t)(0xEE000U)) - -ic_private unicode_t unicode_from_raw(uint8_t c) { - return (IC_UNICODE_RAW + c); -} - -ic_private bool unicode_is_raw(unicode_t u, uint8_t* c) { - if (u >= IC_UNICODE_RAW && u <= IC_UNICODE_RAW + 0xFF) { - *c = (uint8_t)(u - IC_UNICODE_RAW); - return true; - } - else { - return false; - } -} - -ic_private void unicode_to_qutf8(unicode_t u, uint8_t buf[5]) { - memset(buf, 0, 5); - if (u <= 0x7F) { - buf[0] = (uint8_t)u; - } - else if (u <= 0x07FF) { - buf[0] = (0xC0 | ((uint8_t)(u >> 6))); - buf[1] = (0x80 | (((uint8_t)u) & 0x3F)); - } - else if (u <= 0xFFFF) { - buf[0] = (0xE0 | ((uint8_t)(u >> 12))); - buf[1] = (0x80 | (((uint8_t)(u >> 6)) & 0x3F)); - buf[2] = (0x80 | (((uint8_t)u) & 0x3F)); - } - else if (u <= 0x10FFFF) { - if (unicode_is_raw(u, &buf[0])) { - buf[1] = 0; - } - else { - buf[0] = (0xF0 | ((uint8_t)(u >> 18))); - buf[1] = (0x80 | (((uint8_t)(u >> 12)) & 0x3F)); - buf[2] = (0x80 | (((uint8_t)(u >> 6)) & 0x3F)); - buf[3] = (0x80 | (((uint8_t)u) & 0x3F)); - } - } -} - -// is this a utf8 continuation byte? -ic_private bool utf8_is_cont(uint8_t c) { - return ((c & 0xC0) == 0x80); -} - -ic_private unicode_t unicode_from_qutf8(const uint8_t* s, ssize_t len, ssize_t* count) { - unicode_t c0 = 0; - if (len <= 0 || s == NULL) { - goto fail; - } - // 1 byte - c0 = s[0]; - if (c0 <= 0x7F && len >= 1) { - if (count != NULL) *count = 1; - return c0; - } - else if (c0 <= 0xC1) { // invalid continuation byte or invalid 0xC0, 0xC1 - goto fail; - } - // 2 bytes - else if (c0 <= 0xDF && len >= 2 && utf8_is_cont(s[1])) { - if (count != NULL) *count = 2; - return (((c0 & 0x1F) << 6) | (s[1] & 0x3F)); - } - // 3 bytes: reject overlong and surrogate halves - else if (len >= 3 && - ((c0 == 0xE0 && s[1] >= 0xA0 && s[1] <= 0xBF && utf8_is_cont(s[2])) || - (c0 >= 0xE1 && c0 <= 0xEC && utf8_is_cont(s[1]) && utf8_is_cont(s[2])) - )) - { - if (count != NULL) *count = 3; - return (((c0 & 0x0F) << 12) | ((unicode_t)(s[1] & 0x3F) << 6) | (s[2] & 0x3F)); - } - // 4 bytes: reject overlong - else if (len >= 4 && - (((c0 == 0xF0 && s[1] >= 0x90 && s[1] <= 0xBF && utf8_is_cont(s[2]) && utf8_is_cont(s[3])) || - (c0 >= 0xF1 && c0 <= 0xF3 && utf8_is_cont(s[1]) && utf8_is_cont(s[2]) && utf8_is_cont(s[3])) || - (c0 == 0xF4 && s[1] >= 0x80 && s[1] <= 0x8F && utf8_is_cont(s[2]) && utf8_is_cont(s[3]))) - )) - { - if (count != NULL) *count = 4; - return (((c0 & 0x07) << 18) | ((unicode_t)(s[1] & 0x3F) << 12) | ((unicode_t)(s[2] & 0x3F) << 6) | (s[3] & 0x3F)); - } -fail: - if (count != NULL) *count = 1; - return unicode_from_raw(s[0]); -} - - -//------------------------------------------------------------- -// Debug -//------------------------------------------------------------- - -#if defined(IC_NO_DEBUG_MSG) -// nothing -#elif !defined(IC_DEBUG_TO_FILE) -ic_private void debug_msg(const char* fmt, ...) { - if (getenv("ISOCLINE_DEBUG")) { - va_list args; - va_start(args, fmt); - vfprintf(stderr, fmt, args); - va_end(args); - } -} -#else -ic_private void debug_msg(const char* fmt, ...) { - static int debug_init; - static const char* debug_fname = "isocline.debug.txt"; - // initialize? - if (debug_init==0) { - debug_init = -1; - const char* rdebug = getenv("ISOCLINE_DEBUG"); - if (rdebug!=NULL && strcmp(rdebug,"1") == 0) { - FILE* fdbg = fopen(debug_fname, "w"); - if (fdbg!=NULL) { - debug_init = 1; - fclose(fdbg); - } - } - } - if (debug_init <= 0) return; - - // write debug messages - FILE* fdbg = fopen(debug_fname, "a"); - if (fdbg==NULL) return; - va_list args; - va_start(args, fmt); - vfprintf(fdbg, fmt, args); - fclose(fdbg); - va_end(args); -} -#endif - - -//------------------------------------------------------------- -// Allocation -//------------------------------------------------------------- - -ic_private void* mem_malloc(alloc_t* mem, ssize_t sz) { - return mem->malloc(to_size_t(sz)); -} - -ic_private void* mem_zalloc(alloc_t* mem, ssize_t sz) { - void* p = mem_malloc(mem, sz); - if (p != NULL) memset(p, 0, to_size_t(sz)); - return p; -} - -ic_private void* mem_realloc(alloc_t* mem, void* p, ssize_t newsz) { - return mem->realloc(p, to_size_t(newsz)); -} - -ic_private void mem_free(alloc_t* mem, const void* p) { - mem->free((void*)p); -} - -ic_private char* mem_strdup(alloc_t* mem, const char* s) { - if (s==NULL) return NULL; - ssize_t n = ic_strlen(s); - char* p = mem_malloc_tp_n(mem, char, n+1); - if (p == NULL) return NULL; - ic_memcpy(p, s, n+1); - return p; -} - -ic_private char* mem_strndup(alloc_t* mem, const char* s, ssize_t n) { - if (s==NULL || n < 0) return NULL; - char* p = mem_malloc_tp_n(mem, char, n+1); - if (p == NULL) return NULL; - ssize_t i; - for (i = 0; i < n && s[i] != 0; i++) { - p[i] = s[i]; - } - assert(i <= n); - p[i] = 0; - return p; -} - diff --git a/vendor/isocline/common.h b/vendor/isocline/common.h deleted file mode 100644 index d03def57..00000000 --- a/vendor/isocline/common.h +++ /dev/null @@ -1,187 +0,0 @@ -/* ---------------------------------------------------------------------------- - Copyright (c) 2021, Daan Leijen - This is free software; you can redistribute it and/or modify it - under the terms of the MIT License. A copy of the license can be - found in the "LICENSE" file at the root of this distribution. ------------------------------------------------------------------------------*/ - -#pragma once -#ifndef IC_COMMON_H -#define IC_COMMON_H - -//------------------------------------------------------------- -// Headers and defines -//------------------------------------------------------------- - -#include // ssize_t -#include -#include -#include -#include -#include -#include "./isocline.h" // ic_malloc_fun_t, ic_color_t etc. - -# ifdef __cplusplus -# define ic_extern_c extern "C" -# else -# define ic_extern_c -# endif - -#if defined(IC_SEPARATE_OBJS) -# define ic_public ic_extern_c -# if defined(__GNUC__) // includes clang and icc -# define ic_private __attribute__((visibility("hidden"))) -# else -# define ic_private -# endif -#else -# define ic_private static inline -# define ic_public ic_extern_c -#endif - -#define ic_unused(x) (void)(x) - - -//------------------------------------------------------------- -// ssize_t -//------------------------------------------------------------- - -#if defined(_MSC_VER) -typedef intptr_t ssize_t; -#endif - -#define ssizeof(tp) (ssize_t)(sizeof(tp)) -static inline size_t to_size_t(ssize_t sz) { return (sz >= 0 ? (size_t)sz : 0); } -static inline ssize_t to_ssize_t(size_t sz) { return (sz <= SIZE_MAX/2 ? (ssize_t)sz : 0); } - -ic_private void ic_memmove(void* dest, const void* src, ssize_t n); -ic_private void ic_memcpy(void* dest, const void* src, ssize_t n); -ic_private void ic_memset(void* dest, uint8_t value, ssize_t n); -ic_private bool ic_memnmove(void* dest, ssize_t dest_size, const void* src, ssize_t n); - -ic_private ssize_t ic_strlen(const char* s); -ic_private bool ic_strcpy(char* dest, ssize_t dest_size /* including 0 */, const char* src); -ic_private bool ic_strncpy(char* dest, ssize_t dest_size /* including 0 */, const char* src, ssize_t n); - -ic_private bool ic_contains(const char* big, const char* s); -ic_private bool ic_icontains(const char* big, const char* s); -ic_private char ic_tolower(char c); -ic_private void ic_str_tolower(char* s); -ic_private int ic_stricmp(const char* s1, const char* s2); -ic_private int ic_strnicmp(const char* s1, const char* s2, ssize_t n); - - - -//--------------------------------------------------------------------- -// Unicode -// -// We use "qutf-8" (quite like utf-8) encoding and decoding. -// Internally we always use valid utf-8. If we encounter invalid -// utf-8 bytes (or bytes >= 0x80 from any other encoding) we encode -// these as special code points in the "raw plane" (0xEE000 - 0xEE0FF). -// When decoding we are then able to restore such raw bytes as-is. -// See -//--------------------------------------------------------------------- - -typedef uint32_t unicode_t; - -ic_private void unicode_to_qutf8(unicode_t u, uint8_t buf[5]); -ic_private unicode_t unicode_from_qutf8(const uint8_t* s, ssize_t len, ssize_t* nread); // validating - -ic_private unicode_t unicode_from_raw(uint8_t c); -ic_private bool unicode_is_raw(unicode_t u, uint8_t* c); - -ic_private bool utf8_is_cont(uint8_t c); - - -//------------------------------------------------------------- -// Colors -//------------------------------------------------------------- - -// A color is either RGB or an ANSI code. -// (RGB colors have bit 24 set to distinguish them from the ANSI color palette colors.) -// (Isocline will automatically convert from RGB on terminals that do not support full colors) -typedef uint32_t ic_color_t; - -// Create a color from a 24-bit color value. -ic_private ic_color_t ic_rgb(uint32_t hex); - -// Create a color from a 8-bit red/green/blue components. -// The value of each component is capped between 0 and 255. -ic_private ic_color_t ic_rgbx(ssize_t r, ssize_t g, ssize_t b); - -#define IC_COLOR_NONE (0) -#define IC_RGB(rgb) (0x1000000 | (uint32_t)(rgb)) // ic_rgb(rgb) // define to it can be used as a constant - -// ANSI colors. -// The actual colors used is usually determined by the terminal theme -// See -#define IC_ANSI_BLACK (30) -#define IC_ANSI_MAROON (31) -#define IC_ANSI_GREEN (32) -#define IC_ANSI_OLIVE (33) -#define IC_ANSI_NAVY (34) -#define IC_ANSI_PURPLE (35) -#define IC_ANSI_TEAL (36) -#define IC_ANSI_SILVER (37) -#define IC_ANSI_DEFAULT (39) - -#define IC_ANSI_GRAY (90) -#define IC_ANSI_RED (91) -#define IC_ANSI_LIME (92) -#define IC_ANSI_YELLOW (93) -#define IC_ANSI_BLUE (94) -#define IC_ANSI_FUCHSIA (95) -#define IC_ANSI_AQUA (96) -#define IC_ANSI_WHITE (97) - -#define IC_ANSI_DARKGRAY IC_ANSI_GRAY -#define IC_ANSI_LIGHTGRAY IC_ANSI_SILVER -#define IC_ANSI_MAGENTA IC_ANSI_FUCHSIA -#define IC_ANSI_CYAN IC_ANSI_AQUA - - - -//------------------------------------------------------------- -// Debug -//------------------------------------------------------------- - -#if defined(IC_NO_DEBUG_MSG) -#define debug_msg(fmt,...) (void)(0) -#else -ic_private void debug_msg( const char* fmt, ... ); -#endif - - -//------------------------------------------------------------- -// Abstract environment -//------------------------------------------------------------- -struct ic_env_s; -typedef struct ic_env_s ic_env_t; - - -//------------------------------------------------------------- -// Allocation -//------------------------------------------------------------- - -typedef struct alloc_s { - ic_malloc_fun_t* malloc; - ic_realloc_fun_t* realloc; - ic_free_fun_t* free; -} alloc_t; - - -ic_private void* mem_malloc( alloc_t* mem, ssize_t sz ); -ic_private void* mem_zalloc( alloc_t* mem, ssize_t sz ); -ic_private void* mem_realloc( alloc_t* mem, void* p, ssize_t newsz ); -ic_private void mem_free( alloc_t* mem, const void* p ); -ic_private char* mem_strdup( alloc_t* mem, const char* s); -ic_private char* mem_strndup( alloc_t* mem, const char* s, ssize_t n); - -#define mem_zalloc_tp(mem,tp) (tp*)mem_zalloc(mem,ssizeof(tp)) -#define mem_malloc_tp_n(mem,tp,n) (tp*)mem_malloc(mem,(n)*ssizeof(tp)) -#define mem_zalloc_tp_n(mem,tp,n) (tp*)mem_zalloc(mem,(n)*ssizeof(tp)) -#define mem_realloc_tp(mem,tp,p,n) (tp*)mem_realloc(mem,p,(n)*ssizeof(tp)) - - -#endif // IC_COMMON_H diff --git a/vendor/isocline/completers.c b/vendor/isocline/completers.c deleted file mode 100644 index f2276792..00000000 --- a/vendor/isocline/completers.c +++ /dev/null @@ -1,675 +0,0 @@ -/* ---------------------------------------------------------------------------- - Copyright (c) 2021, Daan Leijen - This is free software; you can redistribute it and/or modify it - under the terms of the MIT License. A copy of the license can be - found in the "LICENSE" file at the root of this distribution. ------------------------------------------------------------------------------*/ -#include -#include - -#include "./isocline.h" -#include "common.h" -#include "env.h" -#include "stringbuf.h" -#include "completions.h" - - - -//------------------------------------------------------------- -// Word completion -//------------------------------------------------------------- - -// free variables for word completion -typedef struct word_closure_s { - long delete_before_adjust; - void* prev_env; - ic_completion_fun_t* prev_complete; -} word_closure_t; - - -// word completion callback -static bool token_add_completion_ex(ic_env_t* env, void* closure, const char* replacement, const char* display, const char* help, long delete_before, long delete_after) { - word_closure_t* wenv = (word_closure_t*)(closure); - // call the previous completer with an adjusted delete-before - return (*wenv->prev_complete)(env, wenv->prev_env, replacement, display, help, wenv->delete_before_adjust + delete_before, delete_after); -} - - -ic_public void ic_complete_word(ic_completion_env_t* cenv, const char* prefix, ic_completer_fun_t* fun, - ic_is_char_class_fun_t* is_word_char) -{ - if (is_word_char == NULL) is_word_char = &ic_char_is_nonseparator; - - ssize_t len = ic_strlen(prefix); - ssize_t pos = len; // will be start of the 'word' (excluding a potential start quote) - while (pos > 0) { - // go back one code point - ssize_t ofs = str_prev_ofs(prefix, pos, NULL); - if (ofs <= 0) break; - if (!(*is_word_char)(prefix + (pos - ofs), (long)ofs)) { - break; - } - pos -= ofs; - } - if (pos < 0) { pos = 0; } - - // stop if empty word - // if (len == pos) return; - - // set up the closure - word_closure_t wenv; - wenv.delete_before_adjust = (long)(len - pos); - wenv.prev_complete = cenv->complete; - wenv.prev_env = cenv->env; - cenv->complete = &token_add_completion_ex; - cenv->closure = &wenv; - - // and call the user completion routine - (*fun)(cenv, prefix + pos); - - // restore the original environment - cenv->complete = wenv.prev_complete; - cenv->closure = wenv.prev_env; -} - - -//------------------------------------------------------------- -// Quoted word completion (with escape characters) -//------------------------------------------------------------- - -// free variables for word completion -typedef struct qword_closure_s { - char escape_char; - char quote; - long delete_before_adjust; - stringbuf_t* sbuf; - void* prev_env; - ic_is_char_class_fun_t* is_word_char; - ic_completion_fun_t* prev_complete; -} qword_closure_t; - - -// word completion callback -static bool qword_add_completion_ex(ic_env_t* env, void* closure, const char* replacement, const char* display, const char* help, - long delete_before, long delete_after) { - qword_closure_t* wenv = (qword_closure_t*)(closure); - sbuf_replace( wenv->sbuf, replacement ); - if (wenv->quote != 0) { - // add end quote - sbuf_append_char( wenv->sbuf, wenv->quote); - } - else { - // escape non-word characters if it was not quoted - ssize_t pos = 0; - ssize_t next; - while ( (next = sbuf_next_ofs(wenv->sbuf, pos, NULL)) > 0 ) - { - if (!(*wenv->is_word_char)(sbuf_string(wenv->sbuf) + pos, (long)next)) { // strchr(wenv->non_word_char, sbuf_char_at( wenv->sbuf, pos )) != NULL) { - sbuf_insert_char_at( wenv->sbuf, wenv->escape_char, pos); - pos++; - } - pos += next; - } - } - // and call the previous completion function - return (*wenv->prev_complete)( env, wenv->prev_env, sbuf_string(wenv->sbuf), display, help, wenv->delete_before_adjust + delete_before, delete_after ); -} - - -ic_public void ic_complete_qword( ic_completion_env_t* cenv, const char* prefix, ic_completer_fun_t* fun, ic_is_char_class_fun_t* is_word_char ) { - ic_complete_qword_ex( cenv, prefix, fun, is_word_char, '\\', NULL); -} - - -ic_public void ic_complete_qword_ex( ic_completion_env_t* cenv, const char* prefix, ic_completer_fun_t* fun, - ic_is_char_class_fun_t* is_word_char, char escape_char, const char* quote_chars ) { - if (is_word_char == NULL) is_word_char = &ic_char_is_nonseparator ; - if (quote_chars == NULL) quote_chars = "'\""; - - ssize_t len = ic_strlen(prefix); - ssize_t pos; // will be start of the 'word' (excluding a potential start quote) - char quote = 0; - ssize_t quote_len = 0; - - // 1. look for a starting quote - if (quote_chars[0] != 0) { - // we go forward and count all quotes; if it is uneven, we need to complete quoted. - ssize_t qpos_open = -1; - ssize_t qpos_close = -1; - ssize_t qcount = 0; - pos = 0; - while(pos < len) { - if (prefix[pos] == escape_char && prefix[pos+1] != 0 && - !(*is_word_char)(prefix + pos + 1, 1)) // strchr(non_word_char, prefix[pos+1]) != NULL - { - pos++; // skip escape and next char - } - else if (qcount % 2 == 0 && strchr(quote_chars, prefix[pos]) != NULL) { - // open quote - qpos_open = pos; - quote = prefix[pos]; - qcount++; - } - else if (qcount % 2 == 1 && prefix[pos] == quote) { - // close quote - qpos_close = pos; - qcount++; - } - else if (!(*is_word_char)(prefix + pos, 1)) { // strchr(non_word_char, prefix[pos]) != NULL) { - qpos_close = -1; - } - ssize_t ofs = str_next_ofs( prefix, len, pos, NULL ); - if (ofs <= 0) break; - pos += ofs; - } - if ((qcount % 2 == 0 && qpos_close >= 0) || // if the last quote is only followed by word chars, we still complete it - (qcount % 2 == 1)) // opening quote found - { - quote_len = (len - qpos_open - 1); - pos = qpos_open + 1; // pos points to the word start just after the quote. - } - else { - quote = 0; - } - } - - // 2. if we did not find a quoted word, look for non-word-chars - if (quote == 0) { - pos = len; - while(pos > 0) { - // go back one code point - ssize_t ofs = str_prev_ofs(prefix, pos, NULL ); - if (ofs <= 0) break; - if (!(*is_word_char)(prefix + (pos - ofs), (long)ofs)) { // strchr(non_word_char, prefix[pos - ofs]) != NULL) { - // non word char, break if it is not escaped - if (pos <= ofs || prefix[pos - ofs - 1] != escape_char) break; - // otherwise go on - pos--; // skip escaped char - } - pos -= ofs; - } - } - - // stop if empty word - // if (len == pos) return; - - // allocate new unescaped word prefix - char* word = mem_strndup( cenv->env->mem, prefix + pos, (quote==0 ? len - pos : quote_len)); - if (word == NULL) return; - - if (quote == 0) { - // unescape prefix - ssize_t wlen = len - pos; - ssize_t wpos = 0; - while (wpos < wlen) { - ssize_t ofs = str_next_ofs(word, wlen, wpos, NULL); - if (ofs <= 0) break; - if (word[wpos] == escape_char && word[wpos+1] != 0 && - !(*is_word_char)(word + wpos + 1, (long)ofs)) // strchr(non_word_char, word[wpos+1]) != NULL) { - { - ic_memmove(word + wpos, word + wpos + 1, wlen - wpos /* including 0 */); - } - wpos += ofs; - } - } - #ifdef _WIN32 - else { - // remove inner quote: "c:\Program Files\"Win - ssize_t wlen = len - pos; - ssize_t wpos = 0; - while (wpos < wlen) { - ssize_t ofs = str_next_ofs(word, wlen, wpos, NULL); - if (ofs <= 0) break; - if (word[wpos] == escape_char && word[wpos+1] == quote) { - word[wpos+1] = escape_char; - ic_memmove(word + wpos, word + wpos + 1, wlen - wpos /* including 0 */); - } - wpos += ofs; - } - } - #endif - - // set up the closure - qword_closure_t wenv; - wenv.quote = quote; - wenv.is_word_char = is_word_char; - wenv.escape_char = escape_char; - wenv.delete_before_adjust = (long)(len - pos); - wenv.prev_complete = cenv->complete; - wenv.prev_env = cenv->env; - wenv.sbuf = sbuf_new(cenv->env->mem); - if (wenv.sbuf == NULL) { mem_free(cenv->env->mem, word); return; } - cenv->complete = &qword_add_completion_ex; - cenv->closure = &wenv; - - // and call the user completion routine - (*fun)( cenv, word ); - - // restore the original environment - cenv->complete = wenv.prev_complete; - cenv->closure = wenv.prev_env; - - sbuf_free(wenv.sbuf); - mem_free(cenv->env->mem, word); -} - - - - -//------------------------------------------------------------- -// Complete file names -// Listing files -//------------------------------------------------------------- -#include - -typedef enum file_type_e { - // must follow BSD style LSCOLORS order - FT_DEFAULT = 0, - FT_DIR, - FT_SYM, - FT_SOCK, - FT_PIPE, - FT_BLOCK, - FT_CHAR, - FT_SETUID, - FT_SETGID, - FT_DIR_OW_STICKY, - FT_DIR_OW, - FT_DIR_STICKY, - FT_EXE, - FT_LAST -} file_type_t; - -static int cli_color; // 1 enabled, 0 not initialized, -1 disabled -static const char* lscolors = "exfxcxdxbxegedabagacad"; // default BSD setting -static const char* ls_colors; -static const char* ls_colors_names[] = { "no=","di=","ln=","so=","pi=","bd=","cd=","su=","sg=","tw=","ow=","st=","ex=", NULL }; - -static bool ls_colors_init(void) { - if (cli_color != 0) return (cli_color >= 1); - // colors enabled? - const char* s = getenv("CLICOLOR"); - if (s==NULL || (strcmp(s, "1")!=0 && strcmp(s, "") != 0)) { - cli_color = -1; - return false; - } - cli_color = 1; - s = getenv("LS_COLORS"); - if (s != NULL) { ls_colors = s; } - s = getenv("LSCOLORS"); - if (s != NULL) { lscolors = s; } - return true; -} - -static bool ls_valid_esc(ssize_t c) { - return ((c==0 || c==1 || c==4 || c==7 || c==22 || c==24 || c==27) || - (c >= 30 && c <= 37) || (c >= 40 && c <= 47) || - (c >= 90 && c <= 97) || (c >= 100 && c <= 107)); -} - -static bool ls_colors_from_key(stringbuf_t* sb, const char* key) { - // find key - ssize_t keylen = ic_strlen(key); - if (keylen <= 0) return false; - const char* p = strstr(ls_colors, key); - if (p == NULL) return false; - p += keylen; - if (key[keylen-1] != '=') { - if (*p != '=') return false; - p++; - } - ssize_t len = 0; - while (p[len] != 0 && p[len] != ':') { - len++; - } - if (len <= 0) return false; - sbuf_append(sb, "[ansi-sgr=\"" ); - sbuf_append_n(sb, p, len ); - sbuf_append(sb, "\"]"); - return true; -} - -static int ls_colors_from_char(char c) { - if (c >= 'a' && c <= 'h') { return (c - 'a'); } - else if (c >= 'A' && c <= 'H') { return (c - 'A') + 8; } - else if (c == 'x') { return 256; } - else return 256; // default -} - -static bool ls_colors_append(stringbuf_t* sb, file_type_t ft, const char* ext) { - if (!ls_colors_init()) return false; - if (ls_colors != NULL) { - // GNU style - if (ft == FT_DEFAULT && ext != NULL) { - // first try extension match - if (ls_colors_from_key(sb, ext)) return true; - } - if (ft >= FT_DEFAULT && ft < FT_LAST) { - // then a filetype match - const char* key = ls_colors_names[ft]; - if (ls_colors_from_key(sb, key)) return true; - } - } - else if (lscolors != NULL) { - // BSD style - char fg = 'x'; - char bg = 'x'; - if (ic_strlen(lscolors) > (2*(ssize_t)ft)+1) { - fg = lscolors[2*ft]; - bg = lscolors[2*ft + 1]; - } - sbuf_appendf(sb, "[ansi-color=%d ansi-bgcolor=%d]", ls_colors_from_char(fg), ls_colors_from_char(bg) ); - return true; - } - return false; -} - -static void ls_colorize(bool no_lscolor, stringbuf_t* sb, file_type_t ft, const char* name, const char* ext, char dirsep) { - bool close = (no_lscolor ? false : ls_colors_append( sb, ft, ext)); - sbuf_append(sb, "[!pre]" ); - sbuf_append(sb, name); - if (dirsep != 0) sbuf_append_char(sb, dirsep); - sbuf_append(sb,"[/pre]" ); - if (close) { sbuf_append(sb, "[/]"); } -} - -#if defined(_WIN32) -#include -#include - -static bool os_is_dir(const char* cpath) { - struct _stat64 st = { 0 }; - _stat64(cpath, &st); - return ((st.st_mode & _S_IFDIR) != 0); -} - -static file_type_t os_get_filetype(const char* cpath) { - struct _stat64 st = { 0 }; - _stat64(cpath, &st); - if (((st.st_mode) & _S_IFDIR) != 0) return FT_DIR; - if (((st.st_mode) & _S_IFCHR) != 0) return FT_CHAR; - if (((st.st_mode) & _S_IFIFO) != 0) return FT_PIPE; - if (((st.st_mode) & _S_IEXEC) != 0) return FT_EXE; - return FT_DEFAULT; -} - - -#define dir_cursor intptr_t -#define dir_entry struct __finddata64_t - -static bool os_findfirst(alloc_t* mem, const char* path, dir_cursor* d, dir_entry* entry) { - stringbuf_t* spath = sbuf_new(mem); - if (spath == NULL) return false; - sbuf_append(spath, path); - sbuf_append(spath, "\\*"); - *d = _findfirsti64(sbuf_string(spath), entry); - mem_free(mem,spath); - return (*d != -1); -} - -static bool os_findnext(dir_cursor d, dir_entry* entry) { - return (_findnexti64(d, entry) == 0); -} - -static void os_findclose(dir_cursor d) { - _findclose(d); -} - -static const char* os_direntry_name(dir_entry* entry) { - return entry->name; -} - -static bool os_path_is_absolute( const char* path ) { - if (path != NULL && path[0] != 0 && path[1] == ':' && (path[2] == '\\' || path[2] == '/' || path[2] == 0)) { - char drive = path[0]; - return ((drive >= 'A' && drive <= 'Z') || (drive >= 'a' && drive <= 'z')); - } - else return false; -} - -ic_private char ic_dirsep(void) { - return '\\'; -} -#else - -#include -#include -#include -#include - -static bool os_is_dir(const char* cpath) { - struct stat st; - memset(&st, 0, sizeof(st)); - stat(cpath, &st); - return (S_ISDIR(st.st_mode)); -} - -static file_type_t os_get_filetype(const char* cpath) { - struct stat st; - memset(&st, 0, sizeof(st)); - lstat(cpath, &st); - switch ((st.st_mode)&S_IFMT) { - case S_IFSOCK: return FT_SOCK; - case S_IFLNK: { - return FT_SYM; - } - case S_IFIFO: return FT_PIPE; - case S_IFCHR: return FT_CHAR; - case S_IFBLK: return FT_BLOCK; - case S_IFDIR: { - if ((st.st_mode & S_ISUID) != 0) return FT_SETUID; - if ((st.st_mode & S_ISGID) != 0) return FT_SETGID; - if ((st.st_mode & S_IWGRP) != 0 && (st.st_mode & S_ISVTX) != 0) return FT_DIR_OW_STICKY; - if ((st.st_mode & S_IWGRP)) return FT_DIR_OW; - if ((st.st_mode & S_ISVTX)) return FT_DIR_STICKY; - return FT_DIR; - } - case S_IFREG: - default: { - if ((st.st_mode & S_IXUSR) != 0) return FT_EXE; - return FT_DEFAULT; - } - } -} - - -#define dir_cursor DIR* -#define dir_entry struct dirent* - -static bool os_findnext(dir_cursor d, dir_entry* entry) { - *entry = readdir(d); - return (*entry != NULL); -} - -static bool os_findfirst(alloc_t* mem, const char* cpath, dir_cursor* d, dir_entry* entry) { - ic_unused(mem); - *d = opendir(cpath); - if (*d == NULL) { - return false; - } - else { - return os_findnext(*d, entry); - } -} - -static void os_findclose(dir_cursor d) { - closedir(d); -} - -static const char* os_direntry_name(dir_entry* entry) { - return (*entry)->d_name; -} - -static bool os_path_is_absolute( const char* path ) { - return (path != NULL && path[0] == '/'); -} - -ic_private char ic_dirsep(void) { - return '/'; -} -#endif - - - -//------------------------------------------------------------- -// File completion -//------------------------------------------------------------- - -static bool ends_with_n(const char* name, ssize_t name_len, const char* ending, ssize_t len) { - if (name_len < len) return false; - if (ending == NULL || len <= 0) return true; - for (ssize_t i = 1; i <= len; i++) { - char c1 = name[name_len - i]; - char c2 = ending[len - i]; - #ifdef _WIN32 - if (ic_tolower(c1) != ic_tolower(c2)) return false; - #else - if (c1 != c2) return false; - #endif - } - return true; -} - -static bool match_extension(const char* name, const char* extensions) { - if (extensions == NULL || extensions[0] == 0) return true; - if (name == NULL) return false; - ssize_t name_len = ic_strlen(name); - ssize_t len = ic_strlen(extensions); - ssize_t cur = 0; - //debug_msg("match extensions: %s ~ %s", name, extensions); - for (ssize_t end = 0; end <= len; end++) { - if (extensions[end] == ';' || extensions[end] == 0) { - if (ends_with_n(name, name_len, extensions+cur, (end - cur))) { - return true; - } - cur = end+1; - } - } - return false; -} - -static bool filename_complete_indir( ic_completion_env_t* cenv, stringbuf_t* dir, - stringbuf_t* dir_prefix, stringbuf_t* display, - const char* base_prefix, - char dir_sep, const char* extensions ) -{ - dir_cursor d = 0; - dir_entry entry; - bool cont = true; - if (os_findfirst(cenv->env->mem, sbuf_string(dir), &d, &entry)) { - do { - const char* name = os_direntry_name(&entry); - if (name != NULL && strcmp(name, ".") != 0 && strcmp(name, "..") != 0 && - ic_istarts_with(name, base_prefix)) - { - // possible match, first check if it is a directory - file_type_t ft; - bool isdir; - const ssize_t plen = sbuf_len(dir_prefix); - sbuf_append(dir_prefix, name); - { // check directory and potentially add a dirsep to the dir_prefix - const ssize_t dlen = sbuf_len(dir); - sbuf_append_char(dir,ic_dirsep()); - sbuf_append(dir,name); - ft = os_get_filetype(sbuf_string(dir)); - isdir = os_is_dir(sbuf_string(dir)); - if (isdir && dir_sep != 0) { - sbuf_append_char(dir_prefix,dir_sep); - } - sbuf_delete_from(dir,dlen); // restore dir - } - if (isdir || match_extension(name, extensions)) { - // add completion - sbuf_clear(display); - ls_colorize(cenv->env->no_lscolors, display, ft, name, NULL, (isdir ? dir_sep : 0)); - cont = ic_add_completion_ex(cenv, sbuf_string(dir_prefix), sbuf_string(display), NULL); - } - sbuf_delete_from( dir_prefix, plen ); // restore dir_prefix - } - } while (cont && os_findnext(d, &entry)); - os_findclose(d); - } - return cont; -} - -typedef struct filename_closure_s { - const char* roots; - const char* extensions; - char dir_sep; -} filename_closure_t; - -static void filename_completer( ic_completion_env_t* cenv, const char* prefix ) { - if (prefix == NULL) return; - filename_closure_t* fclosure = (filename_closure_t*)cenv->arg; - stringbuf_t* root_dir = sbuf_new(cenv->env->mem); - stringbuf_t* dir_prefix = sbuf_new(cenv->env->mem); - stringbuf_t* display = sbuf_new(cenv->env->mem); - if (root_dir!=NULL && dir_prefix != NULL && display != NULL) - { - // split prefix in dir_prefix / base. - const char* base = strrchr(prefix,'/'); - #ifdef _WIN32 - const char* base2 = strrchr(prefix,'\\'); - if (base == NULL || base2 > base) base = base2; - #endif - if (base != NULL) { - base++; - sbuf_append_n(dir_prefix, prefix, base - prefix ); // includes dir separator - } - - // absolute path - if (os_path_is_absolute(prefix)) { - // do not use roots but try to complete directly - if (base != NULL) { - sbuf_append_n( root_dir, prefix, (base - prefix)); // include dir separator - } - filename_complete_indir( cenv, root_dir, dir_prefix, display, - (base != NULL ? base : prefix), - fclosure->dir_sep, fclosure->extensions ); - } - else { - // relative path, complete with respect to every root. - const char* next; - const char* root = fclosure->roots; - while ( root != NULL ) { - // create full root in `root_dir` - sbuf_clear(root_dir); - next = strchr(root,';'); - if (next == NULL) { - sbuf_append( root_dir, root ); - root = NULL; - } - else { - sbuf_append_n( root_dir, root, next - root ); - root = next + 1; - } - sbuf_append_char( root_dir, ic_dirsep()); - - // add the dir_prefix to the root - if (base != NULL) { - sbuf_append_n( root_dir, prefix, (base - prefix) - 1); - } - - // and complete in this directory - filename_complete_indir( cenv, root_dir, dir_prefix, display, - (base != NULL ? base : prefix), - fclosure->dir_sep, fclosure->extensions); - } - } - } - sbuf_free(display); - sbuf_free(root_dir); - sbuf_free(dir_prefix); -} - -ic_public void ic_complete_filename( ic_completion_env_t* cenv, const char* prefix, char dir_sep, const char* roots, const char* extensions ) { - if (roots == NULL) roots = "."; - if (extensions == NULL) extensions = ""; - if (dir_sep == 0) dir_sep = ic_dirsep(); - filename_closure_t fclosure; - fclosure.dir_sep = dir_sep; - fclosure.roots = roots; - fclosure.extensions = extensions; - cenv->arg = &fclosure; - ic_complete_qword_ex( cenv, prefix, &filename_completer, &ic_char_is_filename_letter, '\\', "'\""); -} diff --git a/vendor/isocline/completions.c b/vendor/isocline/completions.c deleted file mode 100644 index c0181c34..00000000 --- a/vendor/isocline/completions.c +++ /dev/null @@ -1,326 +0,0 @@ -/* ---------------------------------------------------------------------------- - Copyright (c) 2021, Daan Leijen - This is free software; you can redistribute it and/or modify it - under the terms of the MIT License. A copy of the license can be - found in the "LICENSE" file at the root of this distribution. ------------------------------------------------------------------------------*/ -#include -#include -#include - -#include "./isocline.h" -#include "common.h" -#include "env.h" -#include "stringbuf.h" -#include "completions.h" - - -//------------------------------------------------------------- -// Completions -//------------------------------------------------------------- - -typedef struct completion_s { - const char* replacement; - const char* display; - const char* help; - ssize_t delete_before; - ssize_t delete_after; -} completion_t; - -struct completions_s { - ic_completer_fun_t* completer; - void* completer_arg; - ssize_t completer_max; - ssize_t count; - ssize_t len; - completion_t* elems; - alloc_t* mem; -}; - -static void default_filename_completer( ic_completion_env_t* cenv, const char* prefix ); - -ic_private completions_t* completions_new(alloc_t* mem) { - completions_t* cms = mem_zalloc_tp(mem, completions_t); - if (cms == NULL) return NULL; - cms->mem = mem; - cms->completer = &default_filename_completer; - return cms; -} - -ic_private void completions_free(completions_t* cms) { - if (cms == NULL) return; - completions_clear(cms); - if (cms->elems != NULL) { - mem_free(cms->mem, cms->elems); - cms->elems = NULL; - cms->count = 0; - cms->len = 0; - } - mem_free(cms->mem, cms); // free ourselves -} - - -ic_private void completions_clear(completions_t* cms) { - while (cms->count > 0) { - completion_t* cm = cms->elems + cms->count - 1; - mem_free( cms->mem, cm->display); - mem_free( cms->mem, cm->replacement); - mem_free( cms->mem, cm->help); - memset(cm,0,sizeof(*cm)); - cms->count--; - } -} - -static void completions_push(completions_t* cms, const char* replacement, const char* display, const char* help, ssize_t delete_before, ssize_t delete_after) -{ - if (cms->count >= cms->len) { - ssize_t newlen = (cms->len <= 0 ? 32 : cms->len*2); - completion_t* newelems = mem_realloc_tp(cms->mem, completion_t, cms->elems, newlen ); - if (newelems == NULL) return; - cms->elems = newelems; - cms->len = newlen; - } - assert(cms->count < cms->len); - completion_t* cm = cms->elems + cms->count; - cm->replacement = mem_strdup(cms->mem,replacement); - cm->display = mem_strdup(cms->mem,display); - cm->help = mem_strdup(cms->mem,help); - cm->delete_before = delete_before; - cm->delete_after = delete_after; - cms->count++; -} - -ic_private ssize_t completions_count(completions_t* cms) { - return cms->count; -} - -static bool completions_contains(completions_t* cms, const char* replacement) { - for( ssize_t i = 0; i < cms->count; i++ ) { - const completion_t* c = cms->elems + i; - if (strcmp(replacement,c->replacement) == 0) { return true; } - } - return false; -} - -ic_private bool completions_add(completions_t* cms, const char* replacement, const char* display, const char* help, ssize_t delete_before, ssize_t delete_after) { - if (cms->completer_max <= 0) return false; - cms->completer_max--; - //debug_msg("completion: add: %d,%d, %s\n", delete_before, delete_after, replacement); - if (!completions_contains(cms,replacement)) { - completions_push(cms, replacement, display, help, delete_before, delete_after); - } - return true; -} - -static completion_t* completions_get(completions_t* cms, ssize_t index) { - if (index < 0 || cms->count <= 0 || index >= cms->count) return NULL; - return &cms->elems[index]; -} - -ic_private const char* completions_get_display( completions_t* cms, ssize_t index, const char** help ) { - if (help != NULL) { *help = NULL; } - completion_t* cm = completions_get(cms, index); - if (cm == NULL) return NULL; - if (help != NULL) { *help = cm->help; } - return (cm->display != NULL ? cm->display : cm->replacement); -} - -ic_private const char* completions_get_help( completions_t* cms, ssize_t index ) { - completion_t* cm = completions_get(cms, index); - if (cm == NULL) return NULL; - return cm->help; -} - -ic_private const char* completions_get_hint(completions_t* cms, ssize_t index, const char** help) { - if (help != NULL) { *help = NULL; } - completion_t* cm = completions_get(cms, index); - if (cm == NULL) return NULL; - ssize_t len = ic_strlen(cm->replacement); - if (len < cm->delete_before) return NULL; - const char* hint = (cm->replacement + cm->delete_before); - if (*hint == 0 || utf8_is_cont((uint8_t)(*hint))) return NULL; // utf8 boundary? - if (help != NULL) { *help = cm->help; } - return hint; -} - -ic_private void completions_set_completer(completions_t* cms, ic_completer_fun_t* completer, void* arg) { - cms->completer = completer; - cms->completer_arg = arg; -} - -ic_private void completions_get_completer(completions_t* cms, ic_completer_fun_t** completer, void** arg) { - *completer = cms->completer; - *arg = cms->completer_arg; -} - - -ic_public void* ic_completion_arg( const ic_completion_env_t* cenv ) { - return (cenv == NULL ? NULL : cenv->env->completions->completer_arg); -} - -ic_public bool ic_has_completions( const ic_completion_env_t* cenv ) { - return (cenv == NULL ? false : cenv->env->completions->count > 0); -} - -ic_public bool ic_stop_completing( const ic_completion_env_t* cenv) { - return (cenv == NULL ? true : cenv->env->completions->completer_max <= 0); -} - - -static ssize_t completion_apply( completion_t* cm, stringbuf_t* sbuf, ssize_t pos ) { - if (cm == NULL) return -1; - debug_msg( "completion: apply: %s at %zd\n", cm->replacement, pos); - ssize_t start = pos - cm->delete_before; - if (start < 0) start = 0; - ssize_t n = cm->delete_before + cm->delete_after; - if (ic_strlen(cm->replacement) == n && strncmp(sbuf_string_at(sbuf,start), cm->replacement, to_size_t(n)) == 0) { - // no changes - return -1; - } - else { - sbuf_delete_from_to( sbuf, start, pos + cm->delete_after ); - return sbuf_insert_at(sbuf, cm->replacement, start); - } -} - -ic_private ssize_t completions_apply( completions_t* cms, ssize_t index, stringbuf_t* sbuf, ssize_t pos ) { - completion_t* cm = completions_get(cms, index); - return completion_apply( cm, sbuf, pos ); -} - - -static int completion_compare(const void* p1, const void* p2) { - if (p1 == NULL || p2 == NULL) return 0; - const completion_t* cm1 = (const completion_t*)p1; - const completion_t* cm2 = (const completion_t*)p2; - return ic_stricmp(cm1->replacement, cm2->replacement); -} - -ic_private void completions_sort(completions_t* cms) { - if (cms->count <= 0) return; - qsort(cms->elems, to_size_t(cms->count), sizeof(cms->elems[0]), &completion_compare); -} - -#define IC_MAX_PREFIX (256) - -// find longest common prefix and complete with that. -ic_private ssize_t completions_apply_longest_prefix(completions_t* cms, stringbuf_t* sbuf, ssize_t pos) { - if (cms->count <= 1) { - return completions_apply(cms,0,sbuf,pos); - } - - // set initial prefix to the first entry - completion_t* cm = completions_get(cms, 0); - if (cm == NULL) return -1; - - char prefix[IC_MAX_PREFIX+1]; - ssize_t delete_before = cm->delete_before; - ic_strncpy( prefix, IC_MAX_PREFIX+1, cm->replacement, IC_MAX_PREFIX ); - prefix[IC_MAX_PREFIX] = 0; - - // and visit all others to find the longest common prefix - for(ssize_t i = 1; i < cms->count; i++) { - cm = completions_get(cms,i); - if (cm->delete_before != delete_before) { // deletions must match delete_before - prefix[0] = 0; - break; - } - // check if it is still a prefix - const char* r = cm->replacement; - ssize_t j; - for(j = 0; prefix[j] != 0 && r[j] != 0; j++) { - if (prefix[j] != r[j]) break; - } - prefix[j] = 0; - if (j <= 0) break; - } - - // check the length - ssize_t len = ic_strlen(prefix); - if (len <= 0 || len < delete_before) return -1; - - // we found a prefix :-) - completion_t cprefix; - memset(&cprefix,0,sizeof(cprefix)); - cprefix.delete_before = delete_before; - cprefix.replacement = prefix; - ssize_t newpos = completion_apply( &cprefix, sbuf, pos); - if (newpos < 0) return newpos; - - // adjust all delete_before for the new replacement - for( ssize_t i = 0; i < cms->count; i++) { - cm = completions_get(cms,i); - cm->delete_before = len; - } - - return newpos; -} - - -//------------------------------------------------------------- -// Completer functions -//------------------------------------------------------------- - -ic_public bool ic_add_completions(ic_completion_env_t* cenv, const char* prefix, const char** completions) { - for (const char** pc = completions; *pc != NULL; pc++) { - if (ic_istarts_with(*pc, prefix)) { - if (!ic_add_completion_ex(cenv, *pc, NULL, NULL)) return false; - } - } - return true; -} - -ic_public bool ic_add_completion(ic_completion_env_t* cenv, const char* replacement) { - return ic_add_completion_ex(cenv, replacement, NULL, NULL); -} - -ic_public bool ic_add_completion_ex( ic_completion_env_t* cenv, const char* replacement, const char* display, const char* help ) { - return ic_add_completion_prim(cenv,replacement,display,help,0,0); -} - -ic_public bool ic_add_completion_prim(ic_completion_env_t* cenv, const char* replacement, const char* display, const char* help, long delete_before, long delete_after) { - return (*cenv->complete)(cenv->env, cenv->closure, replacement, display, help, delete_before, delete_after ); -} - -static bool prim_add_completion(ic_env_t* env, void* funenv, const char* replacement, const char* display, const char* help, long delete_before, long delete_after) { - ic_unused(funenv); - return completions_add(env->completions, replacement, display, help, delete_before, delete_after); -} - -ic_public void ic_set_default_completer(ic_completer_fun_t* completer, void* arg) { - ic_env_t* env = ic_get_env(); if (env == NULL) return; - completions_set_completer(env->completions, completer, arg); -} - -ic_private ssize_t completions_generate(struct ic_env_s* env, completions_t* cms, const char* input, ssize_t pos, ssize_t max) { - completions_clear(cms); - if (cms->completer == NULL || input == NULL || ic_strlen(input) < pos) return 0; - - // set up env - ic_completion_env_t cenv; - cenv.env = env; - cenv.input = input, - cenv.cursor = (long)pos; - cenv.arg = cms->completer_arg; - cenv.complete = &prim_add_completion; - cenv.closure = NULL; - const char* prefix = mem_strndup(cms->mem, input, pos); - cms->completer_max = max; - - // and complete - cms->completer(&cenv,prefix); - - // restore - mem_free(cms->mem,prefix); - return completions_count(cms); -} - -// The default completer is no completion is set -static void default_filename_completer( ic_completion_env_t* cenv, const char* prefix ) { - #ifdef _WIN32 - const char sep = '\\'; - #else - const char sep = '/'; - #endif - ic_complete_filename( cenv, prefix, sep, ".", NULL); -} diff --git a/vendor/isocline/completions.h b/vendor/isocline/completions.h deleted file mode 100644 index 8361d507..00000000 --- a/vendor/isocline/completions.h +++ /dev/null @@ -1,52 +0,0 @@ -/* ---------------------------------------------------------------------------- - Copyright (c) 2021, Daan Leijen - This is free software; you can redistribute it and/or modify it - under the terms of the MIT License. A copy of the license can be - found in the "LICENSE" file at the root of this distribution. ------------------------------------------------------------------------------*/ -#pragma once -#ifndef IC_COMPLETIONS_H -#define IC_COMPLETIONS_H - -#include "common.h" -#include "stringbuf.h" - - -//------------------------------------------------------------- -// Completions -//------------------------------------------------------------- -#define IC_MAX_COMPLETIONS_TO_SHOW (1000) -#define IC_MAX_COMPLETIONS_TO_TRY (IC_MAX_COMPLETIONS_TO_SHOW/4) - -typedef struct completions_s completions_t; - -ic_private completions_t* completions_new(alloc_t* mem); -ic_private void completions_free(completions_t* cms); -ic_private void completions_clear(completions_t* cms); -ic_private bool completions_add(completions_t* cms , const char* replacement, const char* display, const char* help, ssize_t delete_before, ssize_t delete_after); -ic_private ssize_t completions_count(completions_t* cms); -ic_private ssize_t completions_generate(struct ic_env_s* env, completions_t* cms , const char* input, ssize_t pos, ssize_t max); -ic_private void completions_sort(completions_t* cms); -ic_private void completions_set_completer(completions_t* cms, ic_completer_fun_t* completer, void* arg); -ic_private const char* completions_get_display(completions_t* cms , ssize_t index, const char** help); -ic_private const char* completions_get_hint(completions_t* cms, ssize_t index, const char** help); -ic_private void completions_get_completer(completions_t* cms, ic_completer_fun_t** completer, void** arg); - -ic_private ssize_t completions_apply(completions_t* cms, ssize_t index, stringbuf_t* sbuf, ssize_t pos); -ic_private ssize_t completions_apply_longest_prefix(completions_t* cms, stringbuf_t* sbuf, ssize_t pos); - -//------------------------------------------------------------- -// Completion environment -//------------------------------------------------------------- -typedef bool (ic_completion_fun_t)( ic_env_t* env, void* funenv, const char* replacement, const char* display, const char* help, long delete_before, long delete_after ); - -struct ic_completion_env_s { - ic_env_t* env; // the isocline environment - const char* input; // current full input - long cursor; // current cursor position - void* arg; // argument given to `ic_set_completer` - void* closure; // free variables for function composition - ic_completion_fun_t* complete; // function that adds a completion -}; - -#endif // IC_COMPLETIONS_H diff --git a/vendor/isocline/editline.c b/vendor/isocline/editline.c deleted file mode 100644 index 270c42d9..00000000 --- a/vendor/isocline/editline.c +++ /dev/null @@ -1,1142 +0,0 @@ -/* ---------------------------------------------------------------------------- - Copyright (c) 2021, Daan Leijen - This is free software; you can redistribute it and/or modify it - under the terms of the MIT License. A copy of the license can be - found in the "LICENSE" file at the root of this distribution. ------------------------------------------------------------------------------*/ -#include -#include - -#include "common.h" -#include "term.h" -#include "tty.h" -#include "env.h" -#include "stringbuf.h" -#include "history.h" -#include "completions.h" -#include "undo.h" -#include "highlight.h" - -//------------------------------------------------------------- -// The editor state -//------------------------------------------------------------- - - - -// editor state -typedef struct editor_s { - stringbuf_t* input; // current user input - stringbuf_t* extra; // extra displayed info (for completion menu etc) - stringbuf_t* hint; // hint displayed as part of the input - stringbuf_t* hint_help; // help for a hint. - ssize_t pos; // current cursor position in the input - ssize_t cur_rows; // current used rows to display our content (including extra content) - ssize_t cur_row; // current row that has the cursor (0 based, relative to the prompt) - ssize_t termw; - bool modified; // has a modification happened? (used for history navigation for example) - bool disable_undo; // temporarily disable auto undo (for history search) - ssize_t history_idx; // current index in the history - editstate_t* undo; // undo buffer - editstate_t* redo; // redo buffer - const char* prompt_text; // text of the prompt before the prompt marker - alloc_t* mem; // allocator - // caches - attrbuf_t* attrs; // reuse attribute buffers - attrbuf_t* attrs_extra; -} editor_t; - - - - - -//------------------------------------------------------------- -// Main edit line -//------------------------------------------------------------- -static char* edit_line( ic_env_t* env, const char* prompt_text ); // defined at bottom -static void edit_refresh(ic_env_t* env, editor_t* eb); - -ic_private char* ic_editline(ic_env_t* env, const char* prompt_text) { - tty_start_raw(env->tty); - term_start_raw(env->term); - char* line = edit_line(env,prompt_text); - term_end_raw(env->term,false); - tty_end_raw(env->tty); - term_writeln(env->term,""); - term_flush(env->term); - return line; -} - - -//------------------------------------------------------------- -// Undo/Redo -//------------------------------------------------------------- - -// capture the current edit state -static void editor_capture(editor_t* eb, editstate_t** es ) { - if (!eb->disable_undo) { - editstate_capture( eb->mem, es, sbuf_string(eb->input), eb->pos ); - } -} - -static void editor_undo_capture(editor_t* eb ) { - editor_capture(eb, &eb->undo ); -} - -static void editor_undo_forget(editor_t* eb) { - if (eb->disable_undo) return; - const char* input = NULL; - ssize_t pos = 0; - editstate_restore(eb->mem, &eb->undo, &input, &pos); - mem_free(eb->mem, input); -} - -static void editor_restore(editor_t* eb, editstate_t** from, editstate_t** to ) { - if (eb->disable_undo) return; - if (*from == NULL) return; - const char* input; - if (to != NULL) { editor_capture( eb, to ); } - if (!editstate_restore( eb->mem, from, &input, &eb->pos )) return; - sbuf_replace( eb->input, input ); - mem_free(eb->mem, input); - eb->modified = false; -} - -static void editor_undo_restore(editor_t* eb, bool with_redo ) { - editor_restore(eb, &eb->undo, (with_redo ? &eb->redo : NULL)); -} - -static void editor_redo_restore(editor_t* eb ) { - editor_restore(eb, &eb->redo, &eb->undo); - eb->modified = false; -} - -static void editor_start_modify(editor_t* eb ) { - editor_undo_capture(eb); - editstate_done(eb->mem, &eb->redo); // clear redo - eb->modified = true; -} - - - -static bool editor_pos_is_at_end(editor_t* eb ) { - return (eb->pos == sbuf_len(eb->input)); -} - -//------------------------------------------------------------- -// Row/Column width and positioning -//------------------------------------------------------------- - - -static void edit_get_prompt_width( ic_env_t* env, editor_t* eb, bool in_extra, ssize_t* promptw, ssize_t* cpromptw ) { - if (in_extra) { - *promptw = 0; - *cpromptw = 0; - } - else { - // todo: cache prompt widths - ssize_t textw = bbcode_column_width(env->bbcode, eb->prompt_text); - ssize_t markerw = bbcode_column_width(env->bbcode, env->prompt_marker); - ssize_t cmarkerw = bbcode_column_width(env->bbcode, env->cprompt_marker); - *promptw = markerw + textw; - *cpromptw = (env->no_multiline_indent || *promptw < cmarkerw ? cmarkerw : *promptw); - } -} - -static ssize_t edit_get_rowcol( ic_env_t* env, editor_t* eb, rowcol_t* rc ) { - ssize_t promptw, cpromptw; - edit_get_prompt_width(env, eb, false, &promptw, &cpromptw); - return sbuf_get_rc_at_pos( eb->input, eb->termw, promptw, cpromptw, eb->pos, rc ); -} - -static void edit_set_pos_at_rowcol( ic_env_t* env, editor_t* eb, ssize_t row, ssize_t col ) { - ssize_t promptw, cpromptw; - edit_get_prompt_width(env, eb, false, &promptw, &cpromptw); - ssize_t pos = sbuf_get_pos_at_rc( eb->input, eb->termw, promptw, cpromptw, row, col ); - if (pos < 0) return; - eb->pos = pos; - edit_refresh(env, eb); -} - -static bool edit_pos_is_at_row_end( ic_env_t* env, editor_t* eb ) { - rowcol_t rc; - edit_get_rowcol( env, eb, &rc ); - return rc.last_on_row; -} - -static void edit_write_prompt( ic_env_t* env, editor_t* eb, ssize_t row, bool in_extra ) { - if (in_extra) return; - bbcode_style_open(env->bbcode, "ic-prompt"); - if (row==0) { - // regular prompt text - bbcode_print( env->bbcode, eb->prompt_text ); - } - else if (!env->no_multiline_indent) { - // multiline continuation indentation - // todo: cache prompt widths - ssize_t textw = bbcode_column_width(env->bbcode, eb->prompt_text ); - ssize_t markerw = bbcode_column_width(env->bbcode, env->prompt_marker); - ssize_t cmarkerw = bbcode_column_width(env->bbcode, env->cprompt_marker); - if (cmarkerw < markerw + textw) { - term_write_repeat(env->term, " ", markerw + textw - cmarkerw ); - } - } - // the marker - bbcode_print(env->bbcode, (row == 0 ? env->prompt_marker : env->cprompt_marker )); - bbcode_style_close(env->bbcode,NULL); -} - -//------------------------------------------------------------- -// Refresh -//------------------------------------------------------------- - -typedef struct refresh_info_s { - ic_env_t* env; - editor_t* eb; - attrbuf_t* attrs; - bool in_extra; - ssize_t first_row; - ssize_t last_row; -} refresh_info_t; - -static bool edit_refresh_rows_iter( - const char* s, - ssize_t row, ssize_t row_start, ssize_t row_len, - ssize_t startw, bool is_wrap, const void* arg, void* res) -{ - ic_unused(res); ic_unused(startw); - const refresh_info_t* info = (const refresh_info_t*)(arg); - term_t* term = info->env->term; - - // debug_msg("edit: line refresh: row %zd, len: %zd\n", row, row_len); - if (row < info->first_row) return false; - if (row > info->last_row) return true; // should not occur - - // term_clear_line(term); - edit_write_prompt(info->env, info->eb, row, info->in_extra); - - //' write output - if (info->attrs == NULL || (info->env->no_highlight && info->env->no_bracematch)) { - term_write_n( term, s + row_start, row_len ); - } - else { - term_write_formatted_n( term, s + row_start, attrbuf_attrs(info->attrs, row_start + row_len) + row_start, row_len ); - } - - // write line ending - if (row < info->last_row) { - if (is_wrap && tty_is_utf8(info->env->tty)) { - #ifndef __APPLE__ - bbcode_print( info->env->bbcode, "[ic-dim]\xE2\x86\x90"); // left arrow - #else - bbcode_print( info->env->bbcode, "[ic-dim]\xE2\x86\xB5" ); // return symbol - #endif - } - term_clear_to_end_of_line(term); - term_writeln(term, ""); - } - else { - term_clear_to_end_of_line(term); - } - return (row >= info->last_row); -} - -static void edit_refresh_rows(ic_env_t* env, editor_t* eb, stringbuf_t* input, attrbuf_t* attrs, - ssize_t promptw, ssize_t cpromptw, bool in_extra, - ssize_t first_row, ssize_t last_row) -{ - if (input == NULL) return; - refresh_info_t info; - info.env = env; - info.eb = eb; - info.attrs = attrs; - info.in_extra = in_extra; - info.first_row = first_row; - info.last_row = last_row; - sbuf_for_each_row( input, eb->termw, promptw, cpromptw, &edit_refresh_rows_iter, &info, NULL); -} - - -static void edit_refresh(ic_env_t* env, editor_t* eb) -{ - // calculate the new cursor row and total rows needed - ssize_t promptw, cpromptw; - edit_get_prompt_width( env, eb, false, &promptw, &cpromptw ); - - if (eb->attrs != NULL) { - highlight( env->mem, env->bbcode, sbuf_string(eb->input), eb->attrs, - (env->no_highlight ? NULL : env->highlighter), env->highlighter_arg ); - } - - // highlight matching braces - if (eb->attrs != NULL && !env->no_bracematch) { - highlight_match_braces(sbuf_string(eb->input), eb->attrs, eb->pos, ic_env_get_match_braces(env), - bbcode_style(env->bbcode,"ic-bracematch"), bbcode_style(env->bbcode,"ic-error")); - } - - // insert hint - if (sbuf_len(eb->hint) > 0) { - if (eb->attrs != NULL) { - attrbuf_insert_at( eb->attrs, eb->pos, sbuf_len(eb->hint), bbcode_style(env->bbcode, "ic-hint") ); - } - sbuf_insert_at(eb->input, sbuf_string(eb->hint), eb->pos ); - } - - // render extra (like a completion menu) - stringbuf_t* extra = NULL; - if (sbuf_len(eb->extra) > 0) { - extra = sbuf_new(eb->mem); - if (extra != NULL) { - if (sbuf_len(eb->hint_help) > 0) { - bbcode_append(env->bbcode, sbuf_string(eb->hint_help), extra, eb->attrs_extra); - } - bbcode_append(env->bbcode, sbuf_string(eb->extra), extra, eb->attrs_extra); - } - } - - // calculate rows and row/col position - rowcol_t rc = { 0 }; - const ssize_t rows_input = sbuf_get_rc_at_pos( eb->input, eb->termw, promptw, cpromptw, eb->pos, &rc ); - rowcol_t rc_extra = { 0 }; - ssize_t rows_extra = 0; - if (extra != NULL) { - rows_extra = sbuf_get_rc_at_pos( extra, eb->termw, 0, 0, 0 /*pos*/, &rc_extra ); - } - const ssize_t rows = rows_input + rows_extra; - debug_msg("edit: refresh: rows %zd, cursor: %zd,%zd (previous rows %zd, cursor row %zd)\n", rows, rc.row, rc.col, eb->cur_rows, eb->cur_row); - - // only render at most terminal height rows - const ssize_t termh = term_get_height(env->term); - ssize_t first_row = 0; // first visible row - ssize_t last_row = rows - 1; // last visible row - if (rows > termh) { - first_row = rc.row - termh + 1; // ensure cursor is visible - if (first_row < 0) first_row = 0; - last_row = first_row + termh - 1; - } - assert(last_row - first_row < termh); - - // reduce flicker - buffer_mode_t bmode = term_set_buffer_mode(env->term, BUFFERED); - - // back up to the first line - term_start_of_line(env->term); - term_up(env->term, (eb->cur_row >= termh ? termh-1 : eb->cur_row) ); - // term_clear_lines_to_end(env->term); // gives flicker in old Windows cmd prompt - - // render rows - edit_refresh_rows( env, eb, eb->input, eb->attrs, promptw, cpromptw, false, first_row, last_row ); - if (rows_extra > 0) { - assert(extra != NULL); - const ssize_t first_rowx = (first_row > rows_input ? first_row - rows_input : 0); - const ssize_t last_rowx = last_row - rows_input; assert(last_rowx >= 0); - edit_refresh_rows(env, eb, extra, eb->attrs_extra, 0, 0, true, first_rowx, last_rowx); - } - - // overwrite trailing rows we do not use anymore - ssize_t rrows = last_row - first_row + 1; // rendered rows - if (rrows < termh && rows < eb->cur_rows) { - ssize_t clear = eb->cur_rows - rows; - while (rrows < termh && clear > 0) { - clear--; - rrows++; - term_writeln(env->term,""); - term_clear_line(env->term); - } - } - - // move cursor back to edit position - term_start_of_line(env->term); - term_up(env->term, first_row + rrows - 1 - rc.row ); - term_right(env->term, rc.col + (rc.row == 0 ? promptw : cpromptw)); - - // and refresh - term_flush(env->term); - - // stop buffering - term_set_buffer_mode(env->term, bmode); - - // restore input by removing the hint - sbuf_delete_at(eb->input, eb->pos, sbuf_len(eb->hint)); - sbuf_delete_at(eb->extra, 0, sbuf_len(eb->hint_help)); - attrbuf_clear(eb->attrs); - attrbuf_clear(eb->attrs_extra); - sbuf_free(extra); - - // update previous - eb->cur_rows = rows; - eb->cur_row = rc.row; -} - -// clear current output -static void edit_clear(ic_env_t* env, editor_t* eb ) { - term_attr_reset(env->term); - term_up(env->term, eb->cur_row); - - // overwrite all rows - for( ssize_t i = 0; i < eb->cur_rows; i++) { - term_clear_line(env->term); - term_writeln(env->term, ""); - } - - // move cursor back - term_up(env->term, eb->cur_rows - eb->cur_row ); -} - - -// clear screen and refresh -static void edit_clear_screen(ic_env_t* env, editor_t* eb ) { - ssize_t cur_rows = eb->cur_rows; - eb->cur_rows = term_get_height(env->term) - 1; - edit_clear(env,eb); - eb->cur_rows = cur_rows; - edit_refresh(env,eb); -} - - -// refresh after a terminal window resized (but before doing further edit operations!) -static bool edit_resize(ic_env_t* env, editor_t* eb ) { - // update dimensions - term_update_dim(env->term); - ssize_t newtermw = term_get_width(env->term); - if (eb->termw == newtermw) return false; - - // recalculate the row layout assuming the hardwrapping for the new terminal width - ssize_t promptw, cpromptw; - edit_get_prompt_width( env, eb, false, &promptw, &cpromptw ); - sbuf_insert_at(eb->input, sbuf_string(eb->hint), eb->pos); // insert used hint - - // render extra (like a completion menu) - stringbuf_t* extra = NULL; - if (sbuf_len(eb->extra) > 0) { - extra = sbuf_new(eb->mem); - if (extra != NULL) { - if (sbuf_len(eb->hint_help) > 0) { - bbcode_append(env->bbcode, sbuf_string(eb->hint_help), extra, NULL); - } - bbcode_append(env->bbcode, sbuf_string(eb->extra), extra, NULL); - } - } - rowcol_t rc = { 0 }; - const ssize_t rows_input = sbuf_get_wrapped_rc_at_pos( eb->input, eb->termw, newtermw, promptw, cpromptw, eb->pos, &rc ); - rowcol_t rc_extra = { 0 }; - ssize_t rows_extra = 0; - if (extra != NULL) { - rows_extra = sbuf_get_wrapped_rc_at_pos(extra, eb->termw, newtermw, 0, 0, 0 /*pos*/, &rc_extra); - } - ssize_t rows = rows_input + rows_extra; - debug_msg("edit: resize: new rows: %zd, cursor row: %zd (previous: rows: %zd, cursor row %zd)\n", rows, rc.row, eb->cur_rows, eb->cur_row); - - // update the newly calculated row and rows - eb->cur_row = rc.row; - if (rows > eb->cur_rows) { - eb->cur_rows = rows; - } - eb->termw = newtermw; - edit_refresh(env,eb); - - // remove hint again - sbuf_delete_at(eb->input, eb->pos, sbuf_len(eb->hint)); - sbuf_free(extra); - return true; -} - -static void editor_append_hint_help(editor_t* eb, const char* help) { - sbuf_clear(eb->hint_help); - if (help != NULL) { - sbuf_replace(eb->hint_help, "[ic-info]"); - sbuf_append(eb->hint_help, help); - sbuf_append(eb->hint_help, "[/ic-info]\n"); - } -} - -// refresh with possible hint -static void edit_refresh_hint(ic_env_t* env, editor_t* eb) { - if (env->no_hint || env->hint_delay > 0) { - // refresh without hint first - edit_refresh(env, eb); - if (env->no_hint) return; - } - - // and see if we can construct a hint (displayed after a delay) - ssize_t count = completions_generate(env, env->completions, sbuf_string(eb->input), eb->pos, 2); - if (count == 1) { - const char* help = NULL; - const char* hint = completions_get_hint(env->completions, 0, &help); - if (hint != NULL) { - sbuf_replace(eb->hint, hint); - editor_append_hint_help(eb, help); - // do auto-tabbing? - if (env->complete_autotab) { - stringbuf_t* sb = sbuf_new(env->mem); // temporary buffer for completion - if (sb != NULL) { - sbuf_replace( sb, sbuf_string(eb->input) ); - ssize_t pos = eb->pos; - const char* extra_hint = hint; - do { - ssize_t newpos = sbuf_insert_at( sb, extra_hint, pos ); - if (newpos <= pos) break; - pos = newpos; - count = completions_generate(env, env->completions, sbuf_string(sb), pos, 2); - if (count == 1) { - const char* extra_help = NULL; - extra_hint = completions_get_hint(env->completions, 0, &extra_help); - if (extra_hint != NULL) { - editor_append_hint_help(eb, extra_help); - sbuf_append(eb->hint, extra_hint); - } - } - } - while(count == 1); - sbuf_free(sb); - } - } - } - } - - if (env->hint_delay <= 0) { - // refresh with hint directly - edit_refresh(env, eb); - } -} - -//------------------------------------------------------------- -// Edit operations -//------------------------------------------------------------- - -static void edit_history_prev(ic_env_t* env, editor_t* eb); -static void edit_history_next(ic_env_t* env, editor_t* eb); - -static void edit_undo_restore(ic_env_t* env, editor_t* eb) { - editor_undo_restore(eb, true); - edit_refresh(env,eb); -} - -static void edit_redo_restore(ic_env_t* env, editor_t* eb) { - editor_redo_restore(eb); - edit_refresh(env,eb); -} - -static void edit_cursor_left(ic_env_t* env, editor_t* eb) { - ssize_t cwidth = 1; - ssize_t prev = sbuf_prev(eb->input,eb->pos,&cwidth); - if (prev < 0) return; - rowcol_t rc; - edit_get_rowcol( env, eb, &rc); - eb->pos = prev; - edit_refresh(env,eb); -} - -static void edit_cursor_right(ic_env_t* env, editor_t* eb) { - ssize_t cwidth = 1; - ssize_t next = sbuf_next(eb->input,eb->pos,&cwidth); - if (next < 0) return; - rowcol_t rc; - edit_get_rowcol( env, eb, &rc); - eb->pos = next; - edit_refresh(env,eb); -} - -static void edit_cursor_line_end(ic_env_t* env, editor_t* eb) { - ssize_t end = sbuf_find_line_end(eb->input,eb->pos); - if (end < 0) return; - eb->pos = end; - edit_refresh(env,eb); -} - -static void edit_cursor_line_start(ic_env_t* env, editor_t* eb) { - ssize_t start = sbuf_find_line_start(eb->input,eb->pos); - if (start < 0) return; - eb->pos = start; - edit_refresh(env,eb); -} - -static void edit_cursor_next_word(ic_env_t* env, editor_t* eb) { - ssize_t end = sbuf_find_word_end(eb->input,eb->pos); - if (end < 0) return; - eb->pos = end; - edit_refresh(env,eb); -} - -static void edit_cursor_prev_word(ic_env_t* env, editor_t* eb) { - ssize_t start = sbuf_find_word_start(eb->input,eb->pos); - if (start < 0) return; - eb->pos = start; - edit_refresh(env,eb); -} - -static void edit_cursor_next_ws_word(ic_env_t* env, editor_t* eb) { - ssize_t end = sbuf_find_ws_word_end(eb->input, eb->pos); - if (end < 0) return; - eb->pos = end; - edit_refresh(env, eb); -} - -static void edit_cursor_prev_ws_word(ic_env_t* env, editor_t* eb) { - ssize_t start = sbuf_find_ws_word_start(eb->input, eb->pos); - if (start < 0) return; - eb->pos = start; - edit_refresh(env, eb); -} - -static void edit_cursor_to_start(ic_env_t* env, editor_t* eb) { - eb->pos = 0; - edit_refresh(env,eb); -} - -static void edit_cursor_to_end(ic_env_t* env, editor_t* eb) { - eb->pos = sbuf_len(eb->input); - edit_refresh(env,eb); -} - - -static void edit_cursor_row_up(ic_env_t* env, editor_t* eb) { - rowcol_t rc; - edit_get_rowcol( env, eb, &rc); - if (rc.row == 0) { - edit_history_prev(env,eb); - } - else { - edit_set_pos_at_rowcol( env, eb, rc.row - 1, rc.col ); - } -} - -static void edit_cursor_row_down(ic_env_t* env, editor_t* eb) { - rowcol_t rc; - ssize_t rows = edit_get_rowcol( env, eb, &rc); - if (rc.row + 1 >= rows) { - edit_history_next(env,eb); - } - else { - edit_set_pos_at_rowcol( env, eb, rc.row + 1, rc.col ); - } -} - - -static void edit_cursor_match_brace(ic_env_t* env, editor_t* eb) { - ssize_t match = find_matching_brace( sbuf_string(eb->input), eb->pos, ic_env_get_match_braces(env), NULL ); - if (match < 0) return; - eb->pos = match; - edit_refresh(env,eb); -} - -static void edit_backspace(ic_env_t* env, editor_t* eb) { - if (eb->pos <= 0) return; - editor_start_modify(eb); - eb->pos = sbuf_delete_char_before(eb->input,eb->pos); - edit_refresh(env,eb); -} - -static void edit_delete_char(ic_env_t* env, editor_t* eb) { - if (eb->pos >= sbuf_len(eb->input)) return; - editor_start_modify(eb); - sbuf_delete_char_at(eb->input,eb->pos); - edit_refresh(env,eb); -} - -static void edit_delete_all(ic_env_t* env, editor_t* eb) { - if (sbuf_len(eb->input) <= 0) return; - editor_start_modify(eb); - sbuf_clear(eb->input); - eb->pos = 0; - edit_refresh(env,eb); -} - -static void edit_delete_to_end_of_line(ic_env_t* env, editor_t* eb) { - ssize_t start = sbuf_find_line_start(eb->input,eb->pos); - if (start < 0) return; - ssize_t end = sbuf_find_line_end(eb->input,eb->pos); - if (end < 0) return; - editor_start_modify(eb); - // if on an empty line, remove it completely - if (start == end && sbuf_char_at(eb->input,end) == '\n') { - end++; - } - else if (start == end && sbuf_char_at(eb->input,start - 1) == '\n') { - eb->pos--; - } - sbuf_delete_from_to( eb->input, eb->pos, end ); - edit_refresh(env,eb); -} - -static void edit_delete_to_start_of_line(ic_env_t* env, editor_t* eb) { - ssize_t start = sbuf_find_line_start(eb->input,eb->pos); - if (start < 0) return; - ssize_t end = sbuf_find_line_end(eb->input,eb->pos); - if (end < 0) return; - editor_start_modify(eb); - // delete start newline if it was an empty line - bool goright = false; - if (start > 0 && sbuf_char_at(eb->input,start-1) == '\n' && start == end) { - // if it is an empty line remove it - start--; - // afterwards, move to start of next line if it exists (so the cursor stays on the same row) - goright = true; - } - sbuf_delete_from_to( eb->input, start, eb->pos ); - eb->pos = start; - if (goright) edit_cursor_right(env,eb); - edit_refresh(env,eb); -} - -static void edit_delete_line(ic_env_t* env, editor_t* eb) { - ssize_t start = sbuf_find_line_start(eb->input,eb->pos); - if (start < 0) return; - ssize_t end = sbuf_find_line_end(eb->input,eb->pos); - if (end < 0) return; - editor_start_modify(eb); - // delete newline as well so no empty line is left; - bool goright = false; - if (start > 0 && sbuf_char_at(eb->input,start-1) == '\n') { - start--; - // afterwards, move to start of next line if it exists (so the cursor stays on the same row) - goright = true; - } - else if (sbuf_char_at(eb->input,end) == '\n') { - end++; - } - sbuf_delete_from_to(eb->input,start,end); - eb->pos = start; - if (goright) edit_cursor_right(env,eb); - edit_refresh(env,eb); -} - -static void edit_delete_to_start_of_word(ic_env_t* env, editor_t* eb) { - ssize_t start = sbuf_find_word_start(eb->input,eb->pos); - if (start < 0) return; - editor_start_modify(eb); - sbuf_delete_from_to( eb->input, start, eb->pos ); - eb->pos = start; - edit_refresh(env,eb); -} - -static void edit_delete_to_end_of_word(ic_env_t* env, editor_t* eb) { - ssize_t end = sbuf_find_word_end(eb->input,eb->pos); - if (end < 0) return; - editor_start_modify(eb); - sbuf_delete_from_to( eb->input, eb->pos, end ); - edit_refresh(env,eb); -} - -static void edit_delete_to_start_of_ws_word(ic_env_t* env, editor_t* eb) { - ssize_t start = sbuf_find_ws_word_start(eb->input, eb->pos); - if (start < 0) return; - editor_start_modify(eb); - sbuf_delete_from_to(eb->input, start, eb->pos); - eb->pos = start; - edit_refresh(env, eb); -} - -static void edit_delete_to_end_of_ws_word(ic_env_t* env, editor_t* eb) { - ssize_t end = sbuf_find_ws_word_end(eb->input, eb->pos); - if (end < 0) return; - editor_start_modify(eb); - sbuf_delete_from_to(eb->input, eb->pos, end); - edit_refresh(env, eb); -} - - -static void edit_delete_word(ic_env_t* env, editor_t* eb) { - ssize_t start = sbuf_find_word_start(eb->input,eb->pos); - if (start < 0) return; - ssize_t end = sbuf_find_word_end(eb->input,eb->pos); - if (end < 0) return; - editor_start_modify(eb); - sbuf_delete_from_to(eb->input,start,end); - eb->pos = start; - edit_refresh(env,eb); -} - -static void edit_swap_char( ic_env_t* env, editor_t* eb ) { - if (eb->pos <= 0 || eb->pos == sbuf_len(eb->input)) return; - editor_start_modify(eb); - eb->pos = sbuf_swap_char(eb->input,eb->pos); - edit_refresh(env,eb); -} - -static void edit_multiline_eol(ic_env_t* env, editor_t* eb) { - if (eb->pos <= 0) return; - if (sbuf_string(eb->input)[eb->pos-1] != env->multiline_eol) return; - editor_start_modify(eb); - // replace line continuation with a real newline - sbuf_delete_at( eb->input, eb->pos-1, 1); - sbuf_insert_at( eb->input, "\n", eb->pos-1); - edit_refresh(env,eb); -} - -static void edit_insert_unicode(ic_env_t* env, editor_t* eb, unicode_t u) { - editor_start_modify(eb); - ssize_t nextpos = sbuf_insert_unicode_at(eb->input, u, eb->pos); - if (nextpos >= 0) eb->pos = nextpos; - edit_refresh_hint(env, eb); -} - -static void edit_auto_brace(ic_env_t* env, editor_t* eb, char c) { - if (env->no_autobrace) return; - const char* braces = ic_env_get_auto_braces(env); - for (const char* b = braces; *b != 0; b += 2) { - if (*b == c) { - const char close = b[1]; - //if (sbuf_char_at(eb->input, eb->pos) != close) { - sbuf_insert_char_at(eb->input, close, eb->pos); - bool balanced = false; - find_matching_brace(sbuf_string(eb->input), eb->pos, braces, &balanced ); - if (!balanced) { - // don't insert if it leads to an unbalanced expression. - sbuf_delete_char_at(eb->input, eb->pos); - } - //} - return; - } - else if (b[1] == c) { - // close brace, check if there we don't overwrite to the right - if (sbuf_char_at(eb->input, eb->pos) == c) { - sbuf_delete_char_at(eb->input, eb->pos); - } - return; - } - } -} - -static void editor_auto_indent(editor_t* eb, const char* pre, const char* post ) { - assert(eb->pos > 0 && sbuf_char_at(eb->input,eb->pos-1) == '\n'); - ssize_t prelen = ic_strlen(pre); - if (prelen > 0) { - if (eb->pos - 1 < prelen) return; - if (!ic_starts_with(sbuf_string(eb->input) + eb->pos - 1 - prelen, pre)) return; - if (!ic_starts_with(sbuf_string(eb->input) + eb->pos, post)) return; - eb->pos = sbuf_insert_at(eb->input, " ", eb->pos); - sbuf_insert_char_at(eb->input, '\n', eb->pos); - } -} - -static void edit_insert_char(ic_env_t* env, editor_t* eb, char c) { - editor_start_modify(eb); - ssize_t nextpos = sbuf_insert_char_at( eb->input, c, eb->pos ); - if (nextpos >= 0) eb->pos = nextpos; - edit_auto_brace(env, eb, c); - if (c=='\n') { - editor_auto_indent(eb, "{", "}"); // todo: custom auto indent tokens? - } - edit_refresh_hint(env,eb); -} - -//------------------------------------------------------------- -// Help -//------------------------------------------------------------- - -#include "editline_help.c" - -//------------------------------------------------------------- -// History -//------------------------------------------------------------- - -#include "editline_history.c" - -//------------------------------------------------------------- -// Completion -//------------------------------------------------------------- - -#include "editline_completion.c" - - -//------------------------------------------------------------- -// Edit line: main edit loop -//------------------------------------------------------------- - -static char* edit_line( ic_env_t* env, const char* prompt_text ) -{ - // set up an edit buffer - editor_t eb; - memset(&eb, 0, sizeof(eb)); - eb.mem = env->mem; - eb.input = sbuf_new(env->mem); - eb.extra = sbuf_new(env->mem); - eb.hint = sbuf_new(env->mem); - eb.hint_help= sbuf_new(env->mem); - eb.termw = term_get_width(env->term); - eb.pos = 0; - eb.cur_rows = 1; - eb.cur_row = 0; - eb.modified = false; - eb.prompt_text = (prompt_text != NULL ? prompt_text : ""); - eb.history_idx = 0; - editstate_init(&eb.undo); - editstate_init(&eb.redo); - if (eb.input==NULL || eb.extra==NULL || eb.hint==NULL || eb.hint_help==NULL) { - return NULL; - } - - // caching - if (!(env->no_highlight && env->no_bracematch)) { - eb.attrs = attrbuf_new(env->mem); - eb.attrs_extra = attrbuf_new(env->mem); - } - - // show prompt - edit_write_prompt(env, &eb, 0, false); - - // always a history entry for the current input - history_push(env->history, ""); - - // process keys - code_t c; // current key code - while(true) { - // read a character - term_flush(env->term); - if (env->hint_delay <= 0 || sbuf_len(eb.hint) == 0) { - // blocking read - c = tty_read(env->tty); - } - else { - // timeout to display hint - if (!tty_read_timeout(env->tty, env->hint_delay, &c)) { - // timed-out - if (sbuf_len(eb.hint) > 0) { - // display hint - edit_refresh(env, &eb); - } - c = tty_read(env->tty); - } - else { - // clear the pending hint if we got input before the delay expired - sbuf_clear(eb.hint); - sbuf_clear(eb.hint_help); - } - } - - // update terminal in case of a resize - if (tty_term_resize_event(env->tty)) { - edit_resize(env,&eb); - } - - // clear hint only after a potential resize (so resize row calculations are correct) - const bool had_hint = (sbuf_len(eb.hint) > 0); - sbuf_clear(eb.hint); - sbuf_clear(eb.hint_help); - - // if the user tries to move into a hint with left-cursor or end, we complete it first - if ((c == KEY_RIGHT || c == KEY_END) && had_hint) { - edit_generate_completions(env, &eb, true); - c = KEY_NONE; - } - - // Operations that may return - if (c == KEY_ENTER) { - if (!env->singleline_only && eb.pos > 0 && - sbuf_string(eb.input)[eb.pos-1] == env->multiline_eol && - edit_pos_is_at_row_end(env,&eb)) - { - // replace line-continuation with newline - edit_multiline_eol(env,&eb); - } - else { - // otherwise done - break; - } - } - else if (c == KEY_CTRL_D) { - if (eb.pos == 0 && editor_pos_is_at_end(&eb)) break; // ctrl+D on empty quits with NULL - edit_delete_char(env,&eb); // otherwise it is like delete - } - else if (c == KEY_CTRL_C || c == KEY_EVENT_STOP) { - break; // ctrl+C or STOP event quits with NULL - } - else if (c == KEY_ESC) { - if (eb.pos == 0 && editor_pos_is_at_end(&eb)) break; // ESC on empty input returns with empty input - edit_delete_all(env,&eb); // otherwise delete the current input - // edit_delete_line(env,&eb); // otherwise delete the current line - } - else if (c == KEY_BELL /* ^G */) { - edit_delete_all(env,&eb); - break; // ctrl+G cancels (and returns empty input) - } - - // Editing Operations - else switch(c) { - // events - case KEY_EVENT_RESIZE: // not used - edit_resize(env,&eb); - break; - case KEY_EVENT_AUTOTAB: - edit_generate_completions(env, &eb, true); - break; - - // completion, history, help, undo - case KEY_TAB: - case WITH_ALT('?'): - edit_generate_completions(env,&eb,false); - break; - case KEY_CTRL_R: - case KEY_CTRL_S: - edit_history_search_with_current_word(env,&eb); - break; - case KEY_CTRL_P: - edit_history_prev(env, &eb); - break; - case KEY_CTRL_N: - edit_history_next(env, &eb); - break; - case KEY_CTRL_L: - edit_clear_screen(env, &eb); - break; - case KEY_CTRL_Z: - case WITH_CTRL('_'): - edit_undo_restore(env, &eb); - break; - case KEY_CTRL_Y: - edit_redo_restore(env, &eb); - break; - case KEY_F1: - edit_show_help(env, &eb); - break; - - // navigation - case KEY_LEFT: - case KEY_CTRL_B: - edit_cursor_left(env,&eb); - break; - case KEY_RIGHT: - case KEY_CTRL_F: - if (eb.pos == sbuf_len(eb.input)) { - edit_generate_completions( env, &eb, false ); - } - else { - edit_cursor_right(env,&eb); - } - break; - case KEY_UP: - edit_cursor_row_up(env,&eb); - break; - case KEY_DOWN: - edit_cursor_row_down(env,&eb); - break; - case KEY_HOME: - case KEY_CTRL_A: - edit_cursor_line_start(env,&eb); - break; - case KEY_END: - case KEY_CTRL_E: - edit_cursor_line_end(env,&eb); - break; - case KEY_CTRL_LEFT: - case WITH_SHIFT(KEY_LEFT): - case WITH_ALT('b'): - edit_cursor_prev_word(env,&eb); - break; - case KEY_CTRL_RIGHT: - case WITH_SHIFT(KEY_RIGHT): - case WITH_ALT('f'): - if (eb.pos == sbuf_len(eb.input)) { - edit_generate_completions( env, &eb, false ); - } - else { - edit_cursor_next_word(env,&eb); - } - break; - case KEY_CTRL_HOME: - case WITH_SHIFT(KEY_HOME): - case KEY_PAGEUP: - case WITH_ALT('<'): - edit_cursor_to_start(env,&eb); - break; - case KEY_CTRL_END: - case WITH_SHIFT(KEY_END): - case KEY_PAGEDOWN: - case WITH_ALT('>'): - edit_cursor_to_end(env,&eb); - break; - case WITH_ALT('m'): - edit_cursor_match_brace(env,&eb); - break; - - // deletion - case KEY_BACKSP: - edit_backspace(env,&eb); - break; - case KEY_DEL: - edit_delete_char(env,&eb); - break; - case WITH_ALT('d'): - edit_delete_to_end_of_word(env,&eb); - break; - case KEY_CTRL_W: - edit_delete_to_start_of_ws_word(env, &eb); - break; - case WITH_ALT(KEY_DEL): - case WITH_ALT(KEY_BACKSP): - edit_delete_to_start_of_word(env,&eb); - break; - case KEY_CTRL_U: - edit_delete_to_start_of_line(env,&eb); - break; - case KEY_CTRL_K: - edit_delete_to_end_of_line(env,&eb); - break; - case KEY_CTRL_T: - edit_swap_char(env,&eb); - break; - - // Editing - case KEY_SHIFT_TAB: - case KEY_LINEFEED: // '\n' (ctrl+J, shift+enter) - if (!env->singleline_only) { - edit_insert_char(env, &eb, '\n'); - } - break; - default: { - char chr; - unicode_t uchr; - if (code_is_ascii_char(c,&chr)) { - edit_insert_char(env,&eb,chr); - } - else if (code_is_unicode(c, &uchr)) { - edit_insert_unicode(env,&eb, uchr); - } - else { - debug_msg( "edit: ignore code: 0x%04x\n", c); - } - break; - } - } - - } - - // goto end - eb.pos = sbuf_len(eb.input); - - // refresh once more but without brace matching - bool bm = env->no_bracematch; - env->no_bracematch = true; - edit_refresh(env,&eb); - env->no_bracematch = bm; - - // save result - char* res; - if ((c == KEY_CTRL_D && sbuf_len(eb.input) == 0) || c == KEY_CTRL_C || c == KEY_EVENT_STOP) { - res = NULL; - } - else if (!tty_is_utf8(env->tty)) { - res = sbuf_strdup_from_utf8(eb.input); - } - else { - res = sbuf_strdup(eb.input); - } - - // update history - history_update(env->history, sbuf_string(eb.input)); - if (res == NULL || sbuf_len(eb.input) <= 1) { ic_history_remove_last(); } // no empty or single-char entries - history_save(env->history); - - // free resources - editstate_done(env->mem, &eb.undo); - editstate_done(env->mem, &eb.redo); - attrbuf_free(eb.attrs); - attrbuf_free(eb.attrs_extra); - sbuf_free(eb.input); - sbuf_free(eb.extra); - sbuf_free(eb.hint); - sbuf_free(eb.hint_help); - - return res; -} - diff --git a/vendor/isocline/editline_completion.c b/vendor/isocline/editline_completion.c deleted file mode 100644 index 1734ef34..00000000 --- a/vendor/isocline/editline_completion.c +++ /dev/null @@ -1,277 +0,0 @@ -/* ---------------------------------------------------------------------------- - Copyright (c) 2021, Daan Leijen - This is free software; you can redistribute it and/or modify it - under the terms of the MIT License. A copy of the license can be - found in the "LICENSE" file at the root of this distribution. ------------------------------------------------------------------------------*/ - -//------------------------------------------------------------- -// Completion menu: this file is included in editline.c -//------------------------------------------------------------- - -// return true if anything changed -static bool edit_complete(ic_env_t* env, editor_t* eb, ssize_t idx) { - editor_start_modify(eb); - ssize_t newpos = completions_apply(env->completions, idx, eb->input, eb->pos); - if (newpos < 0) { - editor_undo_restore(eb,false); - return false; - } - eb->pos = newpos; - edit_refresh(env,eb); - return true; -} - -static bool edit_complete_longest_prefix(ic_env_t* env, editor_t* eb ) { - editor_start_modify(eb); - ssize_t newpos = completions_apply_longest_prefix( env->completions, eb->input, eb->pos ); - if (newpos < 0) { - editor_undo_restore(eb,false); - return false; - } - eb->pos = newpos; - edit_refresh(env,eb); - return true; -} - -ic_private void sbuf_append_tagged( stringbuf_t* sb, const char* tag, const char* content ) { - sbuf_appendf(sb, "[%s]", tag); - sbuf_append(sb,content); - sbuf_append(sb,"[/]"); -} - -static void editor_append_completion(ic_env_t* env, editor_t* eb, ssize_t idx, ssize_t width, bool numbered, bool selected ) { - const char* help = NULL; - const char* display = completions_get_display(env->completions, idx, &help); - if (display == NULL) return; - if (numbered) { - sbuf_appendf(eb->extra, "[ic-info]%s%zd [/]", (selected ? (tty_is_utf8(env->tty) ? "\xE2\x86\x92" : "*") : " "), 1 + idx); - width -= 3; - } - - if (width > 0) { - sbuf_appendf(eb->extra, "[width=\"%zd;left; ;on\"]", width ); - } - if (selected) { - sbuf_append(eb->extra, "[ic-emphasis]"); - } - sbuf_append(eb->extra, display); - if (selected) { sbuf_append(eb->extra,"[/ic-emphasis]"); } - if (help != NULL) { - sbuf_append(eb->extra, " "); - sbuf_append_tagged(eb->extra, "ic-info", help ); - } - if (width > 0) { sbuf_append(eb->extra,"[/width]"); } -} - -// 2 and 3 column output up to 80 wide -#define IC_DISPLAY2_MAX 34 -#define IC_DISPLAY2_COL (3+IC_DISPLAY2_MAX) -#define IC_DISPLAY2_WIDTH (2*IC_DISPLAY2_COL + 2) // 75 - -#define IC_DISPLAY3_MAX 21 -#define IC_DISPLAY3_COL (3+IC_DISPLAY3_MAX) -#define IC_DISPLAY3_WIDTH (3*IC_DISPLAY3_COL + 2*2) // 76 - -static void editor_append_completion2(ic_env_t* env, editor_t* eb, ssize_t col_width, ssize_t idx1, ssize_t idx2, ssize_t selected ) { - editor_append_completion(env, eb, idx1, col_width, true, (idx1 == selected) ); - sbuf_append( eb->extra, " "); - editor_append_completion(env, eb, idx2, col_width, true, (idx2 == selected) ); -} - -static void editor_append_completion3(ic_env_t* env, editor_t* eb, ssize_t col_width, ssize_t idx1, ssize_t idx2, ssize_t idx3, ssize_t selected ) { - editor_append_completion(env, eb, idx1, col_width, true, (idx1 == selected) ); - sbuf_append( eb->extra, " "); - editor_append_completion(env, eb, idx2, col_width, true, (idx2 == selected)); - sbuf_append( eb->extra, " "); - editor_append_completion(env, eb, idx3, col_width, true, (idx3 == selected) ); -} - -static ssize_t edit_completions_max_width( ic_env_t* env, ssize_t count ) { - ssize_t max_width = 0; - for( ssize_t i = 0; i < count; i++) { - const char* help = NULL; - ssize_t w = bbcode_column_width(env->bbcode, completions_get_display(env->completions, i, &help)); - if (help != NULL) { - w += 2 + bbcode_column_width(env->bbcode, help); - } - if (w > max_width) { - max_width = w; - } - } - return max_width; -} - -static void edit_completion_menu(ic_env_t* env, editor_t* eb, bool more_available) { - ssize_t count = completions_count(env->completions); - ssize_t count_displayed = count; - assert(count > 1); - ssize_t selected = (env->complete_nopreview ? 0 : -1); // select first or none - ssize_t percolumn = count; - -again: - // show first 9 (or 8) completions - sbuf_clear(eb->extra); - ssize_t twidth = term_get_width(env->term) - 1; - ssize_t colwidth; - if (count > 3 && ((colwidth = 3 + edit_completions_max_width(env, 9))*3 + 2*2) < twidth) { - // display as a 3 column block - count_displayed = (count > 9 ? 9 : count); - percolumn = 3; - for (ssize_t rw = 0; rw < percolumn; rw++) { - if (rw > 0) sbuf_append(eb->extra, "\n"); - editor_append_completion3(env, eb, colwidth, rw, percolumn+rw, (2*percolumn)+rw, selected); - } - } - else if (count > 4 && ((colwidth = 3 + edit_completions_max_width(env, 8))*2 + 2) < twidth) { - // display as a 2 column block if some entries are too wide for three columns - count_displayed = (count > 8 ? 8 : count); - percolumn = (count_displayed <= 6 ? 3 : 4); - for (ssize_t rw = 0; rw < percolumn; rw++) { - if (rw > 0) sbuf_append(eb->extra, "\n"); - editor_append_completion2(env, eb, colwidth, rw, percolumn+rw, selected); - } - } - else { - // display as a list - count_displayed = (count > 9 ? 9 : count); - percolumn = count_displayed; - for (ssize_t i = 0; i < count_displayed; i++) { - if (i > 0) sbuf_append(eb->extra, "\n"); - editor_append_completion(env, eb, i, -1, true /* numbered */, selected == i); - } - } - if (count > count_displayed) { - if (more_available) { - sbuf_append(eb->extra, "\n[ic-info](press page-down (or ctrl-j) to see all further completions)[/]"); - } - else { - sbuf_appendf(eb->extra, "\n[ic-info](press page-down (or ctrl-j) to see all %zd completions)[/]", count ); - } - } - if (!env->complete_nopreview && selected >= 0 && selected <= count_displayed) { - edit_complete(env,eb,selected); - editor_undo_restore(eb,false); - } - else { - edit_refresh(env, eb); - } - - // read here; if not a valid key, push it back and return to main event loop - code_t c = tty_read(env->tty); - if (tty_term_resize_event(env->tty)) { - edit_resize(env, eb); - } - sbuf_clear(eb->extra); - - // direct selection? - if (c >= '1' && c <= '9') { - ssize_t i = (c - '1'); - if (i < count) { - selected = i; - c = KEY_ENTER; - } - } - - // process commands - if (c == KEY_DOWN || c == KEY_TAB) { - selected++; - if (selected >= count_displayed) { - //term_beep(env->term); - selected = 0; - } - goto again; - } - else if (c == KEY_UP || c == KEY_SHIFT_TAB) { - selected--; - if (selected < 0) { - selected = count_displayed - 1; - //term_beep(env->term); - } - goto again; - } - else if (c == KEY_F1) { - edit_show_help(env, eb); - goto again; - } - else if (c == KEY_ESC) { - completions_clear(env->completions); - edit_refresh(env,eb); - c = 0; // ignore and return - } - else if (selected >= 0 && (c == KEY_ENTER || c == KEY_RIGHT || c == KEY_END)) /* || c == KEY_TAB*/ { - // select the current entry - assert(selected < count); - c = 0; - edit_complete(env, eb, selected); - if (env->complete_autotab) { - tty_code_pushback(env->tty,KEY_EVENT_AUTOTAB); // immediately try to complete again - } - } - else if (!env->complete_nopreview && !code_is_virt_key(c)) { - // if in preview mode, select the current entry and exit the menu - assert(selected < count); - edit_complete(env, eb, selected); - } - else if ((c == KEY_PAGEDOWN || c == KEY_LINEFEED) && count > 9) { - // show all completions - c = 0; - if (more_available) { - // generate all entries (up to the max (= 1000)) - count = completions_generate(env, env->completions, sbuf_string(eb->input), eb->pos, IC_MAX_COMPLETIONS_TO_SHOW); - } - rowcol_t rc; - edit_get_rowcol(env,eb,&rc); - edit_clear(env,eb); - edit_write_prompt(env,eb,0,false); - term_writeln(env->term, ""); - for(ssize_t i = 0; i < count; i++) { - const char* display = completions_get_display(env->completions, i, NULL); - if (display != NULL) { - bbcode_println(env->bbcode, display); - } - } - if (count >= IC_MAX_COMPLETIONS_TO_SHOW) { - bbcode_println(env->bbcode, "[ic-info]... and more.[/]"); - } - else { - bbcode_printf(env->bbcode, "[ic-info](%zd possible completions)[/]\n", count ); - } - for(ssize_t i = 0; i < rc.row+1; i++) { - term_write(env->term, " \n"); - } - eb->cur_rows = 0; - edit_refresh(env,eb); - } - else { - edit_refresh(env,eb); - } - // done - completions_clear(env->completions); - if (c != 0) tty_code_pushback(env->tty,c); -} - -static void edit_generate_completions(ic_env_t* env, editor_t* eb, bool autotab) { - debug_msg( "edit: complete: %zd: %s\n", eb->pos, sbuf_string(eb->input) ); - if (eb->pos < 0) return; - ssize_t count = completions_generate(env, env->completions, sbuf_string(eb->input), eb->pos, IC_MAX_COMPLETIONS_TO_TRY); - bool more_available = (count >= IC_MAX_COMPLETIONS_TO_TRY); - if (count <= 0) { - // no completions - if (!autotab) { term_beep(env->term); } - } - else if (count == 1) { - // complete if only one match - if (edit_complete(env,eb,0 /*idx*/) && env->complete_autotab) { - tty_code_pushback(env->tty,KEY_EVENT_AUTOTAB); - } - } - else { - //term_beep(env->term); - if (!more_available) { - edit_complete_longest_prefix(env,eb); - } - completions_sort(env->completions); - edit_completion_menu( env, eb, more_available); - } -} diff --git a/vendor/isocline/editline_help.c b/vendor/isocline/editline_help.c deleted file mode 100644 index fa07d1db..00000000 --- a/vendor/isocline/editline_help.c +++ /dev/null @@ -1,140 +0,0 @@ -/* ---------------------------------------------------------------------------- - Copyright (c) 2021, Daan Leijen - This is free software; you can redistribute it and/or modify it - under the terms of the MIT License. A copy of the license can be - found in the "LICENSE" file at the root of this distribution. ------------------------------------------------------------------------------*/ - -//------------------------------------------------------------- -// Help: this is included into editline.c -//------------------------------------------------------------- - -static const char* help[] = { - "","Navigation:", - "left," - "^b", "go one character to the left", - "right," - "^f", "go one character to the right", - "up", "go one row up, or back in the history", - "down", "go one row down, or forward in the history", - #ifdef __APPLE__ - "shift-left", - #else - "^left", - #endif - "go to the start of the previous word", - #ifdef __APPLE__ - "shift-right", - #else - "^right", - #endif - "go to the end the current word", - "home," - "^a", "go to the start of the current line", - "end," - "^e", "go to the end of the current line", - "pgup," - "^home", "go to the start of the current input", - "pgdn," - "^end", "go to the end of the current input", - "alt-m", "jump to matching brace", - "^p", "go back in the history", - "^n", "go forward in the history", - "^r,^s", "search the history starting with the current word", - "","", - - "", "Deletion:", - "del,^d", "delete the current character", - "backsp,^h", "delete the previous character", - "^w", "delete to preceding white space", - "alt-backsp", "delete to the start of the current word", - "alt-d", "delete to the end of the current word", - "^u", "delete to the start of the current line", - "^k", "delete to the end of the current line", - "esc", "delete the current input, or done with empty input", - "","", - - "", "Editing:", - "enter", "accept current input", - #ifndef __APPLE__ - "^enter, ^j", "", - "shift-tab", - #else - "shift-tab,^j", - #endif - "create a new line for multi-line input", - //" ", "(or type '\\' followed by enter)", - "^l", "clear screen", - "^t", "swap with previous character (move character backward)", - "^z,^_", "undo", - "^y", "redo", - //"^C", "done with empty input", - //"F1", "show this help", - "tab", "try to complete the current input", - "","", - "","In the completion menu:", - "enter,left", "use the currently selected completion", - "1 - 9", "use completion N from the menu", - "tab,down", "select the next completion", - "shift-tab,up","select the previous completion", - "esc", "exit menu without completing", - "pgdn,^j", "show all further possible completions", - "","", - "","In incremental history search:", - "enter", "use the currently found history entry", - "backsp," - "^z", "go back to the previous match (undo)", - "tab," - "^r", "find the next match", - "shift-tab," - "^s", "find an earlier match", - "esc", "exit search", - " ","", - NULL, NULL -}; - -static const char* help_initial = - "[ic-info]" - "Isocline v1.0, copyright (c) 2021 Daan Leijen.\n" - "This is free software; you can redistribute it and/or\n" - "modify it under the terms of the MIT License.\n" - "See <[url]https://github.com/daanx/isocline[/url]> for further information.\n" - "We use ^ as a shorthand for ctrl-.\n" - "\n" - "Overview:\n" - "\n[ansi-lightgray]" - " home,ctrl-a cursor end,ctrl-e\n" - " ┌────────────────┼───────────────┐ (navigate)\n" - //" │ │ │\n" - #ifndef __APPLE__ - " │ ctrl-left │ ctrl-right │\n" - #else - " │ alt-left │ alt-right │\n" - #endif - " │ ┌───────┼──────┐ │ ctrl-r : search history\n" - " ▼ ▼ ▼ ▼ ▼ tab : complete word\n" - " prompt> [ansi-darkgray]it's the quintessential language[/] shift-tab: insert new line\n" - " ▲ ▲ ▲ ▲ esc : delete input, done\n" - " │ └──────────────┘ │ ctrl-z : undo\n" - " │ alt-backsp alt-d │\n" - //" │ │ │\n" - " └────────────────────────────────┘ (delete)\n" - " ctrl-u ctrl-k\n" - "[/ansi-lightgray][/ic-info]\n"; - -static void edit_show_help(ic_env_t* env, editor_t* eb) { - edit_clear(env, eb); - bbcode_println(env->bbcode, help_initial); - for (ssize_t i = 0; help[i] != NULL && help[i+1] != NULL; i += 2) { - if (help[i][0] == 0) { - bbcode_printf(env->bbcode, "[ic-info]%s[/]\n", help[i+1]); - } - else { - bbcode_printf(env->bbcode, " [ic-emphasis]%-13s[/][ansi-lightgray]%s%s[/]\n", help[i], (help[i+1][0] == 0 ? "" : ": "), help[i+1]); - } - } - - eb->cur_rows = 0; - eb->cur_row = 0; - edit_refresh(env, eb); -} diff --git a/vendor/isocline/editline_history.c b/vendor/isocline/editline_history.c deleted file mode 100644 index 2a0afa1c..00000000 --- a/vendor/isocline/editline_history.c +++ /dev/null @@ -1,260 +0,0 @@ -/* ---------------------------------------------------------------------------- - Copyright (c) 2021, Daan Leijen - This is free software; you can redistribute it and/or modify it - under the terms of the MIT License. A copy of the license can be - found in the "LICENSE" file at the root of this distribution. ------------------------------------------------------------------------------*/ - -//------------------------------------------------------------- -// History search: this file is included in editline.c -//------------------------------------------------------------- - -static void edit_history_at(ic_env_t* env, editor_t* eb, int ofs ) -{ - if (eb->modified) { - history_update(env->history, sbuf_string(eb->input)); // update first entry if modified - eb->history_idx = 0; // and start again - eb->modified = false; - } - const char* entry = history_get(env->history,eb->history_idx + ofs); - // debug_msg( "edit: history: at: %d + %d, found: %s\n", eb->history_idx, ofs, entry); - if (entry == NULL) { - term_beep(env->term); - } - else { - eb->history_idx += ofs; - sbuf_replace(eb->input, entry); - if (ofs > 0) { - // at end of first line when scrolling up - ssize_t end = sbuf_find_line_end(eb->input,0); - eb->pos = (end < 0 ? 0 : end); - } - else { - eb->pos = sbuf_len(eb->input); // at end of last line when scrolling down - } - edit_refresh(env, eb); - } -} - -static void edit_history_prev(ic_env_t* env, editor_t* eb) { - edit_history_at(env,eb, 1 ); -} - -static void edit_history_next(ic_env_t* env, editor_t* eb) { - edit_history_at(env,eb, -1 ); -} - -typedef struct hsearch_s { - struct hsearch_s* next; - ssize_t hidx; - ssize_t match_pos; - ssize_t match_len; - bool cinsert; -} hsearch_t; - -static void hsearch_push( alloc_t* mem, hsearch_t** hs, ssize_t hidx, ssize_t mpos, ssize_t mlen, bool cinsert ) { - hsearch_t* h = mem_zalloc_tp( mem, hsearch_t ); - if (h == NULL) return; - h->hidx = hidx; - h->match_pos = mpos; - h->match_len = mlen; - h->cinsert = cinsert; - h->next = *hs; - *hs = h; -} - -static bool hsearch_pop( alloc_t* mem, hsearch_t** hs, ssize_t* hidx, ssize_t* match_pos, ssize_t* match_len, bool* cinsert ) { - hsearch_t* h = *hs; - if (h == NULL) return false; - *hs = h->next; - if (hidx != NULL) *hidx = h->hidx; - if (match_pos != NULL) *match_pos = h->match_pos; - if (match_len != NULL) *match_len = h->match_len; - if (cinsert != NULL) *cinsert = h->cinsert; - mem_free(mem, h); - return true; -} - -static void hsearch_done( alloc_t* mem, hsearch_t* hs ) { - while (hs != NULL) { - hsearch_t* next = hs->next; - mem_free(mem, hs); - hs = next; - } -} - -static void edit_history_search(ic_env_t* env, editor_t* eb, char* initial ) { - if (history_count( env->history ) <= 0) { - term_beep(env->term); - return; - } - - // update history - if (eb->modified) { - history_update(env->history, sbuf_string(eb->input)); // update first entry if modified - eb->history_idx = 0; // and start again - eb->modified = false; - } - - // set a search prompt and remember the previous state - editor_undo_capture(eb); - eb->disable_undo = true; - bool old_hint = ic_enable_hint(false); - const char* prompt_text = eb->prompt_text; - eb->prompt_text = "history search"; - - // search state - hsearch_t* hs = NULL; // search undo - ssize_t hidx = 1; // current history entry - ssize_t match_pos = 0; // current matched position - ssize_t match_len = 0; // length of the match - const char* hentry = NULL; // current history entry - - // Simulate per character searches for each letter in `initial` (so backspace works) - if (initial != NULL) { - const ssize_t initial_len = ic_strlen(initial); - ssize_t ipos = 0; - while( ipos < initial_len ) { - ssize_t next = str_next_ofs( initial, initial_len, ipos, NULL ); - if (next < 0) break; - hsearch_push( eb->mem, &hs, hidx, match_pos, match_len, true); - char c = initial[ipos + next]; // terminate temporarily - initial[ipos + next] = 0; - if (history_search( env->history, hidx, initial, true, &hidx, &match_pos )) { - match_len = ipos + next; - } - else if (ipos + next >= initial_len) { - term_beep(env->term); - } - initial[ipos + next] = c; // restore - ipos += next; - } - sbuf_replace( eb->input, initial); - eb->pos = ipos; - } - else { - sbuf_clear( eb->input ); - eb->pos = 0; - } - - // Incremental search -again: - hentry = history_get(env->history,hidx); - if (hentry != NULL) { - sbuf_appendf(eb->extra, "[ic-info]%zd. [/][ic-diminish][!pre]", hidx); - sbuf_append_n( eb->extra, hentry, match_pos ); - sbuf_append(eb->extra, "[/pre][u ic-emphasis][!pre]" ); - sbuf_append_n( eb->extra, hentry + match_pos, match_len ); - sbuf_append(eb->extra, "[/pre][/u][!pre]" ); - sbuf_append(eb->extra, hentry + match_pos + match_len ); - sbuf_append(eb->extra, "[/pre][/ic-diminish]"); - if (!env->no_help) { - sbuf_append(eb->extra, "\n[ic-info](use tab for the next match)[/]"); - } - sbuf_append(eb->extra, "\n" ); - } - edit_refresh(env, eb); - - // Wait for input - code_t c = (hentry == NULL ? KEY_ESC : tty_read(env->tty)); - if (tty_term_resize_event(env->tty)) { - edit_resize(env, eb); - } - sbuf_clear(eb->extra); - - // Process commands - if (c == KEY_ESC || c == KEY_BELL /* ^G */ || c == KEY_CTRL_C) { - c = 0; - eb->disable_undo = false; - editor_undo_restore(eb, false); - } - else if (c == KEY_ENTER) { - c = 0; - editor_undo_forget(eb); - sbuf_replace( eb->input, hentry ); - eb->pos = sbuf_len(eb->input); - eb->modified = false; - eb->history_idx = hidx; - } - else if (c == KEY_BACKSP || c == KEY_CTRL_Z) { - // undo last search action - bool cinsert; - if (hsearch_pop(env->mem,&hs, &hidx, &match_pos, &match_len, &cinsert)) { - if (cinsert) edit_backspace(env,eb); - } - goto again; - } - else if (c == KEY_CTRL_R || c == KEY_TAB || c == KEY_UP) { - // search backward - hsearch_push(env->mem, &hs, hidx, match_pos, match_len, false); - if (!history_search( env->history, hidx+1, sbuf_string(eb->input), true, &hidx, &match_pos )) { - hsearch_pop(env->mem,&hs,NULL,NULL,NULL,NULL); - term_beep(env->term); - }; - goto again; - } - else if (c == KEY_CTRL_S || c == KEY_SHIFT_TAB || c == KEY_DOWN) { - // search forward - hsearch_push(env->mem, &hs, hidx, match_pos, match_len, false); - if (!history_search( env->history, hidx-1, sbuf_string(eb->input), false, &hidx, &match_pos )) { - hsearch_pop(env->mem, &hs,NULL,NULL,NULL,NULL); - term_beep(env->term); - }; - goto again; - } - else if (c == KEY_F1) { - edit_show_help(env, eb); - goto again; - } - else { - // insert character and search further backward - char chr; - unicode_t uchr; - if (code_is_ascii_char(c,&chr)) { - hsearch_push(env->mem, &hs, hidx, match_pos, match_len, true); - edit_insert_char(env,eb,chr); - } - else if (code_is_unicode(c,&uchr)) { - hsearch_push(env->mem, &hs, hidx, match_pos, match_len, true); - edit_insert_unicode(env,eb,uchr); - } - else { - // ignore command - term_beep(env->term); - goto again; - } - // search for the new input - if (history_search( env->history, hidx, sbuf_string(eb->input), true, &hidx, &match_pos )) { - match_len = sbuf_len(eb->input); - } - else { - term_beep(env->term); - }; - goto again; - } - - // done - eb->disable_undo = false; - hsearch_done(env->mem,hs); - eb->prompt_text = prompt_text; - ic_enable_hint(old_hint); - edit_refresh(env,eb); - if (c != 0) tty_code_pushback(env->tty, c); -} - -// Start an incremental search with the current word -static void edit_history_search_with_current_word(ic_env_t* env, editor_t* eb) { - char* initial = NULL; - ssize_t start = sbuf_find_word_start( eb->input, eb->pos ); - if (start >= 0) { - const ssize_t next = sbuf_next(eb->input, start, NULL); - if (!ic_char_is_idletter(sbuf_string(eb->input) + start, (long)(next - start))) { - start = next; - } - if (start >= 0 && start < eb->pos) { - initial = mem_strndup(eb->mem, sbuf_string(eb->input) + start, eb->pos - start); - } - } - edit_history_search( env, eb, initial); - mem_free(env->mem, initial); -} diff --git a/vendor/isocline/env.h b/vendor/isocline/env.h deleted file mode 100644 index a5094a18..00000000 --- a/vendor/isocline/env.h +++ /dev/null @@ -1,60 +0,0 @@ -/* ---------------------------------------------------------------------------- - Copyright (c) 2021, Daan Leijen - This is free software; you can redistribute it and/or modify it - under the terms of the MIT License. A copy of the license can be - found in the "LICENSE" file at the root of this distribution. ------------------------------------------------------------------------------*/ -#pragma once -#ifndef IC_ENV_H -#define IC_ENV_H - -#include "./isocline.h" -#include "common.h" -#include "term.h" -#include "tty.h" -#include "stringbuf.h" -#include "history.h" -#include "completions.h" -#include "bbcode.h" - -//------------------------------------------------------------- -// Environment -//------------------------------------------------------------- - -struct ic_env_s { - alloc_t* mem; // potential custom allocator - ic_env_t* next; // next environment (used for proper deallocation) - term_t* term; // terminal - tty_t* tty; // keyboard (NULL if stdin is a pipe, file, etc) - completions_t* completions; // current completions - history_t* history; // edit history - bbcode_t* bbcode; // print with bbcodes - const char* prompt_marker; // the prompt marker (defaults to "> ") - const char* cprompt_marker; // prompt marker for continuation lines (defaults to `prompt_marker`) - ic_highlight_fun_t* highlighter; // highlight callback - void* highlighter_arg; // user state for the highlighter. - const char* match_braces; // matching braces, e.g "()[]{}" - const char* auto_braces; // auto insertion braces, e.g "()[]{}\"\"''" - char multiline_eol; // character used for multiline input ("\") (set to 0 to disable) - bool initialized; // are we initialized? - bool noedit; // is rich editing possible (tty != NULL) - bool singleline_only; // allow only single line editing? - bool complete_nopreview; // do not show completion preview for each selection in the completion menu? - bool complete_autotab; // try to keep completing after a completion? - bool no_multiline_indent; // indent continuation lines to line up under the initial prompt - bool no_help; // show short help line for history search etc. - bool no_hint; // allow hinting? - bool no_highlight; // enable highlighting? - bool no_bracematch; // enable brace matching? - bool no_autobrace; // enable automatic brace insertion? - bool no_lscolors; // use LSCOLORS/LS_COLORS to colorize file name completions? - long hint_delay; // delay before displaying a hint in milliseconds -}; - -ic_private char* ic_editline(ic_env_t* env, const char* prompt_text); - -ic_private ic_env_t* ic_get_env(void); -ic_private const char* ic_env_get_auto_braces(ic_env_t* env); -ic_private const char* ic_env_get_match_braces(ic_env_t* env); - -#endif // IC_ENV_H diff --git a/vendor/isocline/highlight.c b/vendor/isocline/highlight.c deleted file mode 100644 index 59c7255c..00000000 --- a/vendor/isocline/highlight.c +++ /dev/null @@ -1,259 +0,0 @@ -/* ---------------------------------------------------------------------------- - Copyright (c) 2021, Daan Leijen - This is free software; you can redistribute it and/or modify it - under the terms of the MIT License. A copy of the license can be - found in the "LICENSE" file at the root of this distribution. ------------------------------------------------------------------------------*/ - -#include -#include "common.h" -#include "term.h" -#include "stringbuf.h" -#include "attr.h" -#include "bbcode.h" - -//------------------------------------------------------------- -// Syntax highlighting -//------------------------------------------------------------- - -struct ic_highlight_env_s { - attrbuf_t* attrs; - const char* input; - ssize_t input_len; - bbcode_t* bbcode; - alloc_t* mem; - ssize_t cached_upos; // cached unicode position - ssize_t cached_cpos; // corresponding utf-8 byte position -}; - - -ic_private void highlight( alloc_t* mem, bbcode_t* bb, const char* s, attrbuf_t* attrs, ic_highlight_fun_t* highlighter, void* arg ) { - const ssize_t len = ic_strlen(s); - if (len <= 0) return; - attrbuf_set_at(attrs,0,len,attr_none()); // fill to length of s - if (highlighter != NULL) { - ic_highlight_env_t henv; - henv.attrs = attrs; - henv.input = s; - henv.input_len = len; - henv.bbcode = bb; - henv.mem = mem; - henv.cached_cpos = 0; - henv.cached_upos = 0; - (*highlighter)( &henv, s, arg ); - } -} - - -//------------------------------------------------------------- -// Client interface -//------------------------------------------------------------- - -static void pos_adjust( ic_highlight_env_t* henv, ssize_t* ppos, ssize_t* plen ) { - ssize_t pos = *ppos; - ssize_t len = *plen; - if (pos >= henv->input_len) return; - if (pos >= 0 && len >= 0) return; // already character positions - if (henv->input == NULL) return; - - if (pos < 0) { - // negative `pos` is used as the unicode character position (for easy interfacing with Haskell) - ssize_t upos = -pos; - ssize_t cpos = 0; - ssize_t ucount = 0; - if (henv->cached_upos <= upos) { // if we have a cached position, start from there - ucount = henv->cached_upos; - cpos = henv->cached_cpos; - } - while ( ucount < upos ) { - ssize_t next = str_next_ofs(henv->input, henv->input_len, cpos, NULL); - if (next <= 0) return; - ucount++; - cpos += next; - } - *ppos = pos = cpos; - // and cache it to avoid quadratic behavior - henv->cached_upos = upos; - henv->cached_cpos = cpos; - } - if (len < 0) { - // negative `len` is used as a unicode character length - len = -len; - ssize_t ucount = 0; - ssize_t clen = 0; - while (ucount < len) { - ssize_t next = str_next_ofs(henv->input, henv->input_len, pos + clen, NULL); - if (next <= 0) return; - ucount++; - clen += next; - } - *plen = len = clen; - // and update cache if possible - if (henv->cached_cpos == pos) { - henv->cached_upos += ucount; - henv->cached_cpos += clen; - } - } -} - -static void highlight_attr(ic_highlight_env_t* henv, ssize_t pos, ssize_t count, attr_t attr ) { - if (henv==NULL) return; - pos_adjust(henv,&pos,&count); - if (pos < 0 || count <= 0) return; - attrbuf_update_at(henv->attrs, pos, count, attr); -} - -ic_public void ic_highlight(ic_highlight_env_t* henv, long pos, long count, const char* style ) { - if (henv == NULL || style==NULL || style[0]==0 || pos < 0) return; - highlight_attr(henv,pos,count,bbcode_style( henv->bbcode, style )); -} - -ic_public void ic_highlight_formatted(ic_highlight_env_t* henv, const char* s, const char* fmt) { - if (s==NULL || s[0] == 0 || fmt==NULL) return; - attrbuf_t* attrs = attrbuf_new(henv->mem); - stringbuf_t* out = sbuf_new(henv->mem); // todo: avoid allocating out? - if (attrs!=NULL && out != NULL) { - bbcode_append( henv->bbcode, fmt, out, attrs); - const ssize_t len = ic_strlen(s); - if (sbuf_len(out) != len) { - debug_msg("highlight: formatted string content differs from the original input:\n original: %s\n formatted: %s\n", s, fmt); - } - for( ssize_t i = 0; i < len; i++) { - attrbuf_update_at(henv->attrs, i, 1, attrbuf_attr_at(attrs,i)); - } - } - sbuf_free(out); - attrbuf_free(attrs); -} - -//------------------------------------------------------------- -// Brace matching -//------------------------------------------------------------- -#define MAX_NESTING (64) - -typedef struct brace_s { - char close; - bool at_cursor; - ssize_t pos; -} brace_t; - -ic_private void highlight_match_braces(const char* s, attrbuf_t* attrs, ssize_t cursor_pos, const char* braces, attr_t match_attr, attr_t error_attr) -{ - brace_t open[MAX_NESTING+1]; - ssize_t nesting = 0; - const ssize_t brace_len = ic_strlen(braces); - for (long i = 0; i < ic_strlen(s); i++) { - const char c = s[i]; - // push open brace - bool found_open = false; - for (ssize_t b = 0; b < brace_len; b += 2) { - if (c == braces[b]) { - // open brace - if (nesting >= MAX_NESTING) return; // give up - open[nesting].close = braces[b+1]; - open[nesting].pos = i; - open[nesting].at_cursor = (i == cursor_pos - 1); - nesting++; - found_open = true; - break; - } - } - if (found_open) continue; - - // pop to closing brace and potentially highlight - for (ssize_t b = 1; b < brace_len; b += 2) { - if (c == braces[b]) { - // close brace - if (nesting <= 0) { - // unmatched close brace - attrbuf_update_at( attrs, i, 1, error_attr); - } - else { - // can we fix an unmatched brace where we can match by popping just one? - if (open[nesting-1].close != c && nesting > 1 && open[nesting-2].close == c) { - // assume previous open brace was wrong - attrbuf_update_at(attrs, open[nesting-1].pos, 1, error_attr); - nesting--; - } - if (open[nesting-1].close != c) { - // unmatched open brace - attrbuf_update_at( attrs, i, 1, error_attr); - } - else { - // matching brace - nesting--; - if (i == cursor_pos - 1 || (open[nesting].at_cursor && open[nesting].pos != i - 1)) { - // highlight matching brace - attrbuf_update_at(attrs, open[nesting].pos, 1, match_attr); - attrbuf_update_at(attrs, i, 1, match_attr); - } - } - } - break; - } - } - } - // note: don't mark further unmatched open braces as in error -} - - -ic_private ssize_t find_matching_brace(const char* s, ssize_t cursor_pos, const char* braces, bool* is_balanced) -{ - if (is_balanced != NULL) { *is_balanced = false; } - bool balanced = true; - ssize_t match = -1; - brace_t open[MAX_NESTING+1]; - ssize_t nesting = 0; - const ssize_t brace_len = ic_strlen(braces); - for (long i = 0; i < ic_strlen(s); i++) { - const char c = s[i]; - // push open brace - bool found_open = false; - for (ssize_t b = 0; b < brace_len; b += 2) { - if (c == braces[b]) { - // open brace - if (nesting >= MAX_NESTING) return -1; // give up - open[nesting].close = braces[b+1]; - open[nesting].pos = i; - open[nesting].at_cursor = (i == cursor_pos - 1); - nesting++; - found_open = true; - break; - } - } - if (found_open) continue; - - // pop to closing brace - for (ssize_t b = 1; b < brace_len; b += 2) { - if (c == braces[b]) { - // close brace - if (nesting <= 0) { - // unmatched close brace - balanced = false; - } - else { - if (open[nesting-1].close != c) { - // unmatched open brace - balanced = false; - } - else { - // matching brace - nesting--; - if (i == cursor_pos - 1) { - // found matching open brace - match = open[nesting].pos + 1; - } - else if (open[nesting].at_cursor) { - // found matching close brace - match = i + 1; - } - } - } - break; - } - } - } - if (nesting != 0) { balanced = false; } - if (is_balanced != NULL) { *is_balanced = balanced; } - return match; -} diff --git a/vendor/isocline/highlight.h b/vendor/isocline/highlight.h deleted file mode 100644 index 67da02ff..00000000 --- a/vendor/isocline/highlight.h +++ /dev/null @@ -1,24 +0,0 @@ -/* ---------------------------------------------------------------------------- - Copyright (c) 2021, Daan Leijen - This is free software; you can redistribute it and/or modify it - under the terms of the MIT License. A copy of the license can be - found in the "LICENSE" file at the root of this distribution. ------------------------------------------------------------------------------*/ -#pragma once -#ifndef IC_HIGHLIGHT_H -#define IC_HIGHLIGHT_H - -#include "common.h" -#include "attr.h" -#include "term.h" -#include "bbcode.h" - -//------------------------------------------------------------- -// Syntax highlighting -//------------------------------------------------------------- - -ic_private void highlight( alloc_t* mem, bbcode_t* bb, const char* s, attrbuf_t* attrs, ic_highlight_fun_t* highlighter, void* arg ); -ic_private void highlight_match_braces(const char* s, attrbuf_t* attrs, ssize_t cursor_pos, const char* braces, attr_t match_attr, attr_t error_attr); -ic_private ssize_t find_matching_brace(const char* s, ssize_t cursor_pos, const char* braces, bool* is_balanced); - -#endif // IC_HIGHLIGHT_H diff --git a/vendor/isocline/history.c b/vendor/isocline/history.c deleted file mode 100644 index bff53efd..00000000 --- a/vendor/isocline/history.c +++ /dev/null @@ -1,269 +0,0 @@ -/* ---------------------------------------------------------------------------- - Copyright (c) 2021, Daan Leijen - This is free software; you can redistribute it and/or modify it - under the terms of the MIT License. A copy of the license can be - found in the "LICENSE" file at the root of this distribution. ------------------------------------------------------------------------------*/ -#include -#include -#include - -#include "./isocline.h" -#include "common.h" -#include "history.h" -#include "stringbuf.h" - -#define IC_MAX_HISTORY (200) - -struct history_s { - ssize_t count; // current number of entries in use - ssize_t len; // size of elems - const char** elems; // history items (up to count) - const char* fname; // history file - alloc_t* mem; - bool allow_duplicates; // allow duplicate entries? -}; - -ic_private history_t* history_new(alloc_t* mem) { - history_t* h = mem_zalloc_tp(mem,history_t); - h->mem = mem; - return h; -} - -ic_private void history_free(history_t* h) { - if (h == NULL) return; - history_clear(h); - if (h->len > 0) { - mem_free( h->mem, h->elems ); - h->elems = NULL; - h->len = 0; - } - mem_free(h->mem, h->fname); - h->fname = NULL; - mem_free(h->mem, h); // free ourselves -} - -ic_private bool history_enable_duplicates( history_t* h, bool enable ) { - bool prev = h->allow_duplicates; - h->allow_duplicates = enable; - return prev; -} - -ic_private ssize_t history_count(const history_t* h) { - return h->count; -} - -//------------------------------------------------------------- -// push/clear -//------------------------------------------------------------- - -ic_private bool history_update( history_t* h, const char* entry ) { - if (entry==NULL) return false; - history_remove_last(h); - history_push(h,entry); - //debug_msg("history: update: with %s; now at %s\n", entry, history_get(h,0)); - return true; -} - -static void history_delete_at( history_t* h, ssize_t idx ) { - if (idx < 0 || idx >= h->count) return; - mem_free(h->mem, h->elems[idx]); - for(ssize_t i = idx+1; i < h->count; i++) { - h->elems[i-1] = h->elems[i]; - } - h->count--; -} - -ic_private bool history_push( history_t* h, const char* entry ) { - if (h->len <= 0 || entry==NULL) return false; - // remove any older duplicate - if (!h->allow_duplicates) { - for( int i = 0; i < h->count; i++) { - if (strcmp(h->elems[i],entry) == 0) { - history_delete_at(h,i); - } - } - } - // insert at front - if (h->count == h->len) { - // delete oldest entry - history_delete_at(h,0); - } - assert(h->count < h->len); - h->elems[h->count] = mem_strdup(h->mem,entry); - h->count++; - return true; -} - - -static void history_remove_last_n( history_t* h, ssize_t n ) { - if (n <= 0) return; - if (n > h->count) n = h->count; - for( ssize_t i = h->count - n; i < h->count; i++) { - mem_free( h->mem, h->elems[i] ); - } - h->count -= n; - assert(h->count >= 0); -} - -ic_private void history_remove_last(history_t* h) { - history_remove_last_n(h,1); -} - -ic_private void history_clear(history_t* h) { - history_remove_last_n( h, h->count ); -} - -ic_private const char* history_get( const history_t* h, ssize_t n ) { - if (n < 0 || n >= h->count) return NULL; - return h->elems[h->count - n - 1]; -} - -ic_private bool history_search( const history_t* h, ssize_t from /*including*/, const char* search, bool backward, ssize_t* hidx, ssize_t* hpos ) { - const char* p = NULL; - ssize_t i; - if (backward) { - for( i = from; i < h->count; i++ ) { - p = strstr( history_get(h,i), search); - if (p != NULL) break; - } - } - else { - for( i = from; i >= 0; i-- ) { - p = strstr( history_get(h,i), search); - if (p != NULL) break; - } - } - if (p == NULL) return false; - if (hidx != NULL) *hidx = i; - if (hpos != NULL) *hpos = (p - history_get(h,i)); - return true; -} - -//------------------------------------------------------------- -// -//------------------------------------------------------------- - -ic_private void history_load_from(history_t* h, const char* fname, long max_entries ) { - history_clear(h); - h->fname = mem_strdup(h->mem,fname); - if (max_entries == 0) { - assert(h->elems == NULL); - return; - } - if (max_entries < 0 || max_entries > IC_MAX_HISTORY) max_entries = IC_MAX_HISTORY; - h->elems = (const char**)mem_zalloc_tp_n(h->mem, char*, max_entries ); - if (h->elems == NULL) return; - h->len = max_entries; - history_load(h); -} - - - - -//------------------------------------------------------------- -// save/load history to file -//------------------------------------------------------------- - -static char from_xdigit( int c ) { - if (c >= '0' && c <= '9') return (char)(c - '0'); - if (c >= 'A' && c <= 'F') return (char)(10 + (c - 'A')); - if (c >= 'a' && c <= 'f') return (char)(10 + (c - 'a')); - return 0; -} - -static char to_xdigit( uint8_t c ) { - if (c <= 9) return ((char)c + '0'); - if (c >= 10 && c <= 15) return ((char)c - 10 + 'A'); - return '0'; -} - -static bool ic_isxdigit( int c ) { - return ((c >= 'a' && c <= 'f') || (c >= 'A' && c <= 'F') || (c >= '0' && c <= '9')); -} - -static bool history_read_entry( history_t* h, FILE* f, stringbuf_t* sbuf ) { - sbuf_clear(sbuf); - while( !feof(f)) { - int c = fgetc(f); - if (c == EOF || c == '\n') break; - if (c == '\\') { - c = fgetc(f); - if (c == 'n') { sbuf_append(sbuf,"\n"); } - else if (c == 'r') { /* ignore */ } // sbuf_append(sbuf,"\r"); - else if (c == 't') { sbuf_append(sbuf,"\t"); } - else if (c == '\\') { sbuf_append(sbuf,"\\"); } - else if (c == 'x') { - int c1 = fgetc(f); - int c2 = fgetc(f); - if (ic_isxdigit(c1) && ic_isxdigit(c2)) { - char chr = from_xdigit(c1)*16 + from_xdigit(c2); - sbuf_append_char(sbuf,chr); - } - else return false; - } - else return false; - } - else sbuf_append_char(sbuf,(char)c); - } - if (sbuf_len(sbuf)==0 || sbuf_string(sbuf)[0] == '#') return true; - return history_push(h, sbuf_string(sbuf)); -} - -static bool history_write_entry( const char* entry, FILE* f, stringbuf_t* sbuf ) { - sbuf_clear(sbuf); - //debug_msg("history: write: %s\n", entry); - while( entry != NULL && *entry != 0 ) { - char c = *entry++; - if (c == '\\') { sbuf_append(sbuf,"\\\\"); } - else if (c == '\n') { sbuf_append(sbuf,"\\n"); } - else if (c == '\r') { /* ignore */ } // sbuf_append(sbuf,"\\r"); } - else if (c == '\t') { sbuf_append(sbuf,"\\t"); } - else if (c < ' ' || c > '~' || c == '#') { - char c1 = to_xdigit( (uint8_t)c / 16 ); - char c2 = to_xdigit( (uint8_t)c % 16 ); - sbuf_append(sbuf,"\\x"); - sbuf_append_char(sbuf,c1); - sbuf_append_char(sbuf,c2); - } - else sbuf_append_char(sbuf,c); - } - //debug_msg("history: write buf: %s\n", sbuf_string(sbuf)); - - if (sbuf_len(sbuf) > 0) { - sbuf_append(sbuf,"\n"); - fputs(sbuf_string(sbuf),f); - } - return true; -} - -ic_private void history_load( history_t* h ) { - if (h->fname == NULL) return; - FILE* f = fopen(h->fname, "r"); - if (f == NULL) return; - stringbuf_t* sbuf = sbuf_new(h->mem); - if (sbuf != NULL) { - while (!feof(f)) { - if (!history_read_entry(h,f,sbuf)) break; // error - } - sbuf_free(sbuf); - } - fclose(f); -} - -ic_private void history_save( const history_t* h ) { - if (h->fname == NULL) return; - FILE* f = fopen(h->fname, "w"); - if (f == NULL) return; - #ifndef _WIN32 - chmod(h->fname,S_IRUSR|S_IWUSR); - #endif - stringbuf_t* sbuf = sbuf_new(h->mem); - if (sbuf != NULL) { - for( int i = 0; i < h->count; i++ ) { - if (!history_write_entry(h->elems[i],f,sbuf)) break; // error - } - sbuf_free(sbuf); - } - fclose(f); -} diff --git a/vendor/isocline/history.h b/vendor/isocline/history.h deleted file mode 100644 index 76a37160..00000000 --- a/vendor/isocline/history.h +++ /dev/null @@ -1,38 +0,0 @@ -/* ---------------------------------------------------------------------------- - Copyright (c) 2021, Daan Leijen - This is free software; you can redistribute it and/or modify it - under the terms of the MIT License. A copy of the license can be - found in the "LICENSE" file at the root of this distribution. ------------------------------------------------------------------------------*/ -#pragma once -#ifndef IC_HISTORY_H -#define IC_HISTORY_H - -#include "common.h" - -//------------------------------------------------------------- -// History -//------------------------------------------------------------- - -struct history_s; -typedef struct history_s history_t; - -ic_private history_t* history_new(alloc_t* mem); -ic_private void history_free(history_t* h); -ic_private void history_clear(history_t* h); -ic_private bool history_enable_duplicates( history_t* h, bool enable ); -ic_private ssize_t history_count(const history_t* h); - -ic_private void history_load_from(history_t* h, const char* fname, long max_entries); -ic_private void history_load( history_t* h ); -ic_private void history_save( const history_t* h ); - -ic_private bool history_push( history_t* h, const char* entry ); -ic_private bool history_update( history_t* h, const char* entry ); -ic_private const char* history_get( const history_t* h, ssize_t n ); -ic_private void history_remove_last(history_t* h); - -ic_private bool history_search( const history_t* h, ssize_t from, const char* search, bool backward, ssize_t* hidx, ssize_t* hpos); - - -#endif // IC_HISTORY_H diff --git a/vendor/isocline/isocline.c b/vendor/isocline/isocline.c deleted file mode 100644 index 0dff8d6e..00000000 --- a/vendor/isocline/isocline.c +++ /dev/null @@ -1,774 +0,0 @@ -/* ---------------------------------------------------------------------------- - Copyright (c) 2021, Daan Leijen - This is free software; you can redistribute it and/or modify it - under the terms of the MIT License. A copy of the license can be - found in the "LICENSE" file at the root of this distribution. ------------------------------------------------------------------------------*/ - -//------------------------------------------------------------- -// Usually we include all sources one file so no internal -// symbols are public in the libray. -// -// You can compile the entire library just as: -// $ gcc -c src/isocline.c -//------------------------------------------------------------- -#if !defined(IC_SEPARATE_OBJS) -#ifndef _CRT_NONSTDC_NO_WARNINGS -#define _CRT_NONSTDC_NO_WARNINGS // for msvc -#endif -#ifndef _CRT_SECURE_NO_WARNINGS -#define _CRT_SECURE_NO_WARNINGS // for msvc -#endif -#define _XOPEN_SOURCE 700 // for wcwidth -#define _DEFAULT_SOURCE // ensure usleep stays visible with _XOPEN_SOURCE >= 700 -#include "attr.c" -#include "bbcode.c" -#include "common.c" -#include "completers.c" -#include "completions.c" -#include "editline.c" -#include "highlight.c" -#include "history.c" -#include "stringbuf.c" -#include "term.c" -#include "tty.c" -#include "tty_esc.c" -#include "undo.c" -#endif - -//------------------------------------------------------------- -// includes -//------------------------------------------------------------- -#include -#include -#include -#include - -#include "./isocline.h" -#include "common.h" -#include "env.h" - -//------------------------------------------------------------- -// Readline -//------------------------------------------------------------- - -static char *ic_getline(alloc_t *mem); - -ic_public char *ic_readline(const char *prompt_text) { - ic_env_t *env = ic_get_env(); - if (env == NULL) { - return NULL; - } - if (!env->noedit) { - // terminal editing enabled - return ic_editline(env, prompt_text); // in editline.c - } else { - // no editing capability (pipe, dumb terminal, etc) - if (env->tty != NULL && env->term != NULL) { - // if the terminal is not interactive, but we are reading from the tty (keyboard), we display a prompt - term_start_raw(env->term); // set utf8 mode on windows - if (prompt_text != NULL) { - term_write(env->term, prompt_text); - } - term_write(env->term, env->prompt_marker); - term_end_raw(env->term, false); - } - // read directly from stdin - return ic_getline(env->mem); - } -} - -//------------------------------------------------------------- -// Read a line from the stdin stream if there is no editing -// support (like from a pipe, file, or dumb terminal). -//------------------------------------------------------------- - -static char *ic_getline(alloc_t *mem) { - // read until eof or newline - stringbuf_t *sb = sbuf_new(mem); - int c; - while (true) { - c = fgetc(stdin); - if (c == EOF || c == '\n') { - break; - } else { - sbuf_append_char(sb, (char)c); - } - } - return sbuf_free_dup(sb); -} - -//------------------------------------------------------------- -// Formatted output -//------------------------------------------------------------- - -ic_public void ic_printf(const char *fmt, ...) { - va_list ap; - va_start(ap, fmt); - ic_vprintf(fmt, ap); - va_end(ap); -} - -ic_public void ic_vprintf(const char *fmt, va_list args) { - ic_env_t *env = ic_get_env(); - if (env == NULL || env->bbcode == NULL) { - return; - } - bbcode_vprintf(env->bbcode, fmt, args); -} - -ic_public void ic_print(const char *s) { - ic_env_t *env = ic_get_env(); - if (env == NULL || env->bbcode == NULL) { - return; - } - bbcode_print(env->bbcode, s); -} - -ic_public void ic_println(const char *s) { - ic_env_t *env = ic_get_env(); - if (env == NULL || env->bbcode == NULL) { - return; - } - bbcode_println(env->bbcode, s); -} - -void ic_style_def(const char *name, const char *fmt) { - ic_env_t *env = ic_get_env(); - if (env == NULL || env->bbcode == NULL) { - return; - } - bbcode_style_def(env->bbcode, name, fmt); -} - -void ic_style_open(const char *fmt) { - ic_env_t *env = ic_get_env(); - if (env == NULL || env->bbcode == NULL) { - return; - } - bbcode_style_open(env->bbcode, fmt); -} - -void ic_style_close(void) { - ic_env_t *env = ic_get_env(); - if (env == NULL || env->bbcode == NULL) { - return; - } - bbcode_style_close(env->bbcode, NULL); -} - -//------------------------------------------------------------- -// Interface -//------------------------------------------------------------- - -ic_public bool ic_async_stop(void) { - ic_env_t *env = ic_get_env(); - if (env == NULL) { - return false; - } - if (env->tty == NULL) { - return false; - } - return tty_async_stop(env->tty); -} - -static void set_prompt_marker(ic_env_t *env, const char *prompt_marker, const char *cprompt_marker) { - if (prompt_marker == NULL) { - prompt_marker = "> "; - } - if (cprompt_marker == NULL) { - cprompt_marker = prompt_marker; - } - mem_free(env->mem, env->prompt_marker); - mem_free(env->mem, env->cprompt_marker); - env->prompt_marker = mem_strdup(env->mem, prompt_marker); - env->cprompt_marker = mem_strdup(env->mem, cprompt_marker); -} - -ic_public const char *ic_get_prompt_marker(void) { - ic_env_t *env = ic_get_env(); - if (env == NULL) { - return NULL; - } - return env->prompt_marker; -} - -ic_public const char *ic_get_continuation_prompt_marker(void) { - ic_env_t *env = ic_get_env(); - if (env == NULL) { - return NULL; - } - return env->cprompt_marker; -} - -ic_public void ic_set_prompt_marker(const char *prompt_marker, const char *cprompt_marker) { - ic_env_t *env = ic_get_env(); - if (env == NULL) { - return; - } - set_prompt_marker(env, prompt_marker, cprompt_marker); -} - -ic_public bool ic_enable_multiline(bool enable) { - ic_env_t *env = ic_get_env(); - if (env == NULL) { - return false; - } - bool prev = env->singleline_only; - env->singleline_only = !enable; - return !prev; -} - -ic_public bool ic_enable_beep(bool enable) { - ic_env_t *env = ic_get_env(); - if (env == NULL) { - return false; - } - return term_enable_beep(env->term, enable); -} - -ic_public bool ic_enable_color(bool enable) { - ic_env_t *env = ic_get_env(); - if (env == NULL) { - return false; - } - return term_enable_color(env->term, enable); -} - -ic_public bool ic_enable_history_duplicates(bool enable) { - ic_env_t *env = ic_get_env(); - if (env == NULL) { - return false; - } - return history_enable_duplicates(env->history, enable); -} - -ic_public void ic_set_history(const char *fname, long max_entries) { - ic_env_t *env = ic_get_env(); - if (env == NULL) { - return; - } - history_load_from(env->history, fname, max_entries); -} - -ic_public void ic_history_remove_last(void) { - ic_env_t *env = ic_get_env(); - if (env == NULL) { - return; - } - history_remove_last(env->history); -} - -ic_public void ic_history_add(const char *entry) { - ic_env_t *env = ic_get_env(); - if (env == NULL) { - return; - } - history_push(env->history, entry); -} - -ic_public void ic_history_clear(void) { - ic_env_t *env = ic_get_env(); - if (env == NULL) { - return; - } - history_clear(env->history); -} - -ic_public bool ic_enable_auto_tab(bool enable) { - ic_env_t *env = ic_get_env(); - if (env == NULL) { - return false; - } - bool prev = env->complete_autotab; - env->complete_autotab = enable; - return prev; -} - -ic_public bool ic_enable_completion_preview(bool enable) { - ic_env_t *env = ic_get_env(); - if (env == NULL) { - return false; - } - bool prev = env->complete_nopreview; - env->complete_nopreview = !enable; - return !prev; -} - -ic_public bool ic_enable_multiline_indent(bool enable) { - ic_env_t *env = ic_get_env(); - if (env == NULL) { - return false; - } - bool prev = env->no_multiline_indent; - env->no_multiline_indent = !enable; - return !prev; -} - -ic_public bool ic_enable_hint(bool enable) { - ic_env_t *env = ic_get_env(); - if (env == NULL) { - return false; - } - bool prev = env->no_hint; - env->no_hint = !enable; - return !prev; -} - -ic_public long ic_set_hint_delay(long delay_ms) { - ic_env_t *env = ic_get_env(); - if (env == NULL) { - return false; - } - long prev = env->hint_delay; - env->hint_delay = (delay_ms < 0 ? 0 : (delay_ms > 5000 ? 5000 : delay_ms)); - return prev; -} - -ic_public void ic_set_tty_esc_delay(long initial_delay_ms, long followup_delay_ms) { - ic_env_t *env = ic_get_env(); - if (env == NULL) { - return; - } - if (env->tty == NULL) { - return; - } - tty_set_esc_delay(env->tty, initial_delay_ms, followup_delay_ms); -} - -ic_public bool ic_enable_highlight(bool enable) { - ic_env_t *env = ic_get_env(); - if (env == NULL) { - return false; - } - bool prev = env->no_highlight; - env->no_highlight = !enable; - return !prev; -} - -ic_public bool ic_enable_inline_help(bool enable) { - ic_env_t *env = ic_get_env(); - if (env == NULL) { - return false; - } - bool prev = env->no_help; - env->no_help = !enable; - return !prev; -} - -ic_public bool ic_enable_brace_matching(bool enable) { - ic_env_t *env = ic_get_env(); - if (env == NULL) { - return false; - } - bool prev = env->no_bracematch; - env->no_bracematch = !enable; - return !prev; -} - -ic_public void ic_set_matching_braces(const char *brace_pairs) { - ic_env_t *env = ic_get_env(); - if (env == NULL) { - return; - } - mem_free(env->mem, env->match_braces); - env->match_braces = NULL; - if (brace_pairs != NULL) { - ssize_t len = ic_strlen(brace_pairs); - if (len > 0 && (len % 2) == 0) { - env->match_braces = mem_strdup(env->mem, brace_pairs); - } - } -} - -ic_public bool ic_enable_brace_insertion(bool enable) { - ic_env_t *env = ic_get_env(); - if (env == NULL) { - return false; - } - bool prev = env->no_autobrace; - env->no_autobrace = !enable; - return !prev; -} - -ic_public void ic_set_insertion_braces(const char *brace_pairs) { - ic_env_t *env = ic_get_env(); - if (env == NULL) { - return; - } - mem_free(env->mem, env->auto_braces); - env->auto_braces = NULL; - if (brace_pairs != NULL) { - ssize_t len = ic_strlen(brace_pairs); - if (len > 0 && (len % 2) == 0) { - env->auto_braces = mem_strdup(env->mem, brace_pairs); - } - } -} - -ic_private const char *ic_env_get_match_braces(ic_env_t *env) { - return (env->match_braces == NULL ? "()[]{}" : env->match_braces); -} - -ic_private const char *ic_env_get_auto_braces(ic_env_t *env) { - return (env->auto_braces == NULL ? "()[]{}\"\"''" : env->auto_braces); -} - -ic_public void ic_set_default_highlighter(ic_highlight_fun_t *highlighter, void *arg) { - ic_env_t *env = ic_get_env(); - if (env == NULL) { - return; - } - env->highlighter = highlighter; - env->highlighter_arg = arg; -} - -ic_public void ic_free(void *p) { - ic_env_t *env = ic_get_env(); - if (env == NULL) { - return; - } - mem_free(env->mem, p); -} - -ic_public void *ic_malloc(size_t sz) { - ic_env_t *env = ic_get_env(); - if (env == NULL) { - return NULL; - } - return mem_malloc(env->mem, to_ssize_t(sz)); -} - -ic_public const char *ic_strdup(const char *s) { - if (s == NULL) { - return NULL; - } - ic_env_t *env = ic_get_env(); - if (env == NULL) { - return NULL; - } - ssize_t len = ic_strlen(s); - char *p = mem_malloc_tp_n(env->mem, char, len + 1); - if (p == NULL) { - return NULL; - } - ic_memcpy(p, s, len); - p[len] = 0; - return p; -} - -//------------------------------------------------------------- -// Terminal -//------------------------------------------------------------- - -ic_public void ic_term_init(void) { - ic_env_t *env = ic_get_env(); - if (env == NULL) { - return; - } - if (env->term == NULL) { - return; - } - term_start_raw(env->term); -} - -ic_public void ic_term_done(void) { - ic_env_t *env = ic_get_env(); - if (env == NULL) { - return; - } - if (env->term == NULL) { - return; - } - term_end_raw(env->term, false); -} - -ic_public void ic_term_flush(void) { - ic_env_t *env = ic_get_env(); - if (env == NULL) { - return; - } - if (env->term == NULL) { - return; - } - term_flush(env->term); -} - -ic_public void ic_term_write(const char *s) { - ic_env_t *env = ic_get_env(); - if (env == NULL) { - return; - } - if (env->term == NULL) { - return; - } - term_write(env->term, s); -} - -ic_public void ic_term_writeln(const char *s) { - ic_env_t *env = ic_get_env(); - if (env == NULL) { - return; - } - if (env->term == NULL) { - return; - } - term_writeln(env->term, s); -} - -ic_public void ic_term_writef(const char *fmt, ...) { - va_list ap; - va_start(ap, fmt); - ic_term_vwritef(fmt, ap); - va_end(ap); -} - -ic_public void ic_term_vwritef(const char *fmt, va_list args) { - ic_env_t *env = ic_get_env(); - if (env == NULL) { - return; - } - if (env->term == NULL) { - return; - } - term_vwritef(env->term, fmt, args); -} - -ic_public void ic_term_reset(void) { - ic_env_t *env = ic_get_env(); - if (env == NULL) { - return; - } - if (env->term == NULL) { - return; - } - term_attr_reset(env->term); -} - -ic_public void ic_term_style(const char *style) { - ic_env_t *env = ic_get_env(); - if (env == NULL) { - return; - } - if (env->term == NULL || env->bbcode == NULL) { - return; - } - term_set_attr(env->term, bbcode_style(env->bbcode, style)); -} - -ic_public int ic_term_get_color_bits(void) { - ic_env_t *env = ic_get_env(); - if (env == NULL || env->term == NULL) { - return 4; - } - return term_get_color_bits(env->term); -} - -ic_public void ic_term_bold(bool enable) { - ic_env_t *env = ic_get_env(); - if (env == NULL || env->term == NULL) { - return; - } - term_bold(env->term, enable); -} - -ic_public void ic_term_underline(bool enable) { - ic_env_t *env = ic_get_env(); - if (env == NULL || env->term == NULL) { - return; - } - term_underline(env->term, enable); -} - -ic_public void ic_term_italic(bool enable) { - ic_env_t *env = ic_get_env(); - if (env == NULL || env->term == NULL) { - return; - } - term_italic(env->term, enable); -} - -ic_public void ic_term_reverse(bool enable) { - ic_env_t *env = ic_get_env(); - if (env == NULL || env->term == NULL) { - return; - } - term_reverse(env->term, enable); -} - -ic_public void ic_term_color_ansi(bool foreground, int ansi_color) { - ic_env_t *env = ic_get_env(); - if (env == NULL || env->term == NULL) { - return; - } - ic_color_t color = color_from_ansi256(ansi_color); - if (foreground) { - term_color(env->term, color); - } else { - term_bgcolor(env->term, color); - } -} - -ic_public void ic_term_color_rgb(bool foreground, uint32_t hcolor) { - ic_env_t *env = ic_get_env(); - if (env == NULL || env->term == NULL) { - return; - } - ic_color_t color = ic_rgb(hcolor); - if (foreground) { - term_color(env->term, color); - } else { - term_bgcolor(env->term, color); - } -} - -//------------------------------------------------------------- -// Readline with temporary completer and highlighter -//------------------------------------------------------------- - -ic_public char *ic_readline_ex(const char *prompt_text, ic_completer_fun_t *completer, void *completer_arg, ic_highlight_fun_t *highlighter, void *highlighter_arg) { - ic_env_t *env = ic_get_env(); - if (env == NULL) { - return NULL; - } - // save previous - ic_completer_fun_t *prev_completer; - void *prev_completer_arg; - completions_get_completer(env->completions, &prev_completer, &prev_completer_arg); - ic_highlight_fun_t *prev_highlighter = env->highlighter; - void *prev_highlighter_arg = env->highlighter_arg; - // call with current - if (completer != NULL) { - ic_set_default_completer(completer, completer_arg); - } - if (highlighter != NULL) { - ic_set_default_highlighter(highlighter, highlighter_arg); - } - char *res = ic_readline(prompt_text); - // restore previous - ic_set_default_completer(prev_completer, prev_completer_arg); - ic_set_default_highlighter(prev_highlighter, prev_highlighter_arg); - return res; -} - -//------------------------------------------------------------- -// Initialize -//------------------------------------------------------------- - -static void ic_atexit(void); - -static void ic_env_free(ic_env_t *env) { - if (env == NULL) { - return; - } - history_save(env->history); - history_free(env->history); - completions_free(env->completions); - bbcode_free(env->bbcode); - term_free(env->term); - tty_free(env->tty); - mem_free(env->mem, env->cprompt_marker); - mem_free(env->mem, env->prompt_marker); - mem_free(env->mem, env->match_braces); - mem_free(env->mem, env->auto_braces); - env->prompt_marker = NULL; - - // and deallocate ourselves - alloc_t *mem = env->mem; - mem_free(mem, env); - - // and finally the custom memory allocation structure - mem_free(mem, mem); -} - -static ic_env_t *ic_env_create(ic_malloc_fun_t *_malloc, ic_realloc_fun_t *_realloc, ic_free_fun_t *_free) { - if (_malloc == NULL) { - _malloc = &malloc; - } - if (_realloc == NULL) { - _realloc = &realloc; - } - if (_free == NULL) { - _free = &free; - } - // allocate - alloc_t *mem = (alloc_t *)_malloc(sizeof(alloc_t)); - if (mem == NULL) { - return NULL; - } - mem->malloc = _malloc; - mem->realloc = _realloc; - mem->free = _free; - ic_env_t *env = mem_zalloc_tp(mem, ic_env_t); - if (env == NULL) { - mem->free(mem); - return NULL; - } - env->mem = mem; - - // Initialize - env->tty = tty_new(env->mem, -1); // can return NULL - env->term = term_new(env->mem, env->tty, false, false, -1); - env->history = history_new(env->mem); - env->completions = completions_new(env->mem); - env->bbcode = bbcode_new(env->mem, env->term); - env->hint_delay = 400; - - if (env->tty == NULL || env->term == NULL || - env->completions == NULL || env->history == NULL || env->bbcode == NULL || - !term_is_interactive(env->term)) { - env->noedit = true; - } - env->multiline_eol = '\\'; - - bbcode_style_def(env->bbcode, "ic-prompt", "ansi-green"); - bbcode_style_def(env->bbcode, "ic-info", "ansi-darkgray"); - bbcode_style_def(env->bbcode, "ic-diminish", "ansi-lightgray"); - bbcode_style_def(env->bbcode, "ic-emphasis", "#ffffd7"); - bbcode_style_def(env->bbcode, "ic-hint", "ansi-darkgray"); - bbcode_style_def(env->bbcode, "ic-error", "#d70000"); - bbcode_style_def(env->bbcode, "ic-bracematch", "ansi-white"); // color = #F7DC6F" ); - - bbcode_style_def(env->bbcode, "keyword", "#569cd6"); - bbcode_style_def(env->bbcode, "control", "#c586c0"); - bbcode_style_def(env->bbcode, "number", "#b5cea8"); - bbcode_style_def(env->bbcode, "string", "#ce9178"); - bbcode_style_def(env->bbcode, "comment", "#6A9955"); - bbcode_style_def(env->bbcode, "type", "darkcyan"); - bbcode_style_def(env->bbcode, "constant", "#569cd6"); - - set_prompt_marker(env, NULL, NULL); - return env; -} - -static ic_env_t *rpenv; - -static void ic_atexit(void) { - if (rpenv != NULL) { - ic_env_free(rpenv); - rpenv = NULL; - } -} - -ic_private ic_env_t *ic_get_env(void) { - if (rpenv == NULL) { - rpenv = ic_env_create(NULL, NULL, NULL); - if (rpenv != NULL) { - atexit(&ic_atexit); - } - } - return rpenv; -} - -ic_public void ic_init_custom_malloc(ic_malloc_fun_t *_malloc, ic_realloc_fun_t *_realloc, ic_free_fun_t *_free) { - assert(rpenv == NULL); - if (rpenv != NULL) { - ic_env_free(rpenv); - rpenv = ic_env_create(_malloc, _realloc, _free); - } else { - rpenv = ic_env_create(_malloc, _realloc, _free); - if (rpenv != NULL) { - atexit(&ic_atexit); - } - } -} diff --git a/vendor/isocline/isocline.h b/vendor/isocline/isocline.h deleted file mode 100644 index 5e8dc275..00000000 --- a/vendor/isocline/isocline.h +++ /dev/null @@ -1,605 +0,0 @@ -/* ---------------------------------------------------------------------------- - Copyright (c) 2021, Daan Leijen - This is free software; you can redistribute it and/or modify it - under the terms of the MIT License. A copy of the license can be - found in the "LICENSE" file at the root of this distribution. ------------------------------------------------------------------------------*/ -#pragma once -#ifndef IC_ISOCLINE_H -#define IC_ISOCLINE_H - -#ifdef __cplusplus -extern "C" { -#endif - -#include // term_vprintf -#include // bool -#include // size_t -#include // uint32_t - -/*! \mainpage -Isocline C API reference. - -Isocline is a pure C library that can be used as an alternative to the GNU readline library. - -See the [Github repository](https://github.com/daanx/isocline#readme) -for general information and building the library. - -Contents: -- \ref readline -- \ref bbcode -- \ref history -- \ref completion -- \ref highlight -- \ref options -- \ref helper -- \ref completex -- \ref term -- \ref async -- \ref alloc -*/ - -/// \defgroup readline Readline -/// The basic readline interface. -/// \{ - -/// Isocline version: 102 = 1.0.2. -#define IC_VERSION (104) - -/// Read input from the user using rich editing abilities. -/// @param prompt_text The prompt text, can be NULL for the default (""). -/// The displayed prompt becomes `prompt_text` followed by the `prompt_marker` ("> "). -/// @returns the heap allocated input on succes, which should be `free`d by the caller. -/// Returns NULL on error, or if the user typed ctrl+d or ctrl+c. -/// -/// If the standard input (`stdin`) has no editing capability -/// (like a dumb terminal (e.g. `TERM`=`dumb`), running in a debuggen, a pipe or redirected file, etc.) -/// the input is read directly from the input stream up to the -/// next line without editing capability. -/// See also \a ic_set_prompt_marker(), \a ic_style_def() -/// -/// @see ic_set_prompt_marker(), ic_style_def() -char *ic_readline(const char *prompt_text); - -/// \} - -//-------------------------------------------------------------- -/// \defgroup bbcode Formatted Text -/// Formatted text using [bbcode markup](https://github.com/daanx/isocline#bbcode-format). -/// \{ - -/// Print to the terminal while respection bbcode markup. -/// Any unclosed tags are closed automatically at the end of the print. -/// For example: -/// ``` -/// ic_print("[b]bold, [i]bold and italic[/i], [red]red and bold[/][/b] default."); -/// ic_print("[b]bold[/], [i b]bold and italic[/], [yellow on blue]yellow on blue background"); -/// ic_style_add("em","i color=#888800"); -/// ic_print("[em]emphasis"); -/// ``` -/// Properties that can be assigned are: -/// * `color=` _clr_, `bgcolor=` _clr_: where _clr_ is either a hex value `#`RRGGBB or `#`RGB, a -/// standard HTML color name, or an ANSI palette name, like `ansi-maroon`, `ansi-default`, etc. -/// * `bold`,`italic`,`reverse`,`underline`: can be `on` or `off`. -/// * everything else is a style; all HTML and ANSI color names are also a style (so we can just use `red` -/// instead of `color=red`, or `on red` instead of `bgcolor=red`), and there are -/// the `b`, `i`, `u`, and `r` styles for bold, italic, underline, and reverse. -/// -/// See [here](https://github.com/daanx/isocline#bbcode-format) for a description of the full bbcode format. -void ic_print(const char *s); - -/// Print with bbcode markup ending with a newline. -/// @see ic_print() -void ic_println(const char *s); - -/// Print formatted with bbcode markup. -/// @see ic_print() -void ic_printf(const char *fmt, ...); - -/// Print formatted with bbcode markup. -/// @see ic_print -void ic_vprintf(const char *fmt, va_list args); - -/// Define or redefine a style. -/// @param style_name The name of the style. -/// @param fmt The `fmt` string is the content of a tag and can contain -/// other styles. This is very useful to theme the output of a program -/// by assigning standard styles like `em` or `warning` etc. -void ic_style_def(const char *style_name, const char *fmt); - -/// Start a global style that is only reset when calling a matching ic_style_close(). -void ic_style_open(const char *fmt); - -/// End a global style. -void ic_style_close(void); - -/// \} - -//-------------------------------------------------------------- -// History -//-------------------------------------------------------------- -/// \defgroup history History -/// Readline input history. -/// \{ - -/// Enable history. -/// Use a \a NULL filename to not persist the history. Use -1 for max_entries to get the default (200). -void ic_set_history(const char *fname, long max_entries); - -/// Remove the last entry in the history. -/// The last returned input from ic_readline() is automatically added to the history; this function removes it. -void ic_history_remove_last(void); - -/// Clear the history. -void ic_history_clear(void); - -/// Add an entry to the history -void ic_history_add(const char *entry); - -/// \} - -//-------------------------------------------------------------- -// Basic Completion -//-------------------------------------------------------------- - -/// \defgroup completion Completion -/// Basic word completion. -/// \{ - -/// A completion environment -struct ic_completion_env_s; - -/// A completion environment -typedef struct ic_completion_env_s ic_completion_env_t; - -/// A completion callback that is called by isocline when tab is pressed. -/// It is passed a completion environment (containing the current input and the current cursor position), -/// the current input up-to the cursor (`prefix`) -/// and the user given argument when the callback was set. -/// When using completion transformers, like `ic_complete_quoted_word` the `prefix` contains the -/// the word to be completed without escape characters or quotes. -typedef void(ic_completer_fun_t)(ic_completion_env_t *cenv, const char *prefix); - -/// Set the default completion handler. -/// @param completer The completion function -/// @param arg Argument passed to the \a completer. -/// There can only be one default completion function, setting it again disables the previous one. -/// The initial completer use `ic_complete_filename`. -void ic_set_default_completer(ic_completer_fun_t *completer, void *arg); - -/// In a completion callback (usually from ic_complete_word()), use this function to add a completion. -/// (the completion string is copied by isocline and do not need to be preserved or allocated). -/// -/// Returns `true` if the callback should continue trying to find more possible completions. -/// If `false` is returned, the callback should try to return and not add more completions (for improved latency). -bool ic_add_completion(ic_completion_env_t *cenv, const char *completion); - -/// In a completion callback (usually from ic_complete_word()), use this function to add a completion. -/// The `display` is used to display the completion in the completion menu, and `help` is -/// displayed for hints for example. Both can be `NULL` for the default. -/// (all are copied by isocline and do not need to be preserved or allocated). -/// -/// Returns `true` if the callback should continue trying to find more possible completions. -/// If `false` is returned, the callback should try to return and not add more completions (for improved latency). -bool ic_add_completion_ex(ic_completion_env_t *cenv, const char *completion, const char *display, const char *help); - -/// In a completion callback (usually from ic_complete_word()), use this function to add completions. -/// The `completions` array should be terminated with a NULL element, and all elements -/// are added as completions if they start with `prefix`. -/// -/// Returns `true` if the callback should continue trying to find more possible completions. -/// If `false` is returned, the callback should try to return and not add more completions (for improved latency). -bool ic_add_completions(ic_completion_env_t *cenv, const char *prefix, const char **completions); - -/// Complete a filename. -/// Complete a filename given a semi-colon separated list of root directories `roots` and -/// semi-colon separated list of possible extensions (excluding directories). -/// If `roots` is NULL, the current directory is the root ("."). -/// If `extensions` is NULL, any extension will match. -/// Each root directory should _not_ end with a directory separator. -/// If a directory is completed, the `dir_separator` is added at the end if it is not `0`. -/// Usually the `dir_separator` is `/` but it can be set to `\\` on Windows systems. -/// For example: -/// ``` -/// /ho --> /home/ -/// /home/.ba --> /home/.bashrc -/// ``` -/// (This already uses ic_complete_quoted_word() so do not call it from inside a word handler). -void ic_complete_filename(ic_completion_env_t *cenv, const char *prefix, char dir_separator, const char *roots, const char *extensions); - -/// Function that returns whether a (utf8) character (of length `len`) is in a certain character class -/// @see ic_char_is_separator() etc. -typedef bool(ic_is_char_class_fun_t)(const char *s, long len); - -/// Complete a _word_ (i.e. _token_). -/// Calls the user provided function `fun` to complete on the -/// current _word_. Almost all user provided completers should use this function. -/// If `is_word_char` is NULL, the default `&ic_char_is_nonseparator` is used. -/// The `prefix` passed to `fun` is modified to only contain the current word, and -/// any results from `ic_add_completion` are automatically adjusted to replace that part. -/// For example, on the input "hello w", a the user `fun` only gets `w` and can just complete -/// with "world" resulting in "hello world" without needing to consider `delete_before` etc. -/// @see ic_complete_qword() for completing quoted and escaped tokens. -void ic_complete_word(ic_completion_env_t *cenv, const char *prefix, ic_completer_fun_t *fun, ic_is_char_class_fun_t *is_word_char); - -/// Complete a quoted _word_. -/// Calls the user provided function `fun` to complete while taking -/// care of quotes and escape characters. Almost all user provided completers should use -/// this function. The `prefix` passed to `fun` is modified to be unquoted and unescaped, and -/// any results from `ic_add_completion` are automatically quoted and escaped again. -/// For example, completing `hello world`, the `fun` always just completes `hel` or `hello w` to `hello world`, -/// but depending on user input, it will complete as: -/// ``` -/// hel --> hello\ world -/// hello\ w --> hello\ world -/// hello w --> # no completion, the word is just 'w'> -/// "hel --> "hello world" -/// "hello w --> "hello world" -/// ``` -/// with proper quotes and escapes. -/// If `is_word_char` is NULL, the default `&ic_char_is_nonseparator` is used. -/// @see ic_complete_quoted_word() to customize the word boundary, quotes etc. -void ic_complete_qword(ic_completion_env_t *cenv, const char *prefix, ic_completer_fun_t *fun, ic_is_char_class_fun_t *is_word_char); - -/// Complete a _word_. -/// Calls the user provided function `fun` to complete while taking -/// care of quotes and escape characters. Almost all user provided completers should use this function. -/// The `is_word_char` is a set of characters that are part of a "word". Use NULL for the default (`&ic_char_is_nonseparator`). -/// The `escape_char` is the escaping character, usually `\` but use 0 to not have escape characters. -/// The `quote_chars` define the quotes, use NULL for the default `"\'\""` quotes. -/// @see ic_complete_word() which uses the default values for `non_word_chars`, `quote_chars` and `\` for escape characters. -void ic_complete_qword_ex(ic_completion_env_t *cenv, const char *prefix, ic_completer_fun_t fun, ic_is_char_class_fun_t *is_word_char, char escape_char, const char *quote_chars); - -/// \} - -//-------------------------------------------------------------- -/// \defgroup highlight Syntax Highlighting -/// Basic syntax highlighting. -/// \{ - -/// A syntax highlight environment -struct ic_highlight_env_s; -typedef struct ic_highlight_env_s ic_highlight_env_t; - -/// A syntax highlighter callback that is called by readline to syntax highlight user input. -typedef void(ic_highlight_fun_t)(ic_highlight_env_t *henv, const char *input, void *arg); - -/// Set a syntax highlighter. -/// There can only be one highlight function, setting it again disables the previous one. -void ic_set_default_highlighter(ic_highlight_fun_t *highlighter, void *arg); - -/// Set the style of characters starting at position `pos`. -void ic_highlight(ic_highlight_env_t *henv, long pos, long count, const char *style); - -/// Experimental: Convenience callback for a function that highlights `s` using bbcode's. -/// The returned string should be allocated and is free'd by the caller. -typedef char *(ic_highlight_format_fun_t)(const char *s, void *arg); - -/// Experimental: Convenience function for highlighting with bbcodes. -/// Can be called in a `ic_highlight_fun_t` callback to colorize the `input` using the -/// the provided `formatted` input that is the styled `input` with bbcodes. The -/// content of `formatted` without bbcode tags should match `input` exactly. -void ic_highlight_formatted(ic_highlight_env_t *henv, const char *input, const char *formatted); - -/// \} - -//-------------------------------------------------------------- -// Readline with a specific completer and highlighter -//-------------------------------------------------------------- - -/// \defgroup readline -/// \{ - -/// Read input from the user using rich editing abilities, -/// using a particular completion function and highlighter for this call only. -/// both can be NULL in which case the defaults are used. -/// @see ic_readline(), ic_set_prompt_marker(), ic_set_default_completer(), ic_set_default_highlighter(). -char *ic_readline_ex(const char *prompt_text, ic_completer_fun_t *completer, void *completer_arg, ic_highlight_fun_t *highlighter, void *highlighter_arg); - -/// \} - -//-------------------------------------------------------------- -// Options -//-------------------------------------------------------------- - -/// \defgroup options Options -/// \{ - -/// Set a prompt marker and a potential marker for extra lines with multiline input. -/// Pass \a NULL for the `prompt_marker` for the default marker (`"> "`). -/// Pass \a NULL for continuation prompt marker to make it equal to the `prompt_marker`. -void ic_set_prompt_marker(const char *prompt_marker, const char *continuation_prompt_marker); - -/// Get the current prompt marker. -const char *ic_get_prompt_marker(void); - -/// Get the current continuation prompt marker. -const char *ic_get_continuation_prompt_marker(void); - -/// Disable or enable multi-line input (enabled by default). -/// Returns the previous setting. -bool ic_enable_multiline(bool enable); - -/// Disable or enable sound (enabled by default). -/// A beep is used when tab cannot find any completion for example. -/// Returns the previous setting. -bool ic_enable_beep(bool enable); - -/// Disable or enable color output (enabled by default). -/// Returns the previous setting. -bool ic_enable_color(bool enable); - -/// Disable or enable duplicate entries in the history (disabled by default). -/// Returns the previous setting. -bool ic_enable_history_duplicates(bool enable); - -/// Disable or enable automatic tab completion after a completion -/// to expand as far as possible if the completions are unique. (disabled by default). -/// Returns the previous setting. -bool ic_enable_auto_tab(bool enable); - -/// Disable or enable preview of a completion selection (enabled by default) -/// Returns the previous setting. -bool ic_enable_completion_preview(bool enable); - -/// Disable or enable automatic identation of continuation lines in multiline -/// input so it aligns with the initial prompt. -/// Returns the previous setting. -bool ic_enable_multiline_indent(bool enable); - -/// Disable or enable display of short help messages for history search etc. -/// (full help is always dispayed when pressing F1 regardless of this setting) -/// @returns the previous setting. -bool ic_enable_inline_help(bool enable); - -/// Disable or enable hinting (enabled by default) -/// Shows a hint inline when there is a single possible completion. -/// @returns the previous setting. -bool ic_enable_hint(bool enable); - -/// Set millisecond delay before a hint is displayed. Can be zero. (500ms by default). -long ic_set_hint_delay(long delay_ms); - -/// Disable or enable syntax highlighting (enabled by default). -/// This applies regardless whether a syntax highlighter callback was set (`ic_set_highlighter`) -/// Returns the previous setting. -bool ic_enable_highlight(bool enable); - -/// Set millisecond delay for reading escape sequences in order to distinguish -/// a lone ESC from the start of a escape sequence. The defaults are 100ms and 10ms, -/// but it may be increased if working with very slow terminals. -void ic_set_tty_esc_delay(long initial_delay_ms, long followup_delay_ms); - -/// Enable highlighting of matching braces (and error highlight unmatched braces).` -bool ic_enable_brace_matching(bool enable); - -/// Set matching brace pairs. -/// Pass \a NULL for the default `"()[]{}"`. -void ic_set_matching_braces(const char *brace_pairs); - -/// Enable automatic brace insertion (enabled by default). -bool ic_enable_brace_insertion(bool enable); - -/// Set matching brace pairs for automatic insertion. -/// Pass \a NULL for the default `()[]{}\"\"''` -void ic_set_insertion_braces(const char *brace_pairs); - -/// \} - -//-------------------------------------------------------------- -// Advanced Completion -//-------------------------------------------------------------- - -/// \defgroup completex Advanced Completion -/// \{ - -/// Get the raw current input (and cursor position if `cursor` != NULL) for the completion. -/// Usually completer functions should look at their `prefix` though as transformers -/// like `ic_complete_word` may modify the prefix (for example, unescape it). -const char *ic_completion_input(ic_completion_env_t *cenv, long *cursor); - -/// Get the completion argument passed to `ic_set_completer`. -void *ic_completion_arg(const ic_completion_env_t *cenv); - -/// Do we have already some completions? -bool ic_has_completions(const ic_completion_env_t *cenv); - -/// Do we already have enough completions and should we return if possible? (for improved latency) -bool ic_stop_completing(const ic_completion_env_t *cenv); - -/// Primitive completion, cannot be used with most transformers (like `ic_complete_word` and `ic_complete_qword`). -/// When completed, `delete_before` _bytes_ are deleted before the cursor position, -/// `delete_after` _bytes_ are deleted after the cursor, and finally `completion` is inserted. -/// The `display` is used to display the completion in the completion menu, and `help` is displayed -/// with hinting. Both `display` and `help` can be NULL. -/// (all are copied by isocline and do not need to be preserved or allocated). -/// -/// Returns `true` if the callback should continue trying to find more possible completions. -/// If `false` is returned, the callback should try to return and not add more completions (for improved latency). -bool ic_add_completion_prim(ic_completion_env_t *cenv, const char *completion, const char *display, const char *help, long delete_before, long delete_after); - -/// \} - -//-------------------------------------------------------------- -/// \defgroup helper Character Classes. -/// Convenience functions for character classes, highlighting and completion. -/// \{ - -/// Convenience: return the position of a previous code point in a UTF-8 string `s` from postion `pos`. -/// Returns `-1` if `pos <= 0` or `pos > strlen(s)` (or other errors). -long ic_prev_char(const char *s, long pos); - -/// Convenience: return the position of the next code point in a UTF-8 string `s` from postion `pos`. -/// Returns `-1` if `pos < 0` or `pos >= strlen(s)` (or other errors). -long ic_next_char(const char *s, long pos); - -/// Convenience: does a string `s` starts with a given `prefix` ? -bool ic_starts_with(const char *s, const char *prefix); - -/// Convenience: does a string `s` starts with a given `prefix` ignoring (ascii) case? -bool ic_istarts_with(const char *s, const char *prefix); - -/// Convenience: character class for whitespace `[ \t\r\n]`. -bool ic_char_is_white(const char *s, long len); - -/// Convenience: character class for non-whitespace `[^ \t\r\n]`. -bool ic_char_is_nonwhite(const char *s, long len); - -/// Convenience: character class for separators. -/// (``[ \t\r\n,.;:/\\(){}\[\]]``.) -/// This is used for word boundaries in isocline. -bool ic_char_is_separator(const char *s, long len); - -/// Convenience: character class for non-separators. -bool ic_char_is_nonseparator(const char *s, long len); - -/// Convenience: character class for letters (`[A-Za-z]` and any unicode > 0x80). -bool ic_char_is_letter(const char *s, long len); - -/// Convenience: character class for digits (`[0-9]`). -bool ic_char_is_digit(const char *s, long len); - -/// Convenience: character class for hexadecimal digits (`[A-Fa-f0-9]`). -bool ic_char_is_hexdigit(const char *s, long len); - -/// Convenience: character class for identifier letters (`[A-Za-z0-9_-]` and any unicode > 0x80). -bool ic_char_is_idletter(const char *s, long len); - -/// Convenience: character class for filename letters (_not in_ " \t\r\n`@$><=;|&\{\}\(\)\[\]]"). -bool ic_char_is_filename_letter(const char *s, long len); - -/// Convenience: If this is a token start, return the length. Otherwise return 0. -long ic_is_token(const char *s, long pos, ic_is_char_class_fun_t *is_token_char); - -/// Convenience: Does this match the specified token? -/// Ensures not to match prefixes or suffixes, and returns the length of the match (in bytes). -/// E.g. `ic_match_token("function",0,&ic_char_is_letter,"fun")` returns 0. -/// while `ic_match_token("fun x",0,&ic_char_is_letter,"fun"})` returns 3. -long ic_match_token(const char *s, long pos, ic_is_char_class_fun_t *is_token_char, const char *token); - -/// Convenience: Do any of the specified tokens match? -/// Ensures not to match prefixes or suffixes, and returns the length of the match (in bytes). -/// E.g. `ic_match_any_token("function",0,&ic_char_is_letter,{"fun","func",NULL})` returns 0. -/// while `ic_match_any_token("func x",0,&ic_char_is_letter,{"fun","func",NULL})` returns 4. -long ic_match_any_token(const char *s, long pos, ic_is_char_class_fun_t *is_token_char, const char **tokens); - -/// \} - -//-------------------------------------------------------------- -/// \defgroup term Terminal -/// -/// Experimental: Low level terminal output. -/// Ensures basic ANSI SGR escape sequences are processed -/// in a portable way (e.g. on Windows) -/// \{ - -/// Initialize for terminal output. -/// Call this before using the terminal write functions (`ic_term_write`) -/// Does nothing on most platforms but on Windows it sets the console to UTF8 output and possible -/// enables virtual terminal processing. -void ic_term_init(void); - -/// Call this when done with the terminal functions. -void ic_term_done(void); - -/// Flush the terminal output. -/// (happens automatically on newline characters ('\n') as well). -void ic_term_flush(void); - -/// Write a string to the console (and process CSI escape sequences). -void ic_term_write(const char *s); - -/// Write a string to the console and end with a newline -/// (and process CSI escape sequences). -void ic_term_writeln(const char *s); - -/// Write a formatted string to the console. -/// (and process CSI escape sequences) -void ic_term_writef(const char *fmt, ...); - -/// Write a formatted string to the console. -void ic_term_vwritef(const char *fmt, va_list args); - -/// Set text attributes from a style. -void ic_term_style(const char *style); - -/// Set text attribute to bold. -void ic_term_bold(bool enable); - -/// Set text attribute to underline. -void ic_term_underline(bool enable); - -/// Set text attribute to italic. -void ic_term_italic(bool enable); - -/// Set text attribute to reverse video. -void ic_term_reverse(bool enable); - -/// Set text attribute to ansi color palette index between 0 and 255 (or 256 for the ANSI "default" color). -/// (auto matched to smaller palette if not supported) -void ic_term_color_ansi(bool foreground, int color); - -/// Set text attribute to 24-bit RGB color (between `0x000000` and `0xFFFFFF`). -/// (auto matched to smaller palette if not supported) -void ic_term_color_rgb(bool foreground, uint32_t color); - -/// Reset the text attributes. -void ic_term_reset(void); - -/// Get the palette used by the terminal: -/// This is usually initialized from the COLORTERM environment variable. The -/// possible values of COLORTERM for each palette are given in parenthesis. -/// -/// - 1: monochrome (`monochrome`) -/// - 3: old ANSI terminal with 8 colors, using bold for bright (`8color`/`3bit`) -/// - 4: regular ANSI terminal with 16 colors. (`16color`/`4bit`) -/// - 8: terminal with ANSI 256 color palette. (`256color`/`8bit`) -/// - 24: true-color terminal with full RGB colors. (`truecolor`/`24bit`/`direct`) -int ic_term_get_color_bits(void); - -/// \} - -//-------------------------------------------------------------- -/// \defgroup async ASync -/// Async support -/// \{ - -/// Thread-safe way to asynchronously unblock a readline. -/// Behaves as if the user pressed the `ctrl-C` character -/// (resulting in returning NULL from `ic_readline`). -/// Returns `true` if the event was successfully delivered. -/// (This may not be supported on all platforms, but it is -/// functional on Linux, macOS and Windows). -bool ic_async_stop(void); - -/// \} - -//-------------------------------------------------------------- -/// \defgroup alloc Custom Allocation -/// Register allocation functions for custom allocators -/// \{ - -typedef void *(ic_malloc_fun_t)(size_t size); -typedef void *(ic_realloc_fun_t)(void *p, size_t newsize); -typedef void(ic_free_fun_t)(void *p); - -/// Initialize with custom allocation functions. -/// This must be called as the first function in a program! -void ic_init_custom_alloc(ic_malloc_fun_t *_malloc, ic_realloc_fun_t *_realloc, ic_free_fun_t *_free); - -/// Free a potentially custom alloc'd pointer (in particular, the result returned from `ic_readline`) -void ic_free(void *p); - -/// Allocate using the current memory allocator. -void *ic_malloc(size_t sz); - -/// Duplicate a string using the current memory allocator. -const char *ic_strdup(const char *s); - -/// \} - -#ifdef __cplusplus -} -#endif - -#endif /// IC_ISOCLINE_H \ No newline at end of file diff --git a/vendor/isocline/stringbuf.c b/vendor/isocline/stringbuf.c deleted file mode 100644 index 7bbfad04..00000000 --- a/vendor/isocline/stringbuf.c +++ /dev/null @@ -1,1038 +0,0 @@ -/* ---------------------------------------------------------------------------- - Copyright (c) 2021, Daan Leijen - This is free software; you can redistribute it and/or modify it - under the terms of the MIT License. A copy of the license can be - found in the "LICENSE" file at the root of this distribution. ------------------------------------------------------------------------------*/ - -// get `wcwidth` for the column width of unicode characters -// note: for now the OS provided one is unused as we see quite a bit of variation -// among platforms and including our own seems more reliable. -/* -#if defined(__linux__) || defined(__freebsd__) -// use the system supplied one -#if !defined(_XOPEN_SOURCE) -#define _XOPEN_SOURCE 700 // so wcwidth is visible -#endif -#include -#else -*/ -// use our own (also on APPLE as that fails within vscode) -#define wcwidth(c) mk_wcwidth(c) -#include "wcwidth.c" -// #endif - -#include -#include -#include - -#include "common.h" -#include "stringbuf.h" - -//------------------------------------------------------------- -// In place growable utf-8 strings -//------------------------------------------------------------- - -struct stringbuf_s { - char* buf; - ssize_t buflen; - ssize_t count; - alloc_t* mem; -}; - - -//------------------------------------------------------------- -// String column width -//------------------------------------------------------------- - -// column width of a utf8 single character sequence. -static ssize_t utf8_char_width( const char* s, ssize_t n ) { - if (n <= 0) return 0; - - uint8_t b = (uint8_t)s[0]; - int32_t c; - if (b < ' ') { - return 0; - } - else if (b <= 0x7F) { - return 1; - } - else if (b <= 0xC1) { // invalid continuation byte or invalid 0xC0, 0xC1 (check is strictly not necessary as we don't validate..) - return 1; - } - else if (b <= 0xDF && n >= 2) { // b >= 0xC2 // 2 bytes - c = (((b & 0x1F) << 6) | (s[1] & 0x3F)); - assert(c < 0xD800 || c > 0xDFFF); - int w = wcwidth(c); - return w; - } - else if (b <= 0xEF && n >= 3) { // b >= 0xE0 // 3 bytes - c = (((b & 0x0F) << 12) | ((s[1] & 0x3F) << 6) | (s[2] & 0x3F)); - return wcwidth(c); - } - else if (b <= 0xF4 && n >= 4) { // b >= 0xF0 // 4 bytes - c = (((b & 0x07) << 18) | ((s[1] & 0x3F) << 12) | ((s[2] & 0x3F) << 6) | (s[3] & 0x3F)); - return wcwidth(c); - } - else { - // failed - return 1; - } -} - - -// The column width of a codepoint (0, 1, or 2) -static ssize_t char_column_width( const char* s, ssize_t n ) { - if (s == NULL || n <= 0) return 0; - else if ((uint8_t)(*s) < ' ') return 0; // also for CSI escape sequences - else { - ssize_t w = utf8_char_width(s, n); - #ifdef _WIN32 - return (w <= 0 ? 1 : w); // windows console seems to use at least one column - #else - return w; - #endif - } -} - -static ssize_t str_column_width_n( const char* s, ssize_t len ) { - if (s == NULL || len <= 0) return 0; - ssize_t pos = 0; - ssize_t cwidth = 0; - ssize_t cw; - ssize_t ofs; - while (s[pos] != 0 && (ofs = str_next_ofs(s, len, pos, &cw)) > 0) { - cwidth += cw; - pos += ofs; - } - return cwidth; -} - -ic_private ssize_t str_column_width( const char* s ) { - return str_column_width_n( s, ic_strlen(s) ); -} - -ic_private ssize_t str_skip_until_fit( const char* s, ssize_t max_width ) { - if (s == NULL) return 0; - ssize_t cwidth = str_column_width(s); - ssize_t len = ic_strlen(s); - ssize_t pos = 0; - ssize_t next; - ssize_t cw; - while (cwidth > max_width && (next = str_next_ofs(s, len, pos, &cw)) > 0) { - cwidth -= cw; - pos += next; - } - return pos; -} - -ic_private ssize_t str_take_while_fit( const char* s, ssize_t max_width) { - if (s == NULL) return 0; - const ssize_t len = ic_strlen(s); - ssize_t pos = 0; - ssize_t next; - ssize_t cw; - ssize_t cwidth = 0; - while ((next = str_next_ofs(s, len, pos, &cw)) > 0) { - if (cwidth + cw > max_width) break; - cwidth += cw; - pos += next; - } - return pos; -} - - -//------------------------------------------------------------- -// String navigation -//------------------------------------------------------------- - -// get offset of the previous codepoint. does not skip back over CSI sequences. -ic_private ssize_t str_prev_ofs( const char* s, ssize_t pos, ssize_t* width ) { - ssize_t ofs = 0; - if (s != NULL && pos > 0) { - ofs = 1; - while (pos > ofs) { - uint8_t u = (uint8_t)s[pos - ofs]; - if (u < 0x80 || u > 0xBF) break; // continue while follower - ofs++; - } - } - if (width != NULL) *width = char_column_width( s+(pos-ofs), ofs ); - return ofs; -} - -// skip an escape sequence -// -ic_private bool skip_esc( const char* s, ssize_t len, ssize_t* esclen ) { - if (s == NULL || len <= 1 || s[0] != '\x1B') return false; - if (esclen != NULL) *esclen = 0; - if (strchr("[PX^_]",s[1]) != NULL) { - // CSI (ESC [), DCS (ESC P), SOS (ESC X), PM (ESC ^), APC (ESC _), and OSC (ESC ]): terminated with a special sequence - bool finalCSI = (s[1] == '['); // CSI terminates with 0x40-0x7F; otherwise ST (bell or ESC \) - ssize_t n = 2; - while (len > n) { - char c = s[n++]; - if ((finalCSI && (uint8_t)c >= 0x40 && (uint8_t)c <= 0x7F) || // terminating byte: @A–Z[\]^_`a–z{|}~ - (!finalCSI && c == '\x07') || // bell - (c == '\x02')) // STX terminates as well - { - if (esclen != NULL) *esclen = n; - return true; - } - else if (!finalCSI && c == '\x1B' && len > n && s[n] == '\\') { // ST (ESC \) - n++; - if (esclen != NULL) *esclen = n; - return true; - } - } - } - if (strchr(" #%()*+",s[1]) != NULL) { - // assume escape sequence of length 3 (like ESC % G) - if (esclen != NULL) *esclen = 2; - return true; - } - else { - // assume single character escape code (like ESC 7) - if (esclen != NULL) *esclen = 2; - return true; - } - return false; -} - -// Offset to the next codepoint, treats CSI escape sequences as a single code point. -ic_private ssize_t str_next_ofs( const char* s, ssize_t len, ssize_t pos, ssize_t* cwidth ) { - ssize_t ofs = 0; - if (s != NULL && len > pos) { - if (skip_esc(s+pos,len-pos,&ofs)) { - // skip escape sequence - } - else { - ofs = 1; - // utf8 extended character? - while(len > pos + ofs) { - uint8_t u = (uint8_t)s[pos + ofs]; - if (u < 0x80 || u > 0xBF) break; // break if not a follower - ofs++; - } - } - } - if (cwidth != NULL) *cwidth = char_column_width( s+pos, ofs ); - return ofs; -} - -static ssize_t str_limit_to_length( const char* s, ssize_t n ) { - ssize_t i; - for(i = 0; i < n && s[i] != 0; i++) { /* nothing */ } - return i; -} - - -//------------------------------------------------------------- -// String searching prev/next word, line, ws_word -//------------------------------------------------------------- - - -static ssize_t str_find_backward( const char* s, ssize_t len, ssize_t pos, ic_is_char_class_fun_t* match, bool skip_immediate_matches ) { - if (pos > len) pos = len; - if (pos < 0) pos = 0; - ssize_t i = pos; - // skip matching first (say, whitespace in case of the previous start-of-word) - if (skip_immediate_matches) { - do { - ssize_t prev = str_prev_ofs(s, i, NULL); - if (prev <= 0) break; - assert(i - prev >= 0); - if (!match(s + i - prev, (long)prev)) break; - i -= prev; - } while (i > 0); - } - // find match - do { - ssize_t prev = str_prev_ofs(s, i, NULL); - if (prev <= 0) break; - assert(i - prev >= 0); - if (match(s + i - prev, (long)prev)) { - return i; // found; - } - i -= prev; - } while (i > 0); - return -1; // not found -} - -static ssize_t str_find_forward( const char* s, ssize_t len, ssize_t pos, ic_is_char_class_fun_t* match, bool skip_immediate_matches ) { - if (s == NULL || len < 0) return -1; - if (pos > len) pos = len; - if (pos < 0) pos = 0; - ssize_t i = pos; - ssize_t next; - // skip matching first (say, whitespace in case of the next end-of-word) - if (skip_immediate_matches) { - do { - next = str_next_ofs(s, len, i, NULL); - if (next <= 0) break; - assert( i + next <= len); - if (!match(s + i, (long)next)) break; - i += next; - } while (i < len); - } - // and then look - do { - next = str_next_ofs(s, len, i, NULL); - if (next <= 0) break; - assert( i + next <= len); - if (match(s + i, (long)next)) { - return i; // found - } - i += next; - } while (i < len); - return -1; -} - -static bool char_is_linefeed( const char* s, long n ) { - return (n == 1 && (*s == '\n' || *s == 0)); -} - -static ssize_t str_find_line_start( const char* s, ssize_t len, ssize_t pos) { - ssize_t start = str_find_backward(s,len,pos,&char_is_linefeed,false /* don't skip immediate matches */); - return (start < 0 ? 0 : start); -} - -static ssize_t str_find_line_end( const char* s, ssize_t len, ssize_t pos) { - ssize_t end = str_find_forward(s,len,pos, &char_is_linefeed, false); - return (end < 0 ? len : end); -} - -static ssize_t str_find_word_start( const char* s, ssize_t len, ssize_t pos) { - ssize_t start = str_find_backward(s,len,pos, &ic_char_is_idletter,true /* skip immediate matches */); - return (start < 0 ? 0 : start); -} - -static ssize_t str_find_word_end( const char* s, ssize_t len, ssize_t pos) { - ssize_t end = str_find_forward(s,len,pos,&ic_char_is_idletter,true /* skip immediate matches */); - return (end < 0 ? len : end); -} - -static ssize_t str_find_ws_word_start( const char* s, ssize_t len, ssize_t pos) { - ssize_t start = str_find_backward(s,len,pos,&ic_char_is_white,true /* skip immediate matches */); - return (start < 0 ? 0 : start); -} - -static ssize_t str_find_ws_word_end( const char* s, ssize_t len, ssize_t pos) { - ssize_t end = str_find_forward(s,len,pos,&ic_char_is_white,true /* skip immediate matches */); - return (end < 0 ? len : end); -} - - -//------------------------------------------------------------- -// String row/column iteration -//------------------------------------------------------------- - -// invoke a function for each terminal row; returns total row count. -static ssize_t str_for_each_row( const char* s, ssize_t len, ssize_t termw, ssize_t promptw, ssize_t cpromptw, - row_fun_t* fun, const void* arg, void* res ) -{ - if (s == NULL) s = ""; - ssize_t i; - ssize_t rcount = 0; - ssize_t rcol = 0; - ssize_t rstart = 0; - ssize_t startw = promptw; - for(i = 0; i < len; ) { - ssize_t w; - ssize_t next = str_next_ofs(s, len, i, &w); - if (next <= 0) { - debug_msg("str: foreach row: next<=0: len %zd, i %zd, w %zd, buf %s\n", len, i, w, s ); - assert(false); - break; - } - startw = (rcount == 0 ? promptw : cpromptw); - ssize_t termcol = rcol + w + startw + 1 /* for the cursor */; - if (termw != 0 && i != 0 && termcol >= termw) { - // wrap - if (fun != NULL) { - if (fun(s,rcount,rstart,i - rstart,startw,true,arg,res)) return rcount; - } - rcount++; - rstart = i; - rcol = 0; - } - if (s[i] == '\n') { - // newline - if (fun != NULL) { - if (fun(s,rcount,rstart,i - rstart,startw,false,arg,res)) return rcount; - } - rcount++; - rstart = i+1; - rcol = 0; - } - assert (s[i] != 0); - i += next; - rcol += w; - } - if (fun != NULL) { - if (fun(s,rcount,rstart,i - rstart,startw,false,arg,res)) return rcount; - } - return rcount+1; -} - -//------------------------------------------------------------- -// String: get row/column position -//------------------------------------------------------------- - - -static bool str_get_current_pos_iter( - const char* s, - ssize_t row, ssize_t row_start, ssize_t row_len, - ssize_t startw, bool is_wrap, const void* arg, void* res) -{ - ic_unused(is_wrap); ic_unused(startw); - rowcol_t* rc = (rowcol_t*)res; - ssize_t pos = *((ssize_t*)arg); - - if (pos >= row_start && pos <= (row_start + row_len)) { - // found the cursor row - rc->row_start = row_start; - rc->row_len = row_len; - rc->row = row; - rc->col = str_column_width_n( s + row_start, pos - row_start ); - rc->first_on_row = (pos == row_start); - if (is_wrap) { - // if wrapped, we check if the next character is at row_len - ssize_t next = str_next_ofs(s, row_start + row_len, pos, NULL); - rc->last_on_row = (pos + next >= row_start + row_len); - } - else { - // normal last position is right after the last character - rc->last_on_row = (pos >= row_start + row_len); - } - // debug_msg("edit; pos iter: pos: %zd (%c), row_start: %zd, rowlen: %zd\n", pos, s[pos], row_start, row_len); - } - return false; // always continue to count all rows -} - -static ssize_t str_get_rc_at_pos(const char* s, ssize_t len, ssize_t termw, ssize_t promptw, ssize_t cpromptw, ssize_t pos, rowcol_t* rc) { - memset(rc, 0, sizeof(*rc)); - ssize_t rows = str_for_each_row(s, len, termw, promptw, cpromptw, &str_get_current_pos_iter, &pos, rc); - // debug_msg("edit: current pos: (%d, %d) %s %s\n", rc->row, rc->col, rc->first_on_row ? "first" : "", rc->last_on_row ? "last" : ""); - return rows; -} - - - -//------------------------------------------------------------- -// String: get row/column position for a resized terminal -// with potentially "hard-wrapped" rows -//------------------------------------------------------------- -typedef struct wrapped_arg_s { - ssize_t pos; - ssize_t newtermw; -} wrapped_arg_t; - -typedef struct wrowcol_s { - rowcol_t rc; - ssize_t hrows; // count of hard-wrapped extra rows -} wrowcol_t; - -static bool str_get_current_wrapped_pos_iter( - const char* s, - ssize_t row, ssize_t row_start, ssize_t row_len, - ssize_t startw, bool is_wrap, const void* arg, void* res) -{ - ic_unused(is_wrap); - wrowcol_t* wrc = (wrowcol_t*)res; - const wrapped_arg_t* warg = (const wrapped_arg_t*)arg; - - // iterate through the row and record the postion and hard-wraps - ssize_t hwidth = startw; - ssize_t i = 0; - while( i <= row_len ) { // include rowlen as the cursor position can be just after the last character - // get next position and column width - ssize_t cw; - ssize_t next; - bool is_cursor = (warg->pos == row_start+i); - if (i < row_len) { - next = str_next_ofs(s + row_start, row_len, i, &cw); - } - else { - // end of row: take wrap or cursor into account - // (wrap has width 2 as it displays a back-arrow but also has an invisible newline that wraps) - cw = (is_wrap ? 2 : (is_cursor ? 1 : 0)); - next = 1; - } - - if (next > 0) { - if (hwidth + cw > warg->newtermw) { - // hardwrap - hwidth = 0; - wrc->hrows++; - debug_msg("str: found hardwrap: row: %zd, hrows: %zd\n", row, wrc->hrows); - } - } - else { - next++; // ensure we terminate (as we go up to rowlen) - } - - // did we find our position? - if (is_cursor) { - debug_msg("str: found position: row: %zd, hrows: %zd\n", row, wrc->hrows); - wrc->rc.row_start = row_start; - wrc->rc.row_len = row_len; - wrc->rc.row = wrc->hrows + row; - wrc->rc.col = hwidth; - wrc->rc.first_on_row = (i==0); - wrc->rc.last_on_row = (i+next >= row_len - (is_wrap ? 1 : 0)); - } - - // advance - hwidth += cw; - i += next; - } - return false; // always continue to count all rows -} - - -static ssize_t str_get_wrapped_rc_at_pos(const char* s, ssize_t len, ssize_t termw, ssize_t newtermw, ssize_t promptw, ssize_t cpromptw, ssize_t pos, rowcol_t* rc) { - wrapped_arg_t warg; - warg.pos = pos; - warg.newtermw = newtermw; - wrowcol_t wrc; - memset(&wrc,0,sizeof(wrc)); - ssize_t rows = str_for_each_row(s, len, termw, promptw, cpromptw, &str_get_current_wrapped_pos_iter, &warg, &wrc); - debug_msg("edit: wrapped pos: (%zd,%zd) rows %zd %s %s, hrows: %zd\n", wrc.rc.row, wrc.rc.col, rows, wrc.rc.first_on_row ? "first" : "", wrc.rc.last_on_row ? "last" : "", wrc.hrows); - *rc = wrc.rc; - return (rows + wrc.hrows); -} - - -//------------------------------------------------------------- -// Set position -//------------------------------------------------------------- - -static bool str_set_pos_iter( - const char* s, - ssize_t row, ssize_t row_start, ssize_t row_len, - ssize_t startw, bool is_wrap, const void* arg, void* res) -{ - ic_unused(arg); ic_unused(is_wrap); ic_unused(startw); - rowcol_t* rc = (rowcol_t*)arg; - if (rc->row != row) return false; // keep searching - // we found our row - ssize_t col = 0; - ssize_t i = row_start; - ssize_t end = row_start + row_len; - while (col < rc->col && i < end) { - ssize_t cw; - ssize_t next = str_next_ofs(s, row_start + row_len, i, &cw); - if (next <= 0) break; - i += next; - col += cw; - } - *((ssize_t*)res) = i; - return true; // stop iteration -} - -static ssize_t str_get_pos_at_rc(const char* s, ssize_t len, ssize_t termw, ssize_t promptw, ssize_t cpromptw, ssize_t row, ssize_t col /* without prompt */) { - rowcol_t rc; - memset(&rc,0,ssizeof(rc)); - rc.row = row; - rc.col = col; - ssize_t pos = -1; - str_for_each_row(s,len,termw,promptw,cpromptw,&str_set_pos_iter,&rc,&pos); - return pos; -} - - -//------------------------------------------------------------- -// String buffer -//------------------------------------------------------------- -static bool sbuf_ensure_extra(stringbuf_t* s, ssize_t extra) -{ - if (s->buflen >= s->count + extra) return true; - // reallocate; pick good initial size and multiples to increase reuse on allocation - ssize_t newlen = (s->buflen <= 0 ? 120 : (s->buflen > 1000 ? s->buflen + 1000 : 2*s->buflen)); - if (newlen < s->count + extra) newlen = s->count + extra; - if (s->buflen > 0) { - debug_msg("stringbuf: reallocate: old %zd, new %zd\n", s->buflen, newlen); - } - char* newbuf = mem_realloc_tp(s->mem, char, s->buf, newlen+1); // one more for terminating zero - if (newbuf == NULL) { - assert(false); - return false; - } - s->buf = newbuf; - s->buflen = newlen; - s->buf[s->count] = s->buf[s->buflen] = 0; - assert(s->buflen >= s->count + extra); - return true; -} - -static void sbuf_init( stringbuf_t* sbuf, alloc_t* mem ) { - sbuf->mem = mem; - sbuf->buf = NULL; - sbuf->buflen = 0; - sbuf->count = 0; -} - -static void sbuf_done( stringbuf_t* sbuf ) { - mem_free( sbuf->mem, sbuf->buf ); - sbuf->buf = NULL; - sbuf->buflen = 0; - sbuf->count = 0; -} - - -ic_private void sbuf_free( stringbuf_t* sbuf ) { - if (sbuf==NULL) return; - sbuf_done(sbuf); - mem_free(sbuf->mem, sbuf); -} - -ic_private stringbuf_t* sbuf_new( alloc_t* mem ) { - stringbuf_t* sbuf = mem_zalloc_tp(mem,stringbuf_t); - if (sbuf == NULL) return NULL; - sbuf_init(sbuf,mem); - return sbuf; -} - -// free the sbuf and return the current string buffer as the result -ic_private char* sbuf_free_dup(stringbuf_t* sbuf) { - if (sbuf == NULL) return NULL; - char* s = NULL; - if (sbuf->buf != NULL) { - s = mem_realloc_tp(sbuf->mem, char, sbuf->buf, sbuf_len(sbuf)+1); - if (s == NULL) { s = sbuf->buf; } - sbuf->buf = 0; - sbuf->buflen = 0; - sbuf->count = 0; - } - sbuf_free(sbuf); - return s; -} - -ic_private const char* sbuf_string_at( stringbuf_t* sbuf, ssize_t pos ) { - if (pos < 0 || sbuf->count < pos) return NULL; - if (sbuf->buf == NULL) return ""; - assert(sbuf->buf[sbuf->count] == 0); - return sbuf->buf + pos; -} - -ic_private const char* sbuf_string( stringbuf_t* sbuf ) { - return sbuf_string_at( sbuf, 0 ); -} - -ic_private char sbuf_char_at(stringbuf_t* sbuf, ssize_t pos) { - if (sbuf->buf == NULL || pos < 0 || sbuf->count < pos) return 0; - return sbuf->buf[pos]; -} - -ic_private char* sbuf_strdup_at( stringbuf_t* sbuf, ssize_t pos ) { - return mem_strdup(sbuf->mem, sbuf_string_at(sbuf,pos)); -} - -ic_private char* sbuf_strdup( stringbuf_t* sbuf ) { - return mem_strdup(sbuf->mem, sbuf_string(sbuf)); -} - -ic_private ssize_t sbuf_len(const stringbuf_t* s) { - if (s == NULL) return 0; - return s->count; -} - -ic_private ssize_t sbuf_append_vprintf(stringbuf_t* sb, const char* fmt, va_list args) { - const ssize_t min_needed = ic_strlen(fmt); - if (!sbuf_ensure_extra(sb,min_needed + 16)) return sb->count; - ssize_t avail = sb->buflen - sb->count; - va_list args0; - va_copy(args0, args); - ssize_t needed = vsnprintf(sb->buf + sb->count, to_size_t(avail), fmt, args0); - if (needed > avail) { - sb->buf[sb->count] = 0; - if (!sbuf_ensure_extra(sb, needed)) return sb->count; - avail = sb->buflen - sb->count; - needed = vsnprintf(sb->buf + sb->count, to_size_t(avail), fmt, args); - } - assert(needed <= avail); - sb->count += (needed > avail ? avail : (needed >= 0 ? needed : 0)); - assert(sb->count <= sb->buflen); - sb->buf[sb->count] = 0; - return sb->count; -} - -ic_private ssize_t sbuf_appendf(stringbuf_t* sb, const char* fmt, ...) { - va_list args; - va_start( args, fmt); - ssize_t res = sbuf_append_vprintf( sb, fmt, args ); - va_end(args); - return res; -} - - -ic_private ssize_t sbuf_insert_at_n(stringbuf_t* sbuf, const char* s, ssize_t n, ssize_t pos ) { - if (pos < 0 || pos > sbuf->count || s == NULL) return pos; - n = str_limit_to_length(s,n); - if (n <= 0 || !sbuf_ensure_extra(sbuf,n)) return pos; - ic_memmove(sbuf->buf + pos + n, sbuf->buf + pos, sbuf->count - pos); - ic_memcpy(sbuf->buf + pos, s, n); - sbuf->count += n; - sbuf->buf[sbuf->count] = 0; - return (pos + n); -} - -ic_private stringbuf_t* sbuf_split_at( stringbuf_t* sb, ssize_t pos ) { - stringbuf_t* res = sbuf_new(sb->mem); - if (res==NULL || pos < 0) return NULL; - if (pos < sb->count) { - sbuf_append_n(res, sb->buf + pos, sb->count - pos); - sb->count = pos; - } - return res; -} - -ic_private ssize_t sbuf_insert_at(stringbuf_t* sbuf, const char* s, ssize_t pos ) { - return sbuf_insert_at_n( sbuf, s, ic_strlen(s), pos ); -} - -ic_private ssize_t sbuf_insert_char_at(stringbuf_t* sbuf, char c, ssize_t pos ) { - char s[2]; - s[0] = c; - s[1] = 0; - return sbuf_insert_at_n( sbuf, s, 1, pos); -} - -ic_private ssize_t sbuf_insert_unicode_at(stringbuf_t* sbuf, unicode_t u, ssize_t pos) { - uint8_t s[5]; - unicode_to_qutf8(u, s); - return sbuf_insert_at(sbuf, (const char*)s, pos); -} - - - -ic_private void sbuf_delete_at( stringbuf_t* sbuf, ssize_t pos, ssize_t count ) { - if (pos < 0 || pos >= sbuf->count) return; - if (pos + count > sbuf->count) count = sbuf->count - pos; - ic_memmove(sbuf->buf + pos, sbuf->buf + pos + count, sbuf->count - pos - count); - sbuf->count -= count; - sbuf->buf[sbuf->count] = 0; -} - -ic_private void sbuf_delete_from_to( stringbuf_t* sbuf, ssize_t pos, ssize_t end ) { - if (end <= pos) return; - sbuf_delete_at( sbuf, pos, end - pos); -} - -ic_private void sbuf_delete_from(stringbuf_t* sbuf, ssize_t pos ) { - sbuf_delete_at(sbuf, pos, sbuf_len(sbuf) - pos ); -} - - -ic_private void sbuf_clear( stringbuf_t* sbuf ) { - sbuf_delete_at(sbuf, 0, sbuf_len(sbuf)); -} - -ic_private ssize_t sbuf_append_n( stringbuf_t* sbuf, const char* s, ssize_t n ) { - return sbuf_insert_at_n( sbuf, s, n, sbuf_len(sbuf)); -} - -ic_private ssize_t sbuf_append( stringbuf_t* sbuf, const char* s ) { - return sbuf_insert_at( sbuf, s, sbuf_len(sbuf)); -} - -ic_private ssize_t sbuf_append_char( stringbuf_t* sbuf, char c ) { - char buf[2]; - buf[0] = c; - buf[1] = 0; - return sbuf_append( sbuf, buf ); -} - -ic_private void sbuf_replace(stringbuf_t* sbuf, const char* s) { - sbuf_clear(sbuf); - sbuf_append(sbuf,s); -} - -ic_private ssize_t sbuf_next_ofs( stringbuf_t* sbuf, ssize_t pos, ssize_t* cwidth ) { - return str_next_ofs( sbuf->buf, sbuf->count, pos, cwidth); -} - -ic_private ssize_t sbuf_prev_ofs( stringbuf_t* sbuf, ssize_t pos, ssize_t* cwidth ) { - return str_prev_ofs( sbuf->buf, pos, cwidth); -} - -ic_private ssize_t sbuf_next( stringbuf_t* sbuf, ssize_t pos, ssize_t* cwidth) { - ssize_t ofs = sbuf_next_ofs(sbuf,pos,cwidth); - if (ofs <= 0) return -1; - assert(pos + ofs <= sbuf->count); - return pos + ofs; -} - -ic_private ssize_t sbuf_prev( stringbuf_t* sbuf, ssize_t pos, ssize_t* cwidth) { - ssize_t ofs = sbuf_prev_ofs(sbuf,pos,cwidth); - if (ofs <= 0) return -1; - assert(pos - ofs >= 0); - return pos - ofs; -} - -ic_private ssize_t sbuf_delete_char_before( stringbuf_t* sbuf, ssize_t pos ) { - ssize_t n = sbuf_prev_ofs(sbuf, pos, NULL); - if (n <= 0) return 0; - assert( pos - n >= 0 ); - sbuf_delete_at(sbuf, pos - n, n); - return pos - n; -} - -ic_private void sbuf_delete_char_at( stringbuf_t* sbuf, ssize_t pos ) { - ssize_t n = sbuf_next_ofs(sbuf, pos, NULL); - if (n <= 0) return; - assert( pos + n <= sbuf->count ); - sbuf_delete_at(sbuf, pos, n); - return; -} - -ic_private ssize_t sbuf_swap_char( stringbuf_t* sbuf, ssize_t pos ) { - ssize_t next = sbuf_next_ofs(sbuf, pos, NULL); - if (next <= 0) return 0; - ssize_t prev = sbuf_prev_ofs(sbuf, pos, NULL); - if (prev <= 0) return 0; - char buf[64]; - if (prev >= 63) return 0; - ic_memcpy(buf, sbuf->buf + pos - prev, prev ); - ic_memmove(sbuf->buf + pos - prev, sbuf->buf + pos, next); - ic_memmove(sbuf->buf + pos - prev + next, buf, prev); - return pos - prev; -} - -ic_private ssize_t sbuf_find_line_start( stringbuf_t* sbuf, ssize_t pos ) { - return str_find_line_start( sbuf->buf, sbuf->count, pos); -} - -ic_private ssize_t sbuf_find_line_end( stringbuf_t* sbuf, ssize_t pos ) { - return str_find_line_end( sbuf->buf, sbuf->count, pos); -} - -ic_private ssize_t sbuf_find_word_start( stringbuf_t* sbuf, ssize_t pos ) { - return str_find_word_start( sbuf->buf, sbuf->count, pos); -} - -ic_private ssize_t sbuf_find_word_end( stringbuf_t* sbuf, ssize_t pos ) { - return str_find_word_end( sbuf->buf, sbuf->count, pos); -} - -ic_private ssize_t sbuf_find_ws_word_start( stringbuf_t* sbuf, ssize_t pos ) { - return str_find_ws_word_start( sbuf->buf, sbuf->count, pos); -} - -ic_private ssize_t sbuf_find_ws_word_end( stringbuf_t* sbuf, ssize_t pos ) { - return str_find_ws_word_end( sbuf->buf, sbuf->count, pos); -} - -// find row/col position -ic_private ssize_t sbuf_get_pos_at_rc( stringbuf_t* sbuf, ssize_t termw, ssize_t promptw, ssize_t cpromptw, ssize_t row, ssize_t col ) { - return str_get_pos_at_rc( sbuf->buf, sbuf->count, termw, promptw, cpromptw, row, col); -} - -// get row/col for a given position -ic_private ssize_t sbuf_get_rc_at_pos( stringbuf_t* sbuf, ssize_t termw, ssize_t promptw, ssize_t cpromptw, ssize_t pos, rowcol_t* rc ) { - return str_get_rc_at_pos( sbuf->buf, sbuf->count, termw, promptw, cpromptw, pos, rc); -} - -ic_private ssize_t sbuf_get_wrapped_rc_at_pos( stringbuf_t* sbuf, ssize_t termw, ssize_t newtermw, ssize_t promptw, ssize_t cpromptw, ssize_t pos, rowcol_t* rc ) { - return str_get_wrapped_rc_at_pos( sbuf->buf, sbuf->count, termw, newtermw, promptw, cpromptw, pos, rc); -} - -ic_private ssize_t sbuf_for_each_row( stringbuf_t* sbuf, ssize_t termw, ssize_t promptw, ssize_t cpromptw, row_fun_t* fun, void* arg, void* res ) { - if (sbuf == NULL) return 0; - return str_for_each_row( sbuf->buf, sbuf->count, termw, promptw, cpromptw, fun, arg, res); -} - - -// Duplicate and decode from utf-8 (for non-utf8 terminals) -ic_private char* sbuf_strdup_from_utf8(stringbuf_t* sbuf) { - ssize_t len = sbuf_len(sbuf); - if (sbuf == NULL || len <= 0) return NULL; - char* s = mem_zalloc_tp_n(sbuf->mem, char, len); - if (s == NULL) return NULL; - ssize_t dest = 0; - for (ssize_t i = 0; i < len; ) { - ssize_t ofs = sbuf_next_ofs(sbuf, i, NULL); - if (ofs <= 0) { - // invalid input - break; - } - else if (ofs == 1) { - // regular character - s[dest++] = sbuf->buf[i]; - } - else if (sbuf->buf[i] == '\x1B') { - // skip escape sequences - } - else { - // decode unicode - ssize_t nread; - unicode_t uchr = unicode_from_qutf8( (const uint8_t*)(sbuf->buf + i), ofs, &nread); - uint8_t c; - if (unicode_is_raw(uchr, &c)) { - // raw byte, output as is (this will take care of locale specific input) - s[dest++] = (char)c; - } - else if (uchr <= 0x7F) { - // allow ascii - s[dest++] = (char)uchr; - } - else { - // skip unknown unicode characters.. - // todo: convert according to locale? - } - } - i += ofs; - } - assert(dest <= len); - s[dest] = 0; - return s; -} - -//------------------------------------------------------------- -// String helpers -//------------------------------------------------------------- - -ic_public long ic_prev_char( const char* s, long pos ) { - ssize_t len = ic_strlen(s); - if (pos < 0 || pos > len) return -1; - ssize_t ofs = str_prev_ofs( s, pos, NULL ); - if (ofs <= 0) return -1; - return (long)(pos - ofs); -} - -ic_public long ic_next_char( const char* s, long pos ) { - ssize_t len = ic_strlen(s); - if (pos < 0 || pos > len) return -1; - ssize_t ofs = str_next_ofs( s, len, pos, NULL ); - if (ofs <= 0) return -1; - return (long)(pos + ofs); -} - - -// parse a decimal (leave pi unchanged on error) -ic_private bool ic_atoz(const char* s, ssize_t* pi) { - return (sscanf(s, "%zd", pi) == 1); -} - -// parse two decimals separated by a semicolon -ic_private bool ic_atoz2(const char* s, ssize_t* pi, ssize_t* pj) { - return (sscanf(s, "%zd;%zd", pi, pj) == 2); -} - -// parse unsigned 32-bit (leave pu unchanged on error) -ic_private bool ic_atou32(const char* s, uint32_t* pu) { - return (sscanf(s, "%" SCNu32, pu) == 1); -} - - -// Convenience: character class for whitespace `[ \t\r\n]`. -ic_public bool ic_char_is_white(const char* s, long len) { - if (s == NULL || len != 1) return false; - const char c = *s; - return (c==' ' || c == '\t' || c == '\n' || c == '\r'); -} - -// Convenience: character class for non-whitespace `[^ \t\r\n]`. -ic_public bool ic_char_is_nonwhite(const char* s, long len) { - return !ic_char_is_white(s, len); -} - -// Convenience: character class for separators `[ \t\r\n,.;:/\\\(\)\{\}\[\]]`. -ic_public bool ic_char_is_separator(const char* s, long len) { - if (s == NULL || len != 1) return false; - const char c = *s; - return (strchr(" \t\r\n,.;:/\\(){}[]", c) != NULL); -} - -// Convenience: character class for non-separators. -ic_public bool ic_char_is_nonseparator(const char* s, long len) { - return !ic_char_is_separator(s, len); -} - - -// Convenience: character class for digits (`[0-9]`). -ic_public bool ic_char_is_digit(const char* s, long len) { - if (s == NULL || len != 1) return false; - const char c = *s; - return (c >= '0' && c <= '9'); -} - -// Convenience: character class for hexadecimal digits (`[A-Fa-f0-9]`). -ic_public bool ic_char_is_hexdigit(const char* s, long len) { - if (s == NULL || len != 1) return false; - const char c = *s; - return ((c >= '0' && c <= '9') || (c >= 'a' && c <= 'f') || (c >= 'A' && c <= 'F')); -} - -// Convenience: character class for letters (`[A-Za-z]` and any unicode > 0x80). -ic_public bool ic_char_is_letter(const char* s, long len) { - if (s == NULL || len <= 0) return false; - const char c = *s; - return ((uint8_t)c >= 0x80 || (c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z')); -} - -// Convenience: character class for identifier letters (`[A-Za-z0-9_-]` and any unicode > 0x80). -ic_public bool ic_char_is_idletter(const char* s, long len) { - if (s == NULL || len <= 0) return false; - const char c = *s; - return ((uint8_t)c >= 0x80 || (c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z') || (c >= '0' && c <= '9') || (c == '_') || (c == '-')); -} - -// Convenience: character class for filename letters (`[^ \t\r\n`@$><=;|&{(]`). -ic_public bool ic_char_is_filename_letter(const char* s, long len) { - if (s == NULL || len <= 0) return false; - const char c = *s; - return ((uint8_t)c >= 0x80 || (strchr(" \t\r\n`@$><=;|&{}()[]", c) == NULL)); -} - -// Convenience: If this is a token start, returns the length (or <= 0 if not found). -ic_public long ic_is_token(const char* s, long pos, ic_is_char_class_fun_t* is_token_char) { - if (s == NULL || pos < 0 || is_token_char == NULL) return -1; - ssize_t len = ic_strlen(s); - if (pos >= len) return -1; - if (pos > 0 && is_token_char(s + pos -1, 1)) return -1; // token start? - ssize_t i = pos; - while ( i < len ) { - ssize_t next = str_next_ofs(s, len, i, NULL); - if (next <= 0) return -1; - if (!is_token_char(s + i, (long)next)) break; - i += next; - } - return (long)(i - pos); -} - - -static int ic_strncmp(const char* s1, const char* s2, ssize_t n) { - return strncmp(s1, s2, to_size_t(n)); -} - -// Convenience: Does this match the specified token? -// Ensures not to match prefixes or suffixes, and returns the length of the match (in bytes). -// E.g. `ic_match_token("function",0,&ic_char_is_letter,"fun")` returns 0. -ic_public long ic_match_token(const char* s, long pos, ic_is_char_class_fun_t* is_token_char, const char* token) { - long n = ic_is_token(s, pos, is_token_char); - if (n > 0 && token != NULL && n == ic_strlen(token) && ic_strncmp(s + pos, token, n) == 0) { - return n; - } - else { - return 0; - } -} - - -// Convenience: Do any of the specified tokens match? -// Ensures not to match prefixes or suffixes, and returns the length of the match (in bytes). -// Ensures not to match prefixes or suffixes. -// E.g. `ic_match_any_token("function",0,&ic_char_is_letter,{"fun","func",NULL})` returns 0. -ic_public long ic_match_any_token(const char* s, long pos, ic_is_char_class_fun_t* is_token_char, const char** tokens) { - long n = ic_is_token(s, pos, is_token_char); - if (n <= 0 || tokens == NULL) return 0; - for (const char** token = tokens; *token != NULL; token++) { - if (n == ic_strlen(*token) && ic_strncmp(s + pos, *token, n) == 0) { - return n; - } - } - return 0; -} - diff --git a/vendor/isocline/stringbuf.h b/vendor/isocline/stringbuf.h deleted file mode 100644 index 39b21ea4..00000000 --- a/vendor/isocline/stringbuf.h +++ /dev/null @@ -1,121 +0,0 @@ -/* ---------------------------------------------------------------------------- - Copyright (c) 2021, Daan Leijen - This is free software; you can redistribute it and/or modify it - under the terms of the MIT License. A copy of the license can be - found in the "LICENSE" file at the root of this distribution. ------------------------------------------------------------------------------*/ -#pragma once -#ifndef IC_STRINGBUF_H -#define IC_STRINGBUF_H - -#include -#include "common.h" - -//------------------------------------------------------------- -// string buffer -// in-place modified buffer with edit operations -// that grows on demand. -//------------------------------------------------------------- - -// abstract string buffer -struct stringbuf_s; -typedef struct stringbuf_s stringbuf_t; - -ic_private stringbuf_t* sbuf_new( alloc_t* mem ); -ic_private void sbuf_free( stringbuf_t* sbuf ); -ic_private char* sbuf_free_dup(stringbuf_t* sbuf); -ic_private ssize_t sbuf_len(const stringbuf_t* s); - -ic_private const char* sbuf_string_at( stringbuf_t* sbuf, ssize_t pos ); -ic_private const char* sbuf_string( stringbuf_t* sbuf ); -ic_private char sbuf_char_at(stringbuf_t* sbuf, ssize_t pos); -ic_private char* sbuf_strdup_at( stringbuf_t* sbuf, ssize_t pos ); -ic_private char* sbuf_strdup( stringbuf_t* sbuf ); -ic_private char* sbuf_strdup_from_utf8(stringbuf_t* sbuf); // decode to locale - - -ic_private ssize_t sbuf_appendf(stringbuf_t* sb, const char* fmt, ...); -ic_private ssize_t sbuf_append_vprintf(stringbuf_t* sb, const char* fmt, va_list args); - -ic_private stringbuf_t* sbuf_split_at( stringbuf_t* sb, ssize_t pos ); - -// primitive edit operations (inserts return the new position) -ic_private void sbuf_clear(stringbuf_t* sbuf); -ic_private void sbuf_replace(stringbuf_t* sbuf, const char* s); -ic_private void sbuf_delete_at(stringbuf_t* sbuf, ssize_t pos, ssize_t count); -ic_private void sbuf_delete_from_to(stringbuf_t* sbuf, ssize_t pos, ssize_t end); -ic_private void sbuf_delete_from(stringbuf_t* sbuf, ssize_t pos ); -ic_private ssize_t sbuf_insert_at_n(stringbuf_t* sbuf, const char* s, ssize_t n, ssize_t pos ); -ic_private ssize_t sbuf_insert_at(stringbuf_t* sbuf, const char* s, ssize_t pos ); -ic_private ssize_t sbuf_insert_char_at(stringbuf_t* sbuf, char c, ssize_t pos ); -ic_private ssize_t sbuf_insert_unicode_at(stringbuf_t* sbuf, unicode_t u, ssize_t pos); -ic_private ssize_t sbuf_append_n(stringbuf_t* sbuf, const char* s, ssize_t n); -ic_private ssize_t sbuf_append(stringbuf_t* sbuf, const char* s); -ic_private ssize_t sbuf_append_char(stringbuf_t* sbuf, char c); - -// high level edit operations (return the new position) -ic_private ssize_t sbuf_next( stringbuf_t* sbuf, ssize_t pos, ssize_t* cwidth ); -ic_private ssize_t sbuf_prev( stringbuf_t* sbuf, ssize_t pos, ssize_t* cwidth ); -ic_private ssize_t sbuf_next_ofs(stringbuf_t* sbuf, ssize_t pos, ssize_t* cwidth); - -ic_private ssize_t sbuf_delete_char_before( stringbuf_t* sbuf, ssize_t pos ); -ic_private void sbuf_delete_char_at( stringbuf_t* sbuf, ssize_t pos ); -ic_private ssize_t sbuf_swap_char( stringbuf_t* sbuf, ssize_t pos ); - -ic_private ssize_t sbuf_find_line_start( stringbuf_t* sbuf, ssize_t pos ); -ic_private ssize_t sbuf_find_line_end( stringbuf_t* sbuf, ssize_t pos ); -ic_private ssize_t sbuf_find_word_start( stringbuf_t* sbuf, ssize_t pos ); -ic_private ssize_t sbuf_find_word_end( stringbuf_t* sbuf, ssize_t pos ); -ic_private ssize_t sbuf_find_ws_word_start( stringbuf_t* sbuf, ssize_t pos ); -ic_private ssize_t sbuf_find_ws_word_end( stringbuf_t* sbuf, ssize_t pos ); - -// parse a decimal -ic_private bool ic_atoz(const char* s, ssize_t* i); -// parse two decimals separated by a semicolon -ic_private bool ic_atoz2(const char* s, ssize_t* i, ssize_t* j); -ic_private bool ic_atou32(const char* s, uint32_t* pu); - -// row/column info -typedef struct rowcol_s { - ssize_t row; - ssize_t col; - ssize_t row_start; - ssize_t row_len; - bool first_on_row; - bool last_on_row; -} rowcol_t; - -// find row/col position -ic_private ssize_t sbuf_get_pos_at_rc( stringbuf_t* sbuf, ssize_t termw, ssize_t promptw, ssize_t cpromptw, - ssize_t row, ssize_t col ); -// get row/col for a given position -ic_private ssize_t sbuf_get_rc_at_pos( stringbuf_t* sbuf, ssize_t termw, ssize_t promptw, ssize_t cpromptw, - ssize_t pos, rowcol_t* rc ); - -ic_private ssize_t sbuf_get_wrapped_rc_at_pos( stringbuf_t* sbuf, ssize_t termw, ssize_t newtermw, ssize_t promptw, ssize_t cpromptw, - ssize_t pos, rowcol_t* rc ); - -// row iteration -typedef bool (row_fun_t)(const char* s, - ssize_t row, ssize_t row_start, ssize_t row_len, - ssize_t startw, // prompt width - bool is_wrap, const void* arg, void* res); - -ic_private ssize_t sbuf_for_each_row( stringbuf_t* sbuf, ssize_t termw, ssize_t promptw, ssize_t cpromptw, - row_fun_t* fun, void* arg, void* res ); - - -//------------------------------------------------------------- -// Strings -//------------------------------------------------------------- - -// skip a single CSI sequence (ESC [ ...) -ic_private bool skip_csi_esc( const char* s, ssize_t len, ssize_t* esclen ); // used in term.c - -ic_private ssize_t str_column_width( const char* s ); -ic_private ssize_t str_prev_ofs( const char* s, ssize_t pos, ssize_t* cwidth ); -ic_private ssize_t str_next_ofs( const char* s, ssize_t len, ssize_t pos, ssize_t* cwidth ); -ic_private ssize_t str_skip_until_fit( const char* s, ssize_t max_width); // tail that fits -ic_private ssize_t str_take_while_fit( const char* s, ssize_t max_width); // prefix that fits - -#endif // IC_STRINGBUF_H diff --git a/vendor/isocline/term.c b/vendor/isocline/term.c deleted file mode 100644 index 41994948..00000000 --- a/vendor/isocline/term.c +++ /dev/null @@ -1,1124 +0,0 @@ -/* ---------------------------------------------------------------------------- - Copyright (c) 2021, Daan Leijen - This is free software; you can redistribute it and/or modify it - under the terms of the MIT License. A copy of the license can be - found in the "LICENSE" file at the root of this distribution. ------------------------------------------------------------------------------*/ -#include -#include -#include -#include // getenv -#include - -#include "common.h" -#include "tty.h" -#include "term.h" -#include "stringbuf.h" // str_next_ofs - -#if defined(_WIN32) -#include -#define STDOUT_FILENO 1 -#else -#include -#include -#include -#if defined(__linux__) -#include -#endif -#endif - -#define IC_CSI "\x1B[" - -// color support; colors are auto mapped smaller palettes if needed. (see `term_color.c`) -typedef enum palette_e { - MONOCHROME, // no color - ANSI8, // only basic 8 ANSI color (ESC[m, idx: 30-37, +10 for background) - ANSI16, // basic + bright ANSI colors (ESC[m, idx: 30-37, 90-97, +10 for background) - ANSI256, // ANSI 256 color palette (ESC[38;5;m, idx: 0-15 standard color, 16-231 6x6x6 rbg colors, 232-255 gray shades) - ANSIRGB // direct rgb colors supported (ESC[38;2;;;m) -} palette_t; - -// The terminal screen -struct term_s { - int fd_out; // output handle - ssize_t width; // screen column width - ssize_t height; // screen row height - ssize_t raw_enabled; // is raw mode active? counted by start/end pairs - bool nocolor; // show colors? - bool silent; // enable beep? - bool is_utf8; // utf-8 output? determined by the tty - attr_t attr; // current text attributes - palette_t palette; // color support - buffer_mode_t bufmode; // buffer mode - stringbuf_t* buf; // buffer for buffered output - tty_t* tty; // used on posix to get the cursor position - alloc_t* mem; // allocator - #ifdef _WIN32 - HANDLE hcon; // output console handler - WORD hcon_default_attr; // default text attributes - WORD hcon_orig_attr; // original text attributes - DWORD hcon_orig_mode; // original console mode - DWORD hcon_mode; // used console mode - UINT hcon_orig_cp; // original console code-page (locale) - COORD hcon_save_cursor; // saved cursor position (for escape sequence emulation) - #endif -}; - -static bool term_write_direct(term_t* term, const char* s, ssize_t n ); -static inline void term_append_buf(term_t* term, const char* s, ssize_t n); - -//------------------------------------------------------------- -// Colors -//------------------------------------------------------------- - -#include "term_color.c" - -//------------------------------------------------------------- -// Helpers -//------------------------------------------------------------- - -ic_private void term_left(term_t* term, ssize_t n) { - if (n <= 0) return; - term_writef( term, IC_CSI "%zdD", n ); -} - -ic_private void term_right(term_t* term, ssize_t n) { - if (n <= 0) return; - term_writef( term, IC_CSI "%zdC", n ); -} - -ic_private void term_up(term_t* term, ssize_t n) { - if (n <= 0) return; - term_writef( term, IC_CSI "%zdA", n ); -} - -ic_private void term_down(term_t* term, ssize_t n) { - if (n <= 0) return; - term_writef( term, IC_CSI "%zdB", n ); -} - -ic_private void term_clear_line(term_t* term) { - term_write( term, "\r" IC_CSI "K"); -} - -ic_private void term_clear_to_end_of_line(term_t* term) { - term_write(term, IC_CSI "K"); -} - -ic_private void term_start_of_line(term_t* term) { - term_write( term, "\r" ); -} - -ic_private ssize_t term_get_width(term_t* term) { - return term->width; -} - -ic_private ssize_t term_get_height(term_t* term) { - return term->height; -} - -ic_private void term_attr_reset(term_t* term) { - term_write(term, IC_CSI "m" ); -} - -ic_private void term_underline(term_t* term, bool on) { - term_write(term, on ? IC_CSI "4m" : IC_CSI "24m" ); -} - -ic_private void term_reverse(term_t* term, bool on) { - term_write(term, on ? IC_CSI "7m" : IC_CSI "27m"); -} - -ic_private void term_bold(term_t* term, bool on) { - term_write(term, on ? IC_CSI "1m" : IC_CSI "22m" ); -} - -ic_private void term_italic(term_t* term, bool on) { - term_write(term, on ? IC_CSI "3m" : IC_CSI "23m" ); -} - -ic_private void term_writeln(term_t* term, const char* s) { - term_write(term,s); - term_write(term,"\n"); -} - -ic_private void term_write_char(term_t* term, char c) { - char buf[2]; - buf[0] = c; - buf[1] = 0; - term_write_n(term, buf, 1 ); -} - -ic_private attr_t term_get_attr( const term_t* term ) { - return term->attr; -} - -ic_private void term_set_attr( term_t* term, attr_t attr ) { - if (term->nocolor) return; - if (attr.x.color != term->attr.x.color && attr.x.color != IC_COLOR_NONE) { - term_color(term,attr.x.color); - if (term->palette < ANSIRGB && color_is_rgb(attr.x.color)) { - term->attr.x.color = attr.x.color; // actual color may have been approximated but we keep the actual color to avoid updating every time - } - } - if (attr.x.bgcolor != term->attr.x.bgcolor && attr.x.bgcolor != IC_COLOR_NONE) { - term_bgcolor(term,attr.x.bgcolor); - if (term->palette < ANSIRGB && color_is_rgb(attr.x.bgcolor)) { - term->attr.x.bgcolor = attr.x.bgcolor; - } - } - if (attr.x.bold != term->attr.x.bold && attr.x.bold != IC_NONE) { - term_bold(term,attr.x.bold == IC_ON); - } - if (attr.x.underline != term->attr.x.underline && attr.x.underline != IC_NONE) { - term_underline(term,attr.x.underline == IC_ON); - } - if (attr.x.reverse != term->attr.x.reverse && attr.x.reverse != IC_NONE) { - term_reverse(term,attr.x.reverse == IC_ON); - } - if (attr.x.italic != term->attr.x.italic && attr.x.italic != IC_NONE) { - term_italic(term,attr.x.italic == IC_ON); - } - assert(attr.x.color == term->attr.x.color || attr.x.color == IC_COLOR_NONE); - assert(attr.x.bgcolor == term->attr.x.bgcolor || attr.x.bgcolor == IC_COLOR_NONE); - assert(attr.x.bold == term->attr.x.bold || attr.x.bold == IC_NONE); - assert(attr.x.reverse == term->attr.x.reverse || attr.x.reverse == IC_NONE); - assert(attr.x.underline == term->attr.x.underline || attr.x.underline == IC_NONE); - assert(attr.x.italic == term->attr.x.italic || attr.x.italic == IC_NONE); -} - - -/* -ic_private void term_clear_lines_to_end(term_t* term) { - term_write(term, "\r" IC_CSI "J"); -} - -ic_private void term_show_cursor(term_t* term, bool on) { - term_write(term, on ? IC_CSI "?25h" : IC_CSI "?25l"); -} -*/ - -//------------------------------------------------------------- -// Formatted output -//------------------------------------------------------------- - -ic_private void term_writef(term_t* term, const char* fmt, ...) { - va_list ap; - va_start(ap, fmt); - term_vwritef(term,fmt,ap); - va_end(ap); -} - -ic_private void term_vwritef(term_t* term, const char* fmt, va_list args ) { - sbuf_append_vprintf(term->buf, fmt, args); -} - -ic_private void term_write_formatted( term_t* term, const char* s, const attr_t* attrs ) { - term_write_formatted_n( term, s, attrs, ic_strlen(s)); -} - -ic_private void term_write_formatted_n( term_t* term, const char* s, const attr_t* attrs, ssize_t len ) { - if (attrs == NULL) { - // write directly - term_write(term,s); - } - else { - // ensure raw mode from now on - if (term->raw_enabled <= 0) { - term_start_raw(term); - } - // and output with text attributes - const attr_t default_attr = term_get_attr(term); - attr_t attr = attr_none(); - ssize_t i = 0; - ssize_t n = 0; - while( i+n < len && s[i+n] != 0 ) { - if (!attr_is_eq(attr,attrs[i+n])) { - if (n > 0) { - term_write_n( term, s+i, n ); - i += n; - n = 0; - } - attr = attrs[i]; - term_set_attr( term, attr_update_with(default_attr,attr) ); - } - n++; - } - if (n > 0) { - term_write_n( term, s+i, n ); - i += n; - n = 0; - } - assert(s[i] != 0 || i == len); - term_set_attr(term, default_attr); - } -} - -//------------------------------------------------------------- -// Write to the terminal -// The buffered functions are used to reduce cursor flicker -// during refresh -//------------------------------------------------------------- - -ic_private void term_beep(term_t* term) { - if (term->silent) return; - fprintf(stderr,"\x7"); - fflush(stderr); -} - -ic_private void term_write_repeat(term_t* term, const char* s, ssize_t count) { - for (; count > 0; count--) { - term_write(term, s); - } -} - -ic_private void term_write(term_t* term, const char* s) { - if (s == NULL || s[0] == 0) return; - ssize_t n = ic_strlen(s); - term_write_n(term,s,n); -} - -// Primitive terminal write; all writes go through here -ic_private void term_write_n(term_t* term, const char* s, ssize_t n) { - if (s == NULL || n <= 0) return; - // write to buffer to reduce flicker and to process escape sequences (this may flush too) - term_append_buf(term, s, n); -} - - -//------------------------------------------------------------- -// Buffering -//------------------------------------------------------------- - - -ic_private void term_flush(term_t* term) { - if (sbuf_len(term->buf) > 0) { - //term_show_cursor(term,false); - term_write_direct(term, sbuf_string(term->buf), sbuf_len(term->buf)); - //term_show_cursor(term,true); - sbuf_clear(term->buf); - } -} - -ic_private buffer_mode_t term_set_buffer_mode(term_t* term, buffer_mode_t mode) { - buffer_mode_t oldmode = term->bufmode; - if (oldmode != mode) { - if (mode == UNBUFFERED) { - term_flush(term); - } - term->bufmode = mode; - } - return oldmode; -} - -static inline void term_check_flush(term_t* term, bool contains_nl) { - if (term->bufmode == UNBUFFERED || - sbuf_len(term->buf) > 4000 || - (term->bufmode == LINEBUFFERED && contains_nl)) - { - term_flush(term); - } -} - -//------------------------------------------------------------- -// Init -//------------------------------------------------------------- - -static inline void term_init_raw(term_t* term); - -ic_private term_t* term_new(alloc_t* mem, tty_t* tty, bool nocolor, bool silent, int fd_out ) -{ - term_t* term = mem_zalloc_tp(mem, term_t); - if (term == NULL) return NULL; - - term->fd_out = (fd_out < 0 ? STDOUT_FILENO : fd_out); - term->nocolor = nocolor || (isatty(term->fd_out) == 0); - term->silent = silent; - term->mem = mem; - term->tty = tty; // can be NULL - term->width = 80; - term->height = 25; - term->is_utf8 = tty_is_utf8(tty); - term->palette = ANSI16; // almost universally supported - term->buf = sbuf_new(mem); - term->bufmode = LINEBUFFERED; - term->attr = attr_default(); - - // respect NO_COLOR - if (getenv("NO_COLOR") != NULL) { - term->nocolor = true; - } - if (!term->nocolor) { - // detect color palette - // COLORTERM takes precedence - const char* colorterm = getenv("COLORTERM"); - const char* eterm = getenv("TERM"); - if (ic_contains(colorterm,"24bit") || ic_contains(colorterm,"truecolor") || ic_contains(colorterm,"direct")) { - term->palette = ANSIRGB; - } - else if (ic_contains(colorterm,"8bit") || ic_contains(colorterm,"256color")) { term->palette = ANSI256; } - else if (ic_contains(colorterm,"4bit") || ic_contains(colorterm,"16color")) { term->palette = ANSI16; } - else if (ic_contains(colorterm,"3bit") || ic_contains(colorterm,"8color")) { term->palette = ANSI8; } - else if (ic_contains(colorterm,"1bit") || ic_contains(colorterm,"nocolor") || ic_contains(colorterm,"monochrome")) { - term->palette = MONOCHROME; - } - // otherwise check for some specific terminals - else if (getenv("WT_SESSION") != NULL) { term->palette = ANSIRGB; } // Windows terminal - else if (getenv("ITERM_SESSION_ID") != NULL) { term->palette = ANSIRGB; } // iTerm2 terminal - else if (getenv("VSCODE_PID") != NULL) { term->palette = ANSIRGB; } // vscode terminal - else { - // and otherwise fall back to checking TERM - if (ic_contains(eterm,"truecolor") || ic_contains(eterm,"direct") || ic_contains(colorterm,"24bit")) { - term->palette = ANSIRGB; - } - else if (ic_contains(eterm,"alacritty") || ic_contains(eterm,"kitty")) { - term->palette = ANSIRGB; - } - else if (ic_contains(eterm,"256color") || ic_contains(eterm,"gnome")) { - term->palette = ANSI256; - } - else if (ic_contains(eterm,"16color")){ term->palette = ANSI16; } - else if (ic_contains(eterm,"8color")) { term->palette = ANSI8; } - else if (ic_contains(eterm,"monochrome") || ic_contains(eterm,"nocolor") || ic_contains(eterm,"dumb")) { - term->palette = MONOCHROME; - } - } - debug_msg("term: color-bits: %d (COLORTERM=%s, TERM=%s)\n", term_get_color_bits(term), colorterm, eterm); - } - - // read COLUMS/LINES from the environment for a better initial guess. - const char* env_columns = getenv("COLUMNS"); - if (env_columns != NULL) { ic_atoz(env_columns, &term->width); } - const char* env_lines = getenv("LINES"); - if (env_lines != NULL) { ic_atoz(env_lines, &term->height); } - - // initialize raw terminal output and terminal dimensions - term_init_raw(term); - term_update_dim(term); - term_attr_reset(term); // ensure we are at default settings - - return term; -} - -ic_private bool term_is_interactive(const term_t* term) { - ic_unused(term); - // check dimensions (0 is used for debuggers) - // if (term->width <= 0) return false; - - // check editing support - const char* eterm = getenv("TERM"); - debug_msg("term: TERM=%s\n", eterm); - if (eterm != NULL && - (strstr("dumb|DUMB|cons25|CONS25|emacs|EMACS",eterm) != NULL)) { - return false; - } - - return true; -} - -ic_private bool term_enable_beep(term_t* term, bool enable) { - bool prev = term->silent; - term->silent = !enable; - return prev; -} - -ic_private bool term_enable_color(term_t* term, bool enable) { - bool prev = !term->nocolor; - term->nocolor = !enable; - return prev; -} - -ic_private void term_free(term_t* term) { - if (term == NULL) return; - term_flush(term); - term_end_raw(term, true); - sbuf_free(term->buf); term->buf = NULL; - mem_free(term->mem, term); -} - -//------------------------------------------------------------- -// For best portability and applications inserting CSI SGR (ESC[ .. m) -// codes themselves in strings, we interpret these at the -// lowest level so we can have a `term_get_attr` function which -// is needed for bracketed styles etc. -//------------------------------------------------------------- - -static inline void term_append_esc(term_t* term, const char* const s, ssize_t len) { - if (s[1]=='[' && s[len-1] == 'm') { - // it is a CSI SGR sequence: ESC[ ... m - if (term->nocolor) return; // ignore escape sequences if nocolor is set - term->attr = attr_update_with(term->attr, attr_from_esc_sgr(s,len)); - } - // and write out the escape sequence as-is - sbuf_append_n(term->buf, s, len); -} - - -static inline void term_append_utf8(term_t* term, const char* s, ssize_t len) { - ssize_t nread; - unicode_t uchr = unicode_from_qutf8((const uint8_t*)s, len, &nread); - uint8_t c; - if (unicode_is_raw(uchr, &c)) { - // write bytes as is; this also ensure that on non-utf8 terminals characters between 0x80-0xFF - // go through _as is_ due to the qutf8 encoding. - sbuf_append_char(term->buf,(char)c); - } - else if (!term->is_utf8) { - // on non-utf8 terminals still send utf-8 and hope for the best - // todo: we could try to convert to the locale first? - sbuf_append_n(term->buf, s, len); - // sbuf_appendf(term->buf, "\x1B[%" PRIu32 "u", uchr); // unicode escape code - } - else { - // write utf-8 as is - sbuf_append_n(term->buf, s, len); - } -} - -static inline void term_append_buf( term_t* term, const char* s, ssize_t len ) { - ssize_t pos = 0; - bool newline = false; - while (pos < len) { - // handle ascii sequences in bulk - ssize_t ascii = 0; - ssize_t next; - while ((next = str_next_ofs(s, len, pos+ascii, NULL)) > 0 && - (uint8_t)s[pos + ascii] > '\x1B' && (uint8_t)s[pos + ascii] <= 0x7F ) - { - ascii += next; - } - if (ascii > 0) { - sbuf_append_n(term->buf, s+pos, ascii); - pos += ascii; - } - if (next <= 0) break; - - const uint8_t c = (uint8_t)s[pos]; - // handle utf8 sequences (for non-utf8 terminals) - if (c >= 0x80) { - term_append_utf8(term, s+pos, next); - } - // handle escape sequence (note: str_next_ofs considers whole CSI escape sequences at a time) - else if (next > 1 && c == '\x1B') { - term_append_esc(term, s+pos, next); - } - else if (c < ' ' && c != 0 && (c < '\x07' || c > '\x0D')) { - // ignore control characters except \a, \b, \t, \n, \r, and form-feed and vertical tab. - } - else { - if (c == '\n') { newline = true; } - sbuf_append_n(term->buf, s+pos, next); - } - pos += next; - } - // possibly flush - term_check_flush(term, newline); -} - -//------------------------------------------------------------- -// Platform dependent: Write directly to the terminal -//------------------------------------------------------------- - -#if !defined(_WIN32) - -// write to the console without further processing -static bool term_write_direct(term_t* term, const char* s, ssize_t n) { - ssize_t count = 0; - while( count < n ) { - ssize_t nwritten = write(term->fd_out, s + count, to_size_t(n - count)); - if (nwritten > 0) { - count += nwritten; - } - else if (errno != EINTR && errno != EAGAIN) { - debug_msg("term: write failed: length %i, errno %i: \"%s\"\n", n, errno, s); - return false; - } - } - return true; -} - -#else - -//---------------------------------------------------------------------------------- -// On windows we use the new virtual terminal processing if it is available (Windows Terminal) -// but fall back to ansi escape emulation on older systems but also for example -// the PS terminal -// -// note: we use row/col as 1-based ANSI escape while windows X/Y coords are 0-based. -//----------------------------------------------------------------------------------- - -#if !defined(ENABLE_VIRTUAL_TERMINAL_PROCESSING) -#define ENABLE_VIRTUAL_TERMINAL_PROCESSING (0) -#endif -#if !defined(ENABLE_LVB_GRID_WORLDWIDE) -#define ENABLE_LVB_GRID_WORLDWIDE (0) -#endif - -// direct write to the console without further processing -static bool term_write_console(term_t* term, const char* s, ssize_t n ) { - DWORD written; - // WriteConsoleA(term->hcon, s, (DWORD)(to_size_t(n)), &written, NULL); - WriteFile(term->hcon, s, (DWORD)(to_size_t(n)), &written, NULL); // so it can be redirected - return (written == (DWORD)(to_size_t(n))); -} - -static bool term_get_cursor_pos( term_t* term, ssize_t* row, ssize_t* col) { - *row = 0; - *col = 0; - CONSOLE_SCREEN_BUFFER_INFO info; - if (!GetConsoleScreenBufferInfo(term->hcon, &info)) return false; - *row = (ssize_t)info.dwCursorPosition.Y + 1; - *col = (ssize_t)info.dwCursorPosition.X + 1; - return true; -} - -static inline void term_move_cursor_to( term_t* term, ssize_t row, ssize_t col ) { - CONSOLE_SCREEN_BUFFER_INFO info; - if (!GetConsoleScreenBufferInfo( term->hcon, &info )) return; - if (col > info.dwSize.X) col = info.dwSize.X; - if (row > info.dwSize.Y) row = info.dwSize.Y; - if (col <= 0) col = 1; - if (row <= 0) row = 1; - COORD coord; - coord.X = (SHORT)col - 1; - coord.Y = (SHORT)row - 1; - SetConsoleCursorPosition( term->hcon, coord); -} - -static inline void term_cursor_save(term_t* term) { - memset(&term->hcon_save_cursor, 0, sizeof(term->hcon_save_cursor)); - CONSOLE_SCREEN_BUFFER_INFO info; - if (!GetConsoleScreenBufferInfo(term->hcon, &info)) return; - term->hcon_save_cursor = info.dwCursorPosition; -} - -static inline void term_cursor_restore(term_t* term) { - if (term->hcon_save_cursor.X == 0) return; - SetConsoleCursorPosition(term->hcon, term->hcon_save_cursor); -} - -static inline void term_move_cursor( term_t* term, ssize_t drow, ssize_t dcol, ssize_t n ) { - CONSOLE_SCREEN_BUFFER_INFO info; - if (!GetConsoleScreenBufferInfo( term->hcon, &info )) return; - COORD cur = info.dwCursorPosition; - ssize_t col = (ssize_t)cur.X + 1 + n*dcol; - ssize_t row = (ssize_t)cur.Y + 1 + n*drow; - term_move_cursor_to( term, row, col ); -} - -static inline void term_cursor_visible( term_t* term, bool visible ) { - CONSOLE_CURSOR_INFO info; - if (!GetConsoleCursorInfo(term->hcon,&info)) return; - info.bVisible = visible; - SetConsoleCursorInfo(term->hcon,&info); -} - -static inline void term_erase_line( term_t* term, ssize_t mode ) { - CONSOLE_SCREEN_BUFFER_INFO info; - if (!GetConsoleScreenBufferInfo( term->hcon, &info )) return; - DWORD written; - COORD start; - ssize_t length; - if (mode == 2) { - // entire line - start.X = 0; - start.Y = info.dwCursorPosition.Y; - length = (ssize_t)info.srWindow.Right + 1; - } - else if (mode == 1) { - // to start of line - start.X = 0; - start.Y = info.dwCursorPosition.Y; - length = info.dwCursorPosition.X; - } - else { - // to end of line - length = (ssize_t)info.srWindow.Right - info.dwCursorPosition.X + 1; - start = info.dwCursorPosition; - } - FillConsoleOutputAttribute( term->hcon, term->hcon_default_attr, (DWORD)length, start, &written ); - FillConsoleOutputCharacterA( term->hcon, ' ', (DWORD)length, start, &written ); -} - -static inline void term_clear_screen(term_t* term, ssize_t mode) { - CONSOLE_SCREEN_BUFFER_INFO info; - if (!GetConsoleScreenBufferInfo(term->hcon, &info)) return; - COORD start; - start.X = 0; - start.Y = 0; - ssize_t length; - ssize_t width = (ssize_t)info.dwSize.X; - if (mode == 2) { - // entire screen - length = width * info.dwSize.Y; - } - else if (mode == 1) { - // to cursor - length = (width * ((ssize_t)info.dwCursorPosition.Y - 1)) + info.dwCursorPosition.X; - } - else { - // from cursor - start = info.dwCursorPosition; - length = (width * ((ssize_t)info.dwSize.Y - info.dwCursorPosition.Y)) + (width - info.dwCursorPosition.X + 1); - } - DWORD written; - FillConsoleOutputAttribute(term->hcon, term->hcon_default_attr, (DWORD)length, start, &written); - FillConsoleOutputCharacterA(term->hcon, ' ', (DWORD)length, start, &written); -} - -static WORD attr_color[8] = { - 0, // black - FOREGROUND_RED, // maroon - FOREGROUND_GREEN, // green - FOREGROUND_RED | FOREGROUND_GREEN, // orange - FOREGROUND_BLUE, // navy - FOREGROUND_RED | FOREGROUND_BLUE, // purple - FOREGROUND_GREEN | FOREGROUND_BLUE, // teal - FOREGROUND_RED | FOREGROUND_GREEN | FOREGROUND_BLUE, // light gray -}; - -static inline void term_set_win_attr( term_t* term, attr_t ta ) { - WORD def_attr = term->hcon_default_attr; - CONSOLE_SCREEN_BUFFER_INFO info; - if (!GetConsoleScreenBufferInfo( term->hcon, &info )) return; - WORD cur_attr = info.wAttributes; - WORD attr = cur_attr; - if (ta.x.color != IC_COLOR_NONE) { - if (ta.x.color >= IC_ANSI_BLACK && ta.x.color <= IC_ANSI_SILVER) { - attr = (attr & 0xFFF0) | attr_color[ta.x.color - IC_ANSI_BLACK]; - } - else if (ta.x.color >= IC_ANSI_GRAY && ta.x.color <= IC_ANSI_WHITE) { - attr = (attr & 0xFFF0) | attr_color[ta.x.color - IC_ANSI_GRAY] | FOREGROUND_INTENSITY; - } - else if (ta.x.color == IC_ANSI_DEFAULT) { - attr = (attr & 0xFFF0) | (def_attr & 0x000F); - } - } - if (ta.x.bgcolor != IC_COLOR_NONE) { - if (ta.x.bgcolor >= IC_ANSI_BLACK && ta.x.bgcolor <= IC_ANSI_SILVER) { - attr = (attr & 0xFF0F) | (WORD)(attr_color[ta.x.bgcolor - IC_ANSI_BLACK] << 4); - } - else if (ta.x.bgcolor >= IC_ANSI_GRAY && ta.x.bgcolor <= IC_ANSI_WHITE) { - attr = (attr & 0xFF0F) | (WORD)(attr_color[ta.x.bgcolor - IC_ANSI_GRAY] << 4) | BACKGROUND_INTENSITY; - } - else if (ta.x.bgcolor == IC_ANSI_DEFAULT) { - attr = (attr & 0xFF0F) | (def_attr & 0x00F0); - } - } - if (ta.x.underline != IC_NONE) { - attr = (attr & ~COMMON_LVB_UNDERSCORE) | (ta.x.underline == IC_ON ? COMMON_LVB_UNDERSCORE : 0); - } - if (ta.x.reverse != IC_NONE) { - attr = (attr & ~COMMON_LVB_REVERSE_VIDEO) | (ta.x.reverse == IC_ON ? COMMON_LVB_REVERSE_VIDEO : 0); - } - if (attr != cur_attr) { - SetConsoleTextAttribute(term->hcon, attr); - } -} - -static ssize_t esc_param( const char* s, ssize_t def ) { - if (*s == '?') s++; - ssize_t n = def; - ic_atoz(s, &n); - return n; -} - -static inline void esc_param2( const char* s, ssize_t* p1, ssize_t* p2, ssize_t def ) { - if (*s == '?') s++; - *p1 = def; - *p2 = def; - ic_atoz2(s, p1, p2); -} - -// Emulate escape sequences on older windows. -static inline void term_write_esc( term_t* term, const char* s, ssize_t len ) { - ssize_t row; - ssize_t col; - - if (s[1] == '[') { - switch (s[len-1]) { - case 'A': - term_move_cursor(term, -1, 0, esc_param(s+2, 1)); - break; - case 'B': - term_move_cursor(term, 1, 0, esc_param(s+2, 1)); - break; - case 'C': - term_move_cursor(term, 0, 1, esc_param(s+2, 1)); - break; - case 'D': - term_move_cursor(term, 0, -1, esc_param(s+2, 1)); - break; - case 'H': - esc_param2(s+2, &row, &col, 1); - term_move_cursor_to(term, row, col); - break; - case 'K': - term_erase_line(term, esc_param(s+2, 0)); - break; - case 'm': - term_set_win_attr( term, attr_from_esc_sgr(s,len) ); - break; - - // support some less standard escape codes (currently not used by isocline) - case 'E': // line down - term_get_cursor_pos(term, &row, &col); - row += esc_param(s+2, 1); - term_move_cursor_to(term, row, 1); - break; - case 'F': // line up - term_get_cursor_pos(term, &row, &col); - row -= esc_param(s+2, 1); - term_move_cursor_to(term, row, 1); - break; - case 'G': // absolute column - term_get_cursor_pos(term, &row, &col); - col = esc_param(s+2, 1); - term_move_cursor_to(term, row, col); - break; - case 'J': - term_clear_screen(term, esc_param(s+2, 0)); - break; - case 'h': - if (strncmp(s+2, "?25h", 4) == 0) { - term_cursor_visible(term, true); - } - break; - case 'l': - if (strncmp(s+2, "?25l", 4) == 0) { - term_cursor_visible(term, false); - } - break; - case 's': - term_cursor_save(term); - break; - case 'u': - term_cursor_restore(term); - break; - // otherwise ignore - } - } - else if (s[1] == '7') { - term_cursor_save(term); - } - else if (s[1] == '8') { - term_cursor_restore(term); - } - else { - // otherwise ignore - } -} - -static bool term_write_direct(term_t* term, const char* s, ssize_t len ) { - term_cursor_visible(term,false); // reduce flicker - ssize_t pos = 0; - if ((term->hcon_mode & ENABLE_VIRTUAL_TERMINAL_PROCESSING) != 0) { - // use the builtin virtual terminal processing. (enables truecolor for example) - term_write_console(term, s, len); - pos = len; - } - else { - // emulate escape sequences - while( pos < len ) { - // handle non-control in bulk (including utf-8 sequences) - // (We don't need to handle utf-8 separately as we set the codepage to always be in utf-8 mode) - ssize_t nonctrl = 0; - ssize_t next; - while( (next = str_next_ofs( s, len, pos+nonctrl, NULL )) > 0 && - (uint8_t)s[pos + nonctrl] >= ' ' && (uint8_t)s[pos + nonctrl] <= 0x7F) { - nonctrl += next; - } - if (nonctrl > 0) { - term_write_console(term, s+pos, nonctrl); - pos += nonctrl; - } - if (next <= 0) break; - - if ((uint8_t)s[pos] >= 0x80) { - // utf8 is already processed - term_write_console(term, s+pos, next); - } - else if (next > 1 && s[pos] == '\x1B') { - // handle control (note: str_next_ofs considers whole CSI escape sequences at a time) - term_write_esc(term, s+pos, next); - } - else if (next == 1 && (s[pos] == '\r' || s[pos] == '\n' || s[pos] == '\t' || s[pos] == '\b')) { - term_write_console( term, s+pos, next); - } - else { - // ignore - } - pos += next; - } - } - term_cursor_visible(term,true); - assert(pos == len); - return (pos == len); - -} -#endif - - - -//------------------------------------------------------------- -// Update terminal dimensions -//------------------------------------------------------------- - -#if !defined(_WIN32) - -// send escape query that may return a response on the tty -static bool term_esc_query_raw( term_t* term, const char* query, char* buf, ssize_t buflen ) -{ - if (buf==NULL || buflen <= 0 || query[0] == 0) return false; - bool osc = (query[1] == ']'); - if (!term_write_direct(term, query, ic_strlen(query))) return false; - debug_msg("term: read tty query response to: ESC %s\n", query + 1); - return tty_read_esc_response( term->tty, query[1], osc, buf, buflen ); -} - -static bool term_esc_query( term_t* term, const char* query, char* buf, ssize_t buflen ) -{ - if (!tty_start_raw(term->tty)) return false; - bool ok = term_esc_query_raw(term,query,buf,buflen); - tty_end_raw(term->tty); - return ok; -} - -// get the cursor position via an ESC[6n -static bool term_get_cursor_pos( term_t* term, ssize_t* row, ssize_t* col) -{ - // send escape query - char buf[128]; - if (!term_esc_query(term,"\x1B[6n",buf,128)) return false; - if (!ic_atoz2(buf,row,col)) return false; - return true; -} - -static inline void term_set_cursor_pos( term_t* term, ssize_t row, ssize_t col ) { - term_writef( term, IC_CSI "%zd;%zdH", row, col ); -} - -ic_private bool term_update_dim(term_t* term) { - ssize_t cols = 0; - ssize_t rows = 0; - struct winsize ws; - if (ioctl(term->fd_out, TIOCGWINSZ, &ws) >= 0) { - // ioctl succeeded - cols = ws.ws_col; // debuggers return 0 for the column - rows = ws.ws_row; - } - else { - // determine width by querying the cursor position - debug_msg("term: ioctl term-size failed: %d,%d\n", ws.ws_row, ws.ws_col); - ssize_t col0 = 0; - ssize_t row0 = 0; - if (term_get_cursor_pos(term,&row0,&col0)) { - term_set_cursor_pos(term,999,999); - ssize_t col1 = 0; - ssize_t row1 = 0; - if (term_get_cursor_pos(term,&row1,&col1)) { - cols = col1; - rows = row1; - } - term_set_cursor_pos(term,row0,col0); - } - else { - // cannot query position - // return 0 column - } - } - - // update width and return whether it changed. - bool changed = (term->width != cols || term->height != rows); - debug_msg("terminal dim: %zd,%zd: %s\n", rows, cols, changed ? "changed" : "unchanged"); - if (cols > 0) { - term->width = cols; - term->height = rows; - } - return changed; -} - -#else - -ic_private bool term_update_dim(term_t* term) { - if (term->hcon == 0) { - term->hcon = GetConsoleWindow(); - } - ssize_t rows = 0; - ssize_t cols = 0; - CONSOLE_SCREEN_BUFFER_INFO sbinfo; - if (GetConsoleScreenBufferInfo(term->hcon, &sbinfo)) { - cols = (ssize_t)sbinfo.srWindow.Right - (ssize_t)sbinfo.srWindow.Left + 1; - rows = (ssize_t)sbinfo.srWindow.Bottom - (ssize_t)sbinfo.srWindow.Top + 1; - } - bool changed = (term->width != cols || term->height != rows); - term->width = cols; - term->height = rows; - debug_msg("term: update dim: %zd, %zd\n", term->height, term->width ); - return changed; -} - -#endif - - - -//------------------------------------------------------------- -// Enable/disable terminal raw mode -//------------------------------------------------------------- - -#if !defined(_WIN32) - -// On non-windows, the terminal is set in raw mode by the tty. - -ic_private void term_start_raw(term_t* term) { - term->raw_enabled++; -} - -ic_private void term_end_raw(term_t* term, bool force) { - if (term->raw_enabled <= 0) return; - if (!force) { - term->raw_enabled--; - } - else { - term->raw_enabled = 0; - } -} - -static bool term_esc_query_color_raw(term_t* term, int color_idx, uint32_t* color ) { - char buf[128+1]; - snprintf(buf,128,"\x1B]4;%d;?\x1B\\", color_idx); - if (!term_esc_query_raw( term, buf, buf, 128 )) { - debug_msg("esc query response not received\n"); - return false; - } - if (buf[0] != '4') return false; - const char* rgb = strchr(buf,':'); - if (rgb==NULL) return false; - rgb++; // skip ':' - unsigned int r,g,b; - if (sscanf(rgb,"%x/%x/%x",&r,&g,&b) != 3) return false; - if (rgb[2]!='/') { // 48-bit rgb, hexadecimal round to 24-bit - r = (r+0x7F)/0x100; // note: can "overflow", e.g. 0xFFFF -> 0x100. (and we need `ic_cap8` to convert.) - g = (g+0x7F)/0x100; - b = (b+0x7F)/0x100; - } - *color = (ic_cap8(r)<<16) | (ic_cap8(g)<<8) | ic_cap8(b); - debug_msg("color query: %02x,%02x,%02x: %06x\n", r, g, b, *color); - return true; -} - -// update ansi 16 color palette for better color approximation -static inline void term_update_ansi16(term_t* term) { - debug_msg("update ansi colors\n"); - #if defined(GIO_CMAP) - // try ioctl first (on Linux) - uint8_t cmap[48]; - memset(cmap,0,48); - if (ioctl(term->fd_out,GIO_CMAP,&cmap) >= 0) { - // success - for(ssize_t i = 0; i < 48; i+=3) { - uint32_t color = ((uint32_t)(cmap[i]) << 16) | ((uint32_t)(cmap[i+1]) << 8) | cmap[i+2]; - debug_msg("term (ioctl) ansi color %d: 0x%06x\n", i, color); - ansi256[i] = color; - } - return; - } - else { - debug_msg("ioctl GIO_CMAP failed: entry 1: 0x%02x%02x%02x\n", cmap[3], cmap[4], cmap[5]); - } - #endif - // this seems to be unreliable on some systems (Ubuntu+Gnome terminal) so only enable when known ok. - #if __APPLE__ - // otherwise use OSC 4 escape sequence query - if (tty_start_raw(term->tty)) { - for(ssize_t i = 0; i < 16; i++) { - uint32_t color; - if (!term_esc_query_color_raw(term, i, &color)) break; - debug_msg("term ansi color %d: 0x%06x\n", i, color); - ansi256[i] = color; - } - tty_end_raw(term->tty); - } - #endif -} - -static inline void term_init_raw(term_t* term) { - if (term->palette < ANSIRGB) { - term_update_ansi16(term); - } -} - -#else - -ic_private void term_start_raw(term_t* term) { - if (term->raw_enabled++ > 0) return; - CONSOLE_SCREEN_BUFFER_INFO info; - if (GetConsoleScreenBufferInfo(term->hcon, &info)) { - term->hcon_orig_attr = info.wAttributes; - } - term->hcon_orig_cp = GetConsoleOutputCP(); - SetConsoleOutputCP(CP_UTF8); - if (term->hcon_mode == 0) { - // first time initialization - DWORD mode = ENABLE_PROCESSED_OUTPUT | ENABLE_WRAP_AT_EOL_OUTPUT | ENABLE_LVB_GRID_WORLDWIDE; // for \r \n and \b - // use escape sequence handling if available and the terminal supports it (so we can use rgb colors in Windows terminal) - // Unfortunately, in plain powershell, we can successfully enable terminal processing - // but it still fails to render correctly; so we require the palette be large enough (like in Windows Terminal) - if (term->palette >= ANSI256 && SetConsoleMode(term->hcon, mode | ENABLE_VIRTUAL_TERMINAL_PROCESSING)) { - term->hcon_mode = mode | ENABLE_VIRTUAL_TERMINAL_PROCESSING; - debug_msg("term: console mode: virtual terminal processing enabled\n"); - } - // no virtual terminal processing, emulate instead - else if (SetConsoleMode(term->hcon, mode)) { - term->hcon_mode = mode; - term->palette = ANSI16; - } - GetConsoleMode(term->hcon, &mode); - debug_msg("term: console mode: orig: 0x%x, new: 0x%x, current 0x%x\n", term->hcon_orig_mode, term->hcon_mode, mode); - } - else { - SetConsoleMode(term->hcon, term->hcon_mode); - } -} - -ic_private void term_end_raw(term_t* term, bool force) { - if (term->raw_enabled <= 0) return; - if (!force && term->raw_enabled > 1) { - term->raw_enabled--; - } - else { - term->raw_enabled = 0; - SetConsoleMode(term->hcon, term->hcon_orig_mode); - SetConsoleOutputCP(term->hcon_orig_cp); - SetConsoleTextAttribute(term->hcon, term->hcon_orig_attr); - } -} - -static inline void term_init_raw(term_t* term) { - term->hcon = GetStdHandle(STD_OUTPUT_HANDLE); - GetConsoleMode(term->hcon, &term->hcon_orig_mode); - CONSOLE_SCREEN_BUFFER_INFOEX info; - memset(&info, 0, sizeof(info)); - info.cbSize = sizeof(info); - if (GetConsoleScreenBufferInfoEx(term->hcon, &info)) { - // store default attributes - term->hcon_default_attr = info.wAttributes; - // update our color table with the actual colors used. - for (unsigned i = 0; i < 16; i++) { - COLORREF cr = info.ColorTable[i]; - uint32_t color = (ic_cap8(GetRValue(cr))<<16) | (ic_cap8(GetGValue(cr))<<8) | ic_cap8(GetBValue(cr)); // COLORREF = BGR - // index is also in reverse in the bits 0 and 2 - unsigned j = (i&0x08) | ((i&0x04)>>2) | (i&0x02) | (i&0x01)<<2; - debug_msg("term: ansi color %d is 0x%06x\n", j, color); - ansi256[j] = color; - } - } - else { - DWORD err = GetLastError(); - debug_msg("term: cannot get console screen buffer: %d %x", err, err); - } - term_start_raw(term); // initialize the hcon_mode - term_end_raw(term,false); -} - -#endif diff --git a/vendor/isocline/term.h b/vendor/isocline/term.h deleted file mode 100644 index 50bfd968..00000000 --- a/vendor/isocline/term.h +++ /dev/null @@ -1,85 +0,0 @@ -/* ---------------------------------------------------------------------------- - Copyright (c) 2021, Daan Leijen - This is free software; you can redistribute it and/or modify it - under the terms of the MIT License. A copy of the license can be - found in the "LICENSE" file at the root of this distribution. ------------------------------------------------------------------------------*/ -#pragma once -#ifndef IC_TERM_H -#define IC_TERM_H - -#include "common.h" -#include "tty.h" -#include "stringbuf.h" -#include "attr.h" - -struct term_s; -typedef struct term_s term_t; - -typedef enum buffer_mode_e { - UNBUFFERED, - LINEBUFFERED, - BUFFERED, -} buffer_mode_t; - -// Primitives -ic_private term_t* term_new(alloc_t* mem, tty_t* tty, bool nocolor, bool silent, int fd_out); -ic_private void term_free(term_t* term); - -ic_private bool term_is_interactive(const term_t* term); -ic_private void term_start_raw(term_t* term); -ic_private void term_end_raw(term_t* term, bool force); - -ic_private bool term_enable_beep(term_t* term, bool enable); -ic_private bool term_enable_color(term_t* term, bool enable); - -ic_private void term_flush(term_t* term); -ic_private buffer_mode_t term_set_buffer_mode(term_t* term, buffer_mode_t mode); - -ic_private void term_write_n(term_t* term, const char* s, ssize_t n); -ic_private void term_write(term_t* term, const char* s); -ic_private void term_writeln(term_t* term, const char* s); -ic_private void term_write_char(term_t* term, char c); - -ic_private void term_write_repeat(term_t* term, const char* s, ssize_t count ); -ic_private void term_beep(term_t* term); - -ic_private bool term_update_dim(term_t* term); - -ic_private ssize_t term_get_width(term_t* term); -ic_private ssize_t term_get_height(term_t* term); -ic_private int term_get_color_bits(term_t* term); - -// Helpers -ic_private void term_writef(term_t* term, const char* fmt, ...); -ic_private void term_vwritef(term_t* term, const char* fmt, va_list args); - -ic_private void term_left(term_t* term, ssize_t n); -ic_private void term_right(term_t* term, ssize_t n); -ic_private void term_up(term_t* term, ssize_t n); -ic_private void term_down(term_t* term, ssize_t n); -ic_private void term_start_of_line(term_t* term ); -ic_private void term_clear_line(term_t* term); -ic_private void term_clear_to_end_of_line(term_t* term); -// ic_private void term_clear_lines_to_end(term_t* term); - - -ic_private void term_attr_reset(term_t* term); -ic_private void term_underline(term_t* term, bool on); -ic_private void term_reverse(term_t* term, bool on); -ic_private void term_bold(term_t* term, bool on); -ic_private void term_italic(term_t* term, bool on); - -ic_private void term_color(term_t* term, ic_color_t color); -ic_private void term_bgcolor(term_t* term, ic_color_t color); - -// Formatted output - -ic_private attr_t term_get_attr( const term_t* term ); -ic_private void term_set_attr( term_t* term, attr_t attr ); -ic_private void term_write_formatted( term_t* term, const char* s, const attr_t* attrs ); -ic_private void term_write_formatted_n( term_t* term, const char* s, const attr_t* attrs, ssize_t n ); - -ic_private ic_color_t color_from_ansi256(ssize_t i); - -#endif // IC_TERM_H diff --git a/vendor/isocline/term_color.c b/vendor/isocline/term_color.c deleted file mode 100644 index 98af3cf4..00000000 --- a/vendor/isocline/term_color.c +++ /dev/null @@ -1,371 +0,0 @@ -/* ---------------------------------------------------------------------------- - Copyright (c) 2021, Daan Leijen - This is free software; you can redistribute it and/or modify it - under the terms of the MIT License. A copy of the license can be - found in the "LICENSE" file at the root of this distribution. ------------------------------------------------------------------------------*/ - -// This file is included in "term.c" - -//------------------------------------------------------------- -// Standard ANSI palette for 256 colors -//------------------------------------------------------------- - -static uint32_t ansi256[256] = { - // not const as on some platforms (e.g. Windows, xterm) we update the first 16 entries with the actual used colors. - // 0, standard ANSI - 0x000000, 0x800000, 0x008000, 0x808000, 0x000080, 0x800080, - 0x008080, 0xc0c0c0, - // 8, bright ANSI - 0x808080, 0xff0000, 0x00ff00, 0xffff00, 0x0000ff, 0xff00ff, - 0x00ffff, 0xffffff, - // 6x6x6 RGB colors - // 16 - 0x000000, 0x00005f, 0x000087, 0x0000af, 0x0000d7, 0x0000ff, - 0x005f00, 0x005f5f, 0x005f87, 0x005faf, 0x005fd7, 0x005fff, - 0x008700, 0x00875f, 0x008787, 0x0087af, 0x0087d7, 0x0087ff, - 0x00af00, 0x00af5f, 0x00af87, 0x00afaf, 0x00afd7, 0x00afff, - 0x00d700, 0x00d75f, 0x00d787, 0x00d7af, 0x00d7d7, 0x00d7ff, - 0x00ff00, 0x00ff5f, 0x00ff87, 0x00ffaf, 0x00ffd7, 0x00ffff, - // 52 - 0x5f0000, 0x5f005f, 0x5f0087, 0x5f00af, 0x5f00d7, 0x5f00ff, - 0x5f5f00, 0x5f5f5f, 0x5f5f87, 0x5f5faf, 0x5f5fd7, 0x5f5fff, - 0x5f8700, 0x5f875f, 0x5f8787, 0x5f87af, 0x5f87d7, 0x5f87ff, - 0x5faf00, 0x5faf5f, 0x5faf87, 0x5fafaf, 0x5fafd7, 0x5fafff, - 0x5fd700, 0x5fd75f, 0x5fd787, 0x5fd7af, 0x5fd7d7, 0x5fd7ff, - 0x5fff00, 0x5fff5f, 0x5fff87, 0x5fffaf, 0x5fffd7, 0x5fffff, - // 88 - 0x870000, 0x87005f, 0x870087, 0x8700af, 0x8700d7, 0x8700ff, - 0x875f00, 0x875f5f, 0x875f87, 0x875faf, 0x875fd7, 0x875fff, - 0x878700, 0x87875f, 0x878787, 0x8787af, 0x8787d7, 0x8787ff, - 0x87af00, 0x87af5f, 0x87af87, 0x87afaf, 0x87afd7, 0x87afff, - 0x87d700, 0x87d75f, 0x87d787, 0x87d7af, 0x87d7d7, 0x87d7ff, - 0x87ff00, 0x87ff5f, 0x87ff87, 0x87ffaf, 0x87ffd7, 0x87ffff, - // 124 - 0xaf0000, 0xaf005f, 0xaf0087, 0xaf00af, 0xaf00d7, 0xaf00ff, - 0xaf5f00, 0xaf5f5f, 0xaf5f87, 0xaf5faf, 0xaf5fd7, 0xaf5fff, - 0xaf8700, 0xaf875f, 0xaf8787, 0xaf87af, 0xaf87d7, 0xaf87ff, - 0xafaf00, 0xafaf5f, 0xafaf87, 0xafafaf, 0xafafd7, 0xafafff, - 0xafd700, 0xafd75f, 0xafd787, 0xafd7af, 0xafd7d7, 0xafd7ff, - 0xafff00, 0xafff5f, 0xafff87, 0xafffaf, 0xafffd7, 0xafffff, - // 160 - 0xd70000, 0xd7005f, 0xd70087, 0xd700af, 0xd700d7, 0xd700ff, - 0xd75f00, 0xd75f5f, 0xd75f87, 0xd75faf, 0xd75fd7, 0xd75fff, - 0xd78700, 0xd7875f, 0xd78787, 0xd787af, 0xd787d7, 0xd787ff, - 0xd7af00, 0xd7af5f, 0xd7af87, 0xd7afaf, 0xd7afd7, 0xd7afff, - 0xd7d700, 0xd7d75f, 0xd7d787, 0xd7d7af, 0xd7d7d7, 0xd7d7ff, - 0xd7ff00, 0xd7ff5f, 0xd7ff87, 0xd7ffaf, 0xd7ffd7, 0xd7ffff, - // 196 - 0xff0000, 0xff005f, 0xff0087, 0xff00af, 0xff00d7, 0xff00ff, - 0xff5f00, 0xff5f5f, 0xff5f87, 0xff5faf, 0xff5fd7, 0xff5fff, - 0xff8700, 0xff875f, 0xff8787, 0xff87af, 0xff87d7, 0xff87ff, - 0xffaf00, 0xffaf5f, 0xffaf87, 0xffafaf, 0xffafd7, 0xffafff, - 0xffd700, 0xffd75f, 0xffd787, 0xffd7af, 0xffd7d7, 0xffd7ff, - 0xffff00, 0xffff5f, 0xffff87, 0xffffaf, 0xffffd7, 0xffffff, - // 232, gray scale - 0x080808, 0x121212, 0x1c1c1c, 0x262626, 0x303030, 0x3a3a3a, - 0x444444, 0x4e4e4e, 0x585858, 0x626262, 0x6c6c6c, 0x767676, - 0x808080, 0x8a8a8a, 0x949494, 0x9e9e9e, 0xa8a8a8, 0xb2b2b2, - 0xbcbcbc, 0xc6c6c6, 0xd0d0d0, 0xdadada, 0xe4e4e4, 0xeeeeee -}; - - -//------------------------------------------------------------- -// Create colors -//------------------------------------------------------------- - -// Create a color from a 24-bit color value. -ic_private ic_color_t ic_rgb(uint32_t hex) { - return (ic_color_t)(0x1000000 | (hex & 0xFFFFFF)); -} - -// Limit an int to values between 0 and 255. -static uint32_t ic_cap8(ssize_t i) { - return (i < 0 ? 0 : (i > 255 ? 255 : (uint32_t)i)); -} - -// Create a color from a 24-bit color value. -ic_private ic_color_t ic_rgbx(ssize_t r, ssize_t g, ssize_t b) { - return ic_rgb( (ic_cap8(r)<<16) | (ic_cap8(g)<<8) | ic_cap8(b) ); -} - - -//------------------------------------------------------------- -// Match an rgb color to a ansi8, ansi16, or ansi256 -//------------------------------------------------------------- - -static bool color_is_rgb( ic_color_t color ) { - return (color >= IC_RGB(0)); // bit 24 is set for rgb colors -} - -static void color_to_rgb(ic_color_t color, int* r, int* g, int* b) { - assert(color_is_rgb(color)); - *r = ((color >> 16) & 0xFF); - *g = ((color >> 8) & 0xFF); - *b = (color & 0xFF); -} - -ic_private ic_color_t color_from_ansi256(ssize_t i) { - if (i >= 0 && i < 8) { - return (IC_ANSI_BLACK + (uint32_t)i); - } - else if (i >= 8 && i < 16) { - return (IC_ANSI_DARKGRAY + (uint32_t)(i - 8)); - } - else if (i >= 16 && i <= 255) { - return ic_rgb( ansi256[i] ); - } - else if (i == 256) { - return IC_ANSI_DEFAULT; - } - else { - return IC_ANSI_DEFAULT; - } -} - -static bool is_grayish(int r, int g, int b) { - return (abs(r-g) <= 4) && (abs((r+g)/2 - b) <= 4); -} - -static bool is_grayish_color( uint32_t rgb ) { - int r, g, b; - color_to_rgb(IC_RGB(rgb),&r,&g,&b); - return is_grayish(r,g,b); -} - -static int_least32_t sqr(int_least32_t x) { - return x*x; -} - -// Approximation to delta-E CIE color distance using much -// simpler calculations. See . -// This is essentialy weighted euclidean distance but the weight distribution -// depends on how big the "red" component of the color is. -// We do not take the square root as we only need to find -// the minimal distance (and multiply by 256 to increase precision). -// Needs at least 28-bit signed integers to avoid overflow. -static int_least32_t rgb_distance_rmean( uint32_t color, int r2, int g2, int b2 ) { - int r1, g1, b1; - color_to_rgb(IC_RGB(color),&r1,&g1,&b1); - int_least32_t rmean = (r1 + r2) / 2; - int_least32_t dr2 = sqr(r1 - r2); - int_least32_t dg2 = sqr(g1 - g2); - int_least32_t db2 = sqr(b1 - b2); - int_least32_t dist = ((512+rmean)*dr2) + 1024*dg2 + ((767-rmean)*db2); - return dist; -} - -// Another approximation to delta-E CIE color distance using -// simpler calculations. Similar to `rmean` but adds an adjustment factor -// based on the "red/blue" difference. -static int_least32_t rgb_distance_rbmean( uint32_t color, int r2, int g2, int b2 ) { - int r1, g1, b1; - color_to_rgb(IC_RGB(color),&r1,&g1,&b1); - int_least32_t rmean = (r1 + r2) / 2; - int_least32_t dr2 = sqr(r1 - r2); - int_least32_t dg2 = sqr(g1 - g2); - int_least32_t db2 = sqr(b1 - b2); - int_least32_t dist = 2*dr2 + 4*dg2 + 3*db2 + ((rmean*(dr2 - db2))/256); - return dist; -} - - -// Maintain a small cache of recently used colors. Should be short enough to be effectively constant time. -// If we ever use a more expensive color distance method, we may increase the size a bit (64?) -// (Initial zero initialized cache is valid.) -#define RGB_CACHE_LEN (16) -typedef struct rgb_cache_s { - int last; - int indices[RGB_CACHE_LEN]; - ic_color_t colors[RGB_CACHE_LEN]; -} rgb_cache_t; - -// remember a color in the LRU cache -void rgb_remember( rgb_cache_t* cache, ic_color_t color, int idx ) { - if (cache == NULL) return; - cache->colors[cache->last] = color; - cache->indices[cache->last] = idx; - cache->last++; - if (cache->last >= RGB_CACHE_LEN) { cache->last = 0; } -} - -// quick lookup in cache; -1 on failure -int rgb_lookup( const rgb_cache_t* cache, ic_color_t color ) { - if (cache != NULL) { - for(int i = 0; i < RGB_CACHE_LEN; i++) { - if (cache->colors[i] == color) return cache->indices[i]; - } - } - return -1; -} - -// return the index of the closest matching color -static int rgb_match( uint32_t* palette, int start, int len, rgb_cache_t* cache, ic_color_t color ) { - assert(color_is_rgb(color)); - // in cache? - int min = rgb_lookup(cache,color); - if (min >= 0) { - return min; - } - // otherwise find closest color match in the palette - int r, g, b; - color_to_rgb(color,&r,&g,&b); - min = start; - int_least32_t mindist = (INT_LEAST32_MAX)/4; - for(int i = start; i < len; i++) { - //int_least32_t dist = rgb_distance_rbmean(palette[i],r,g,b); - int_least32_t dist = rgb_distance_rmean(palette[i],r,g,b); - if (is_grayish_color(palette[i]) != is_grayish(r, g, b)) { - // with few colors, make it less eager to substitute a gray for a non-gray (or the other way around) - if (len <= 16) { - dist *= 4; - } - else { - dist = (dist/4)*5; - } - } - if (dist < mindist) { - min = i; - mindist = dist; - } - } - rgb_remember(cache,color,min); - return min; -} - - -// Match RGB to an index in the ANSI 256 color table -static int rgb_to_ansi256(ic_color_t color) { - static rgb_cache_t ansi256_cache; - int c = rgb_match(ansi256, 16, 256, &ansi256_cache, color); // not the first 16 ANSI colors as those may be different - //debug_msg("term: rgb %x -> ansi 256: %d\n", color, c ); - return c; -} - -// Match RGB to an ANSI 16 color code (30-37, 90-97) -static int color_to_ansi16(ic_color_t color) { - if (!color_is_rgb(color)) { - return (int)color; - } - else { - static rgb_cache_t ansi16_cache; - int c = rgb_match(ansi256, 0, 16, &ansi16_cache, color); - //debug_msg("term: rgb %x -> ansi 16: %d\n", color, c ); - return (c < 8 ? 30 + c : 90 + c - 8); - } -} - -// Match RGB to an ANSI 16 color code (30-37, 90-97) -// but assuming the bright colors are simulated using 'bold'. -static int color_to_ansi8(ic_color_t color) { - if (!color_is_rgb(color)) { - return (int)color; - } - else { - // match to basic 8 colors first - static rgb_cache_t ansi8_cache; - int c = 30 + rgb_match(ansi256, 0, 8, &ansi8_cache, color); - // and then adjust for brightness - int r, g, b; - color_to_rgb(color,&r,&g,&b); - if (r>=196 || g>=196 || b>=196) c += 60; - //debug_msg("term: rgb %x -> ansi 8: %d\n", color, c ); - return c; - } -} - - -//------------------------------------------------------------- -// Emit color escape codes based on the terminal capability -//------------------------------------------------------------- - -static void fmt_color_ansi8( char* buf, ssize_t len, ic_color_t color, bool bg ) { - int c = color_to_ansi8(color) + (bg ? 10 : 0); - if (c >= 90) { - snprintf(buf, to_size_t(len), IC_CSI "1;%dm", c - 60); - } - else { - snprintf(buf, to_size_t(len), IC_CSI "22;%dm", c ); - } -} - -static void fmt_color_ansi16( char* buf, ssize_t len, ic_color_t color, bool bg ) { - snprintf( buf, to_size_t(len), IC_CSI "%dm", color_to_ansi16(color) + (bg ? 10 : 0) ); -} - -static void fmt_color_ansi256( char* buf, ssize_t len, ic_color_t color, bool bg ) { - if (!color_is_rgb(color)) { - fmt_color_ansi16(buf,len,color,bg); - } - else { - snprintf( buf, to_size_t(len), IC_CSI "%d;5;%dm", (bg ? 48 : 38), rgb_to_ansi256(color) ); - } -} - -static void fmt_color_rgb( char* buf, ssize_t len, ic_color_t color, bool bg ) { - if (!color_is_rgb(color)) { - fmt_color_ansi16(buf,len,color,bg); - } - else { - int r,g,b; - color_to_rgb(color, &r,&g,&b); - snprintf( buf, to_size_t(len), IC_CSI "%d;2;%d;%d;%dm", (bg ? 48 : 38), r, g, b ); - } -} - -static void fmt_color_ex(char* buf, ssize_t len, palette_t palette, ic_color_t color, bool bg) { - if (color == IC_COLOR_NONE || palette == MONOCHROME) return; - if (palette == ANSI8) { - fmt_color_ansi8(buf,len,color,bg); - } - else if (!color_is_rgb(color) || palette == ANSI16) { - fmt_color_ansi16(buf,len,color,bg); - } - else if (palette == ANSI256) { - fmt_color_ansi256(buf,len,color,bg); - } - else { - fmt_color_rgb(buf,len,color,bg); - } -} - -static void term_color_ex(term_t* term, ic_color_t color, bool bg) { - char buf[128+1]; - fmt_color_ex(buf,128,term->palette,color,bg); - term_write(term,buf); -} - -//------------------------------------------------------------- -// Main API functions -//------------------------------------------------------------- - -ic_private void term_color(term_t* term, ic_color_t color) { - term_color_ex(term,color,false); -} - -ic_private void term_bgcolor(term_t* term, ic_color_t color) { - term_color_ex(term,color,true); -} - -ic_private void term_append_color(term_t* term, stringbuf_t* sbuf, ic_color_t color) { - char buf[128+1]; - fmt_color_ex(buf,128,term->palette,color,false); - sbuf_append(sbuf,buf); -} - -ic_private void term_append_bgcolor(term_t* term, stringbuf_t* sbuf, ic_color_t color) { - char buf[128+1]; - fmt_color_ex(buf, 128, term->palette, color, true); - sbuf_append(sbuf, buf); -} - -ic_private int term_get_color_bits(term_t* term) { - switch (term->palette) { - case MONOCHROME: return 1; - case ANSI8: return 3; - case ANSI16: return 4; - case ANSI256: return 8; - case ANSIRGB: return 24; - default: return 4; - } -} diff --git a/vendor/isocline/tty.c b/vendor/isocline/tty.c deleted file mode 100644 index 16766a8b..00000000 --- a/vendor/isocline/tty.c +++ /dev/null @@ -1,889 +0,0 @@ -/* ---------------------------------------------------------------------------- - Copyright (c) 2021, Daan Leijen - This is free software; you can redistribute it and/or modify it - under the terms of the MIT License. A copy of the license can be - found in the "LICENSE" file at the root of this distribution. ------------------------------------------------------------------------------*/ -#include -#include -#include -#include -#include - -#include "tty.h" - -#if defined(_WIN32) -#include -#include -#define isatty(fd) _isatty(fd) -#define read(fd,s,n) _read(fd,s,n) -#define STDIN_FILENO 0 -#if (_WIN32_WINNT < 0x0600) -WINBASEAPI ULONGLONG WINAPI GetTickCount64(VOID); -#endif -#else -#include -#include -#include -#include -#include -#include -#if !defined(FIONREAD) -#include -#endif -#endif - -#define TTY_PUSH_MAX (32) - -struct tty_s { - int fd_in; // input handle - bool raw_enabled; // is raw mode enabled? - bool is_utf8; // is the input stream in utf-8 mode? - bool has_term_resize_event; // are resize events generated? - bool term_resize_event; // did a term resize happen? - alloc_t* mem; // memory allocator - code_t pushbuf[TTY_PUSH_MAX]; // push back buffer for full key codes - ssize_t push_count; - uint8_t cpushbuf[TTY_PUSH_MAX]; // low level push back buffer for bytes - ssize_t cpush_count; - long esc_initial_timeout; // initial ms wait to see if ESC starts an escape sequence - long esc_timeout; // follow up delay for characters in an escape sequence - #if defined(_WIN32) - HANDLE hcon; // console input handle - DWORD hcon_orig_mode; // original console mode - #else - struct termios orig_ios; // original terminal settings - struct termios raw_ios; // raw terminal settings - #endif -}; - - -//------------------------------------------------------------- -// Forward declarations of platform dependent primitives below -//------------------------------------------------------------- - -ic_private bool tty_readc_noblock(tty_t* tty, uint8_t* c, long timeout_ms); // does not modify `c` when no input (false is returned) - -//------------------------------------------------------------- -// Key code helpers -//------------------------------------------------------------- - -ic_private bool code_is_ascii_char(code_t c, char* chr ) { - if (c >= ' ' && c <= 0x7F) { - if (chr != NULL) *chr = (char)c; - return true; - } - else { - if (chr != NULL) *chr = 0; - return false; - } -} - -ic_private bool code_is_unicode(code_t c, unicode_t* uchr) { - if (c <= KEY_UNICODE_MAX) { - if (uchr != NULL) *uchr = c; - return true; - } - else { - if (uchr != NULL) *uchr = 0; - return false; - } -} - -ic_private bool code_is_virt_key(code_t c ) { - return (KEY_NO_MODS(c) <= 0x20 || KEY_NO_MODS(c) >= KEY_VIRT); -} - - -//------------------------------------------------------------- -// Read a key code -//------------------------------------------------------------- -static code_t modify_code( code_t code ); - -static code_t tty_read_utf8( tty_t* tty, uint8_t c0 ) { - uint8_t buf[5]; - memset(buf, 0, 5); - - // try to read as many bytes as potentially needed - buf[0] = c0; - ssize_t count = 1; - if (c0 > 0x7F) { - if (tty_readc_noblock(tty, buf+count, tty->esc_timeout)) { - count++; - if (c0 > 0xDF) { - if (tty_readc_noblock(tty, buf+count, tty->esc_timeout)) { - count++; - if (c0 > 0xEF) { - if (tty_readc_noblock(tty, buf+count, tty->esc_timeout)) { - count++; - } - } - } - } - } - } - - buf[count] = 0; - debug_msg("tty: read utf8: count: %zd: %02x,%02x,%02x,%02x", count, buf[0], buf[1], buf[2], buf[3]); - - // decode the utf8 to unicode - ssize_t read = 0; - code_t code = key_unicode(unicode_from_qutf8(buf, count, &read)); - - // push back unused bytes (in the case of invalid utf8) - while (count > read) { - count--; - if (count >= 0 && count <= 4) { // to help the static analyzer - tty_cpush_char(tty, buf[count]); - } - } - return code; -} - -// pop a code from the pushback buffer. -static inline bool tty_code_pop(tty_t* tty, code_t* code); - - -// read a single char/key -ic_private bool tty_read_timeout(tty_t* tty, long timeout_ms, code_t* code) -{ - // is there a push_count back code? - if (tty_code_pop(tty,code)) { - return code; - } - - // read a single char/byte from a character stream - uint8_t c; - if (!tty_readc_noblock(tty, &c, timeout_ms)) return false; - - if (c == KEY_ESC) { - // escape sequence? - *code = tty_read_esc(tty, tty->esc_initial_timeout, tty->esc_timeout); - } - else if (c <= 0x7F) { - // ascii - *code = key_unicode(c); - } - else if (tty->is_utf8) { - // utf8 sequence - *code = tty_read_utf8(tty,c); - } - else { - // c >= 0x80 but tty is not utf8; use raw plane so we can translate it back in the end - *code = key_unicode( unicode_from_raw(c) ); - } - - *code = modify_code(*code); - return true; -} - -// Transform virtual keys to be more portable across platforms -static code_t modify_code( code_t code ) { - code_t key = KEY_NO_MODS(code); - code_t mods = KEY_MODS(code); - debug_msg( "tty: readc %s%s%s 0x%03x ('%c')\n", - mods&KEY_MOD_SHIFT ? "shift+" : "", mods&KEY_MOD_CTRL ? "ctrl+" : "", mods&KEY_MOD_ALT ? "alt+" : "", - key, (key >= ' ' && key <= '~' ? key : ' ')); - - // treat KEY_RUBOUT (0x7F) as KEY_BACKSP - if (key == KEY_RUBOUT) { - code = KEY_BACKSP | mods; - } - // ctrl+'_' is translated to '\x1F' on Linux, translate it back - else if (key == key_char('\x1F') && (mods & KEY_MOD_ALT) == 0) { - key = '_'; - code = WITH_CTRL(key_char('_')); - } - // treat ctrl/shift + enter always as KEY_LINEFEED for portability - else if (key == KEY_ENTER && (mods == KEY_MOD_SHIFT || mods == KEY_MOD_ALT || mods == KEY_MOD_CTRL)) { - code = KEY_LINEFEED; - } - // treat ctrl+tab always as shift+tab for portability - else if (code == WITH_CTRL(KEY_TAB)) { - code = KEY_SHIFT_TAB; - } - // treat ctrl+end/alt+>/alt-down and ctrl+home/alt+') || code == WITH_CTRL(KEY_END)) { - code = KEY_PAGEDOWN; - } - else if (code == WITH_ALT(KEY_UP) || code == WITH_ALT('<') || code == WITH_CTRL(KEY_HOME)) { - code = KEY_PAGEUP; - } - - // treat C0 codes without KEY_MOD_CTRL - if (key < ' ' && (mods&KEY_MOD_CTRL) != 0) { - code &= ~KEY_MOD_CTRL; - } - - return code; -} - - -// read a single char/key -ic_private code_t tty_read(tty_t* tty) -{ - code_t code; - if (!tty_read_timeout(tty, -1, &code)) return KEY_NONE; - return code; -} - -//------------------------------------------------------------- -// Read back an ANSI query response -//------------------------------------------------------------- - -ic_private bool tty_read_esc_response(tty_t* tty, char esc_start, bool final_st, char* buf, ssize_t buflen ) -{ - buf[0] = 0; - ssize_t len = 0; - uint8_t c = 0; - if (!tty_readc_noblock(tty, &c, 2*tty->esc_initial_timeout) || c != '\x1B') { - debug_msg("initial esc response failed: 0x%02x\n", c); - return false; - } - if (!tty_readc_noblock(tty, &c, tty->esc_timeout) || (c != esc_start)) return false; - while( len < buflen ) { - if (!tty_readc_noblock(tty, &c, tty->esc_timeout)) return false; - if (final_st) { - // OSC is terminated by BELL, or ESC \ (ST) (and STX) - if (c=='\x07' || c=='\x02') { - break; - } - else if (c=='\x1B') { - uint8_t c1; - if (!tty_readc_noblock(tty, &c1, tty->esc_timeout)) return false; - if (c1=='\\') break; - tty_cpush_char(tty,c1); - } - } - else { - if (c == '\x02') { // STX - break; - } - else if (!((c >= '0' && c <= '9') || strchr("<=>?;:",c) != NULL)) { - buf[len++] = (char)c; // for non-OSC save the terminating character - break; - } - } - buf[len++] = (char)c; - } - buf[len] = 0; - debug_msg("tty: escape query response: %s\n", buf); - return true; -} - -//------------------------------------------------------------- -// High level code pushback -//------------------------------------------------------------- - -static inline bool tty_code_pop( tty_t* tty, code_t* code ) { - if (tty->push_count <= 0) return false; - tty->push_count--; - *code = tty->pushbuf[tty->push_count]; - return true; -} - -ic_private void tty_code_pushback( tty_t* tty, code_t c ) { - // note: must be signal safe - if (tty->push_count >= TTY_PUSH_MAX) return; - tty->pushbuf[tty->push_count] = c; - tty->push_count++; -} - - -//------------------------------------------------------------- -// low-level character pushback (for escape sequences and windows) -//------------------------------------------------------------- - -ic_private bool tty_cpop(tty_t* tty, uint8_t* c) { - if (tty->cpush_count <= 0) { // do not modify c on failure (see `tty_decode_unicode`) - return false; - } - else { - tty->cpush_count--; - *c = tty->cpushbuf[tty->cpush_count]; - return true; - } -} - -static inline void tty_cpush(tty_t* tty, const char* s) { - ssize_t len = ic_strlen(s); - if (tty->push_count + len > TTY_PUSH_MAX) { - debug_msg("tty: cpush buffer full! (pushing %s)\n", s); - assert(false); - return; - } - for (ssize_t i = 0; i < len; i++) { - tty->cpushbuf[tty->cpush_count + i] = (uint8_t)( s[len - i - 1] ); - } - tty->cpush_count += len; - return; -} - -// convenience function for small sequences -static inline void tty_cpushf(tty_t* tty, const char* fmt, ...) { - va_list args; - va_start(args,fmt); - char buf[TTY_PUSH_MAX+1]; - vsnprintf(buf,TTY_PUSH_MAX,fmt,args); - buf[TTY_PUSH_MAX] = 0; - tty_cpush(tty,buf); - va_end(args); - return; -} - -ic_private void tty_cpush_char(tty_t* tty, uint8_t c) { - uint8_t buf[2]; - buf[0] = c; - buf[1] = 0; - tty_cpush(tty, (const char*)buf); -} - - -//------------------------------------------------------------- -// Push escape codes (used on Windows to insert keys) -//------------------------------------------------------------- - -static unsigned csi_mods(code_t mods) { - unsigned m = 1; - if (mods&KEY_MOD_SHIFT) m += 1; - if (mods&KEY_MOD_ALT) m += 2; - if (mods&KEY_MOD_CTRL) m += 4; - return m; -} - -// Push ESC [ ; ~ -static inline void tty_cpush_csi_vt( tty_t* tty, code_t mods, uint32_t vtcode ) { - tty_cpushf(tty,"\x1B[%u;%u~", vtcode, csi_mods(mods) ); -} - -// push ESC [ 1 ; -static inline void tty_cpush_csi_xterm( tty_t* tty, code_t mods, char xcode ) { - tty_cpushf(tty,"\x1B[1;%u%c", csi_mods(mods), xcode ); -} - -// push ESC [ ; u -static inline void tty_cpush_csi_unicode( tty_t* tty, code_t mods, uint32_t unicode ) { - if ((unicode < 0x80 && mods == 0) || - (mods == KEY_MOD_CTRL && unicode < ' ' && unicode != KEY_TAB && unicode != KEY_ENTER - && unicode != KEY_LINEFEED && unicode != KEY_BACKSP) || - (mods == KEY_MOD_SHIFT && unicode >= ' ' && unicode <= KEY_RUBOUT)) { - tty_cpush_char(tty,(uint8_t)unicode); - } - else { - tty_cpushf(tty,"\x1B[%u;%uu", unicode, csi_mods(mods) ); - } -} - -//------------------------------------------------------------- -// Init -//------------------------------------------------------------- - -static inline bool tty_init_raw(tty_t* tty); -static inline void tty_done_raw(tty_t* tty); - -static inline bool tty_init_utf8(tty_t* tty) { - #ifdef _WIN32 - tty->is_utf8 = true; - #else - const char* loc = setlocale(LC_ALL,""); - tty->is_utf8 = (ic_icontains(loc,"UTF-8") || ic_icontains(loc,"utf8") || ic_stricmp(loc,"C") == 0); - debug_msg("tty: utf8: %s (loc=%s)\n", tty->is_utf8 ? "true" : "false", loc); - #endif - return true; -} - -ic_private tty_t* tty_new(alloc_t* mem, int fd_in) -{ - tty_t* tty = mem_zalloc_tp(mem, tty_t); - tty->mem = mem; - tty->fd_in = (fd_in < 0 ? STDIN_FILENO : fd_in); - #if defined(__APPLE__) - tty->esc_initial_timeout = 200; // apple use ESC+ for alt- - #else - tty->esc_initial_timeout = 100; - #endif - tty->esc_timeout = 10; - if (!(isatty(tty->fd_in) && tty_init_raw(tty) && tty_init_utf8(tty))) { - tty_free(tty); - return NULL; - } - return tty; -} - -ic_private void tty_free(tty_t* tty) { - if (tty==NULL) return; - tty_end_raw(tty); - tty_done_raw(tty); - mem_free(tty->mem,tty); -} - -ic_private bool tty_is_utf8(const tty_t* tty) { - if (tty == NULL) return true; - return (tty->is_utf8); -} - -ic_private bool tty_term_resize_event(tty_t* tty) { - if (tty == NULL) return true; - if (tty->has_term_resize_event) { - if (!tty->term_resize_event) return false; - tty->term_resize_event = false; // reset. - } - return true; // always return true on systems without a resize event (more expensive but still ok) -} - -ic_private void tty_set_esc_delay(tty_t* tty, long initial_delay_ms, long followup_delay_ms) { - tty->esc_initial_timeout = (initial_delay_ms < 0 ? 0 : (initial_delay_ms > 1000 ? 1000 : initial_delay_ms)); - tty->esc_timeout = (followup_delay_ms < 0 ? 0 : (followup_delay_ms > 1000 ? 1000 : followup_delay_ms)); -} - -//------------------------------------------------------------- -// Unix -//------------------------------------------------------------- -#if !defined(_WIN32) - -static inline bool tty_readc_blocking(tty_t* tty, uint8_t* c) { - if (tty_cpop(tty,c)) return true; - *c = 0; - ssize_t nread = read(tty->fd_in, (char*)c, 1); - if (nread < 0 && errno == EINTR) { - // can happen on SIGWINCH signal for terminal resize - } - return (nread == 1); -} - - -// non blocking read -- with a small timeout used for reading escape sequences. -ic_private bool tty_readc_noblock(tty_t* tty, uint8_t* c, long timeout_ms) -{ - // in our pushback buffer? - if (tty_cpop(tty, c)) return true; - - // blocking read? - if (timeout_ms < 0) { - return tty_readc_blocking(tty,c); - } - - // if supported, peek first if any char is available. - #if defined(FIONREAD) - { int navail = 0; - if (ioctl(0, FIONREAD, &navail) == 0) { - if (navail >= 1) { - return tty_readc_blocking(tty, c); - } - else if (timeout_ms == 0) { - return false; // return early if there is no input available (with a zero timeout) - } - } - } - #endif - - // otherwise block for at most timeout milliseconds - #if defined(FD_SET) - // we can use select to detect when input becomes available - fd_set readset; - struct timeval time; - FD_ZERO(&readset); - FD_SET(tty->fd_in, &readset); - time.tv_sec = (timeout_ms > 0 ? timeout_ms / 1000 : 0); - time.tv_usec = (timeout_ms > 0 ? 1000*(timeout_ms % 1000) : 0); - if (select(tty->fd_in + 1, &readset, NULL, NULL, &time) == 1) { - // input available - return tty_readc_blocking(tty, c); - } - #else - // no select, we cannot timeout; use usleeps :-( - // todo: this seems very rare nowadays; should be even support this? - do { - // peek ahead if possible - #if defined(FIONREAD) - int navail = 0; - if (ioctl(0, FIONREAD, &navail) == 0 && navail >= 1) { - return tty_readc_blocking(tty, c); - } - #elif defined(O_NONBLOCK) - // use a temporary non-blocking read mode - int fstatus = fcntl(tty->fd_in, F_GETFL, 0); - if (fstatus != -1) { - if (fcntl(tty->fd_in, F_SETFL, (fstatus | O_NONBLOCK)) != -1) { - char buf[2] = { 0, 0 }; - ssize_t nread = read(tty->fd_in, buf, 1); - fcntl(tty->fd_in, F_SETFL, fstatus); - if (nread >= 1) { - *c = (uint8_t)buf[0]; - return true; - } - } - } - #else - #error "define an nonblocking read for this platform" - #endif - // and sleep a bit - if (timeout_ms > 0) { - usleep(50*1000L); // sleep at most 0.05s at a time - timeout_ms -= 100; - if (timeout_ms < 0) { timeout_ms = 0; } - } - } - while (timeout_ms > 0); - #endif - return false; -} - -#if defined(TIOCSTI) -ic_private bool tty_async_stop(const tty_t* tty) { - // insert ^C in the input stream - char c = KEY_CTRL_C; - return (ioctl(tty->fd_in, TIOCSTI, &c) >= 0); -} -#else -ic_private bool tty_async_stop(const tty_t* tty) { - return false; -} -#endif - -// We install various signal handlers to restore the terminal settings -// in case of a terminating signal. This is also used to catch terminal window resizes. -// This is not strictly needed so this can be disabled on -// (older) platforms that do not support signal handling well. -#if defined(SIGWINCH) && defined(SA_RESTART) // ensure basic signal functionality is defined - -// store the tty in a global so we access it on unexpected termination -static tty_t* sig_tty; // = NULL - -// Catch all termination signals (and SIGWINCH) -typedef struct signal_handler_s { - int signum; - union { - int _avoid_warning; - struct sigaction previous; - } action; -} signal_handler_t; - -static signal_handler_t sighandlers[] = { - { SIGWINCH, {0} }, - { SIGTERM , {0} }, - { SIGINT , {0} }, - { SIGQUIT , {0} }, - { SIGHUP , {0} }, - { SIGSEGV , {0} }, - { SIGTRAP , {0} }, - { SIGBUS , {0} }, - { SIGTSTP , {0} }, - { SIGTTIN , {0} }, - { SIGTTOU , {0} }, - { 0 , {0} } -}; - -static inline bool sigaction_is_valid( struct sigaction* sa ) { - return (sa->sa_sigaction != NULL && sa->sa_handler != SIG_DFL && sa->sa_handler != SIG_IGN); -} - -// Generic signal handler -static inline void sig_handler(int signum, siginfo_t* siginfo, void* uap ) { - if (signum == SIGWINCH) { - if (sig_tty != NULL) { - sig_tty->term_resize_event = true; - } - } - else { - // the rest are termination signals; restore the terminal mode. (`tcsetattr` is signal-safe) - if (sig_tty != NULL && sig_tty->raw_enabled) { - tcsetattr(sig_tty->fd_in, TCSAFLUSH, &sig_tty->orig_ios); - sig_tty->raw_enabled = false; - } - } - // call previous handler - signal_handler_t* sh = sighandlers; - while( sh->signum != 0 && sh->signum != signum) { sh++; } - if (sh->signum == signum) { - if (sigaction_is_valid(&sh->action.previous)) { - (sh->action.previous.sa_sigaction)(signum, siginfo, uap); - } - } -} - -static inline void signals_install(tty_t* tty) { - sig_tty = tty; - // generic signal handler - struct sigaction handler; - memset(&handler,0,sizeof(handler)); - sigemptyset(&handler.sa_mask); - handler.sa_sigaction = &sig_handler; - handler.sa_flags = SA_RESTART; - // install for all signals - for( signal_handler_t* sh = sighandlers; sh->signum != 0; sh++ ) { - if (sigaction( sh->signum, NULL, &sh->action.previous) == 0) { // get previous - if (sh->action.previous.sa_handler != SIG_IGN) { // if not to be ignored - if (sigaction( sh->signum, &handler, &sh->action.previous ) < 0) { // install our handler - sh->action.previous.sa_sigaction = NULL; // do not restore on error - } - else if (sh->signum == SIGWINCH) { - sig_tty->has_term_resize_event = true; - }; - } - } - } -} - -static inline void signals_restore(void) { - // restore all signal handlers - for( signal_handler_t* sh = sighandlers; sh->signum != 0; sh++ ) { - if (sigaction_is_valid(&sh->action.previous)) { - sigaction( sh->signum, &sh->action.previous, NULL ); - }; - } - sig_tty = NULL; -} - -#else -static inline void signals_install(tty_t* tty) { - ic_unused(tty); - // nothing -} -static inline void signals_restore(void) { - // nothing -} - -#endif - -ic_private bool tty_start_raw(tty_t* tty) { - if (tty == NULL) return false; - if (tty->raw_enabled) return true; - if (tcsetattr(tty->fd_in,TCSAFLUSH,&tty->raw_ios) < 0) return false; - tty->raw_enabled = true; - return true; -} - -ic_private void tty_end_raw(tty_t* tty) { - if (tty == NULL) return; - if (!tty->raw_enabled) return; - tty->cpush_count = 0; - if (tcsetattr(tty->fd_in,TCSAFLUSH,&tty->orig_ios) < 0) return; - tty->raw_enabled = false; -} - -static inline bool tty_init_raw(tty_t* tty) -{ - // Set input to raw mode. See . - if (tcgetattr(tty->fd_in,&tty->orig_ios) == -1) return false; - tty->raw_ios = tty->orig_ios; - // input: no break signal, no \r to \n, no parity check, no 8-bit to 7-bit, no flow control - tty->raw_ios.c_iflag &= ~(unsigned long)(BRKINT | ICRNL | INPCK | ISTRIP | IXON); - // control: allow 8-bit - tty->raw_ios.c_cflag |= CS8; - // local: no echo, no line-by-line (canonical), no extended input processing, no signals for ^z,^c - tty->raw_ios.c_lflag &= ~(unsigned long)(ECHO | ICANON | IEXTEN | ISIG); - // 1 byte at a time, no delay - tty->raw_ios.c_cc[VTIME] = 0; - tty->raw_ios.c_cc[VMIN] = 1; - - // store in global so our signal handlers can restore the terminal mode - signals_install(tty); - - return true; -} - -static inline void tty_done_raw(tty_t* tty) { - ic_unused(tty); - signals_restore(); -} - - -#else - -//------------------------------------------------------------- -// Windows -// For best portability we push CSI escape sequences directly -// to the character stream (instead of returning key codes). -//------------------------------------------------------------- - -static inline void tty_waitc_console(tty_t* tty, long timeout_ms); - -ic_private bool tty_readc_noblock(tty_t* tty, uint8_t* c, long timeout_ms) { // don't modify `c` if there is no input - // in our pushback buffer? - if (tty_cpop(tty, c)) return true; - // any events in the input queue? - tty_waitc_console(tty, timeout_ms); - return tty_cpop(tty, c); -} - -// Read from the console input events and push escape codes into the tty cbuffer. -static inline void tty_waitc_console(tty_t* tty, long timeout_ms) -{ - // wait for a key down event - INPUT_RECORD inp; - DWORD count; - uint32_t surrogate_hi = 0; - while (true) { - // check if there are events if in non-blocking timeout mode - if (timeout_ms >= 0) { - // first peek ahead - if (!GetNumberOfConsoleInputEvents(tty->hcon, &count)) return; - if (count == 0) { - if (timeout_ms == 0) { - // out of time - return; - } - else { - // wait for input events for at most timeout milli seconds - ULONGLONG start_ms = GetTickCount64(); - DWORD res = WaitForSingleObject(tty->hcon, (DWORD)timeout_ms); - switch (res) { - case WAIT_OBJECT_0: { - // input is available, decrease our timeout - ULONGLONG waited_ms = (GetTickCount64() - start_ms); - timeout_ms -= (long)waited_ms; - if (timeout_ms < 0) { - timeout_ms = 0; - } - break; - } - case WAIT_TIMEOUT: - case WAIT_ABANDONED: - case WAIT_FAILED: - default: - return; - } - } - } - } - - // (blocking) Read from the input - if (!ReadConsoleInputW(tty->hcon, &inp, 1, &count)) return; - if (count != 1) return; - - // resize event? - if (inp.EventType == WINDOW_BUFFER_SIZE_EVENT) { - tty->term_resize_event = true; - continue; - } - - // wait for key down events - if (inp.EventType != KEY_EVENT) continue; - - // the modifier state - DWORD modstate = inp.Event.KeyEvent.dwControlKeyState; - - // we need to handle shift up events separately - if (!inp.Event.KeyEvent.bKeyDown && inp.Event.KeyEvent.wVirtualKeyCode == VK_SHIFT) { - modstate &= (DWORD)~SHIFT_PRESSED; - } - - // ignore AltGr - DWORD altgr = LEFT_CTRL_PRESSED | RIGHT_ALT_PRESSED; - if ((modstate & altgr) == altgr) { modstate &= ~altgr; } - - - // get modifiers - code_t mods = 0; - if ((modstate & ( RIGHT_CTRL_PRESSED | LEFT_CTRL_PRESSED )) != 0) mods |= KEY_MOD_CTRL; - if ((modstate & ( RIGHT_ALT_PRESSED | LEFT_ALT_PRESSED )) != 0) mods |= KEY_MOD_ALT; - if ((modstate & SHIFT_PRESSED) != 0) mods |= KEY_MOD_SHIFT; - - // virtual keys - uint32_t chr = (uint32_t)inp.Event.KeyEvent.uChar.UnicodeChar; - WORD virt = inp.Event.KeyEvent.wVirtualKeyCode; - debug_msg("tty: console %s: %s%s%s virt 0x%04x, chr 0x%04x ('%c')\n", inp.Event.KeyEvent.bKeyDown ? "down" : "up", mods&KEY_MOD_CTRL ? "ctrl-" : "", mods&KEY_MOD_ALT ? "alt-" : "", mods&KEY_MOD_SHIFT ? "shift-" : "", virt, chr, chr); - - // only process keydown events (except for Alt-up which is used for unicode pasting...) - if (!inp.Event.KeyEvent.bKeyDown && virt != VK_MENU) { - continue; - } - - if (chr == 0) { - switch (virt) { - case VK_UP: tty_cpush_csi_xterm(tty, mods, 'A'); return; - case VK_DOWN: tty_cpush_csi_xterm(tty, mods, 'B'); return; - case VK_RIGHT: tty_cpush_csi_xterm(tty, mods, 'C'); return; - case VK_LEFT: tty_cpush_csi_xterm(tty, mods, 'D'); return; - case VK_END: tty_cpush_csi_xterm(tty, mods, 'F'); return; - case VK_HOME: tty_cpush_csi_xterm(tty, mods, 'H'); return; - case VK_DELETE: tty_cpush_csi_vt(tty,mods,3); return; - case VK_PRIOR: tty_cpush_csi_vt(tty,mods,5); return; //page up - case VK_NEXT: tty_cpush_csi_vt(tty,mods,6); return; //page down - case VK_TAB: tty_cpush_csi_unicode(tty,mods,9); return; - case VK_RETURN: tty_cpush_csi_unicode(tty,mods,13); return; - default: { - uint32_t vtcode = 0; - if (virt >= VK_F1 && virt <= VK_F5) { - vtcode = 10 + (virt - VK_F1); - } - else if (virt >= VK_F6 && virt <= VK_F10) { - vtcode = 17 + (virt - VK_F6); - } - else if (virt >= VK_F11 && virt <= VK_F12) { - vtcode = 13 + (virt - VK_F11); - } - if (vtcode > 0) { - tty_cpush_csi_vt(tty,mods,vtcode); - return; - } - } - } - // ignore other control keys (shift etc). - } - // high surrogate pair - else if (chr >= 0xD800 && chr <= 0xDBFF) { - surrogate_hi = (chr - 0xD800); - } - // low surrogate pair - else if (chr >= 0xDC00 && chr <= 0xDFFF) { - chr = ((surrogate_hi << 10) + (chr - 0xDC00) + 0x10000); - tty_cpush_csi_unicode(tty,mods,chr); - surrogate_hi = 0; - return; - } - // regular character - else { - tty_cpush_csi_unicode(tty,mods,chr); - return; - } - } -} - -ic_private bool tty_async_stop(const tty_t* tty) { - // send ^c - INPUT_RECORD events[2]; - memset(events, 0, 2*sizeof(INPUT_RECORD)); - events[0].EventType = KEY_EVENT; - events[0].Event.KeyEvent.bKeyDown = TRUE; - events[0].Event.KeyEvent.uChar.AsciiChar = KEY_CTRL_C; - events[1] = events[0]; - events[1].Event.KeyEvent.bKeyDown = FALSE; - DWORD nwritten = 0; - WriteConsoleInput(tty->hcon, events, 2, &nwritten); - return (nwritten == 2); -} - -ic_private bool tty_start_raw(tty_t* tty) { - if (tty->raw_enabled) return true; - GetConsoleMode(tty->hcon,&tty->hcon_orig_mode); - DWORD mode = ENABLE_QUICK_EDIT_MODE // cut&paste allowed - | ENABLE_WINDOW_INPUT // to catch resize events - // | ENABLE_VIRTUAL_TERMINAL_INPUT - // | ENABLE_PROCESSED_INPUT - ; - SetConsoleMode(tty->hcon, mode ); - tty->raw_enabled = true; - return true; -} - -ic_private void tty_end_raw(tty_t* tty) { - if (!tty->raw_enabled) return; - SetConsoleMode(tty->hcon, tty->hcon_orig_mode ); - tty->raw_enabled = false; -} - -static inline bool tty_init_raw(tty_t* tty) { - tty->hcon = GetStdHandle( STD_INPUT_HANDLE ); - tty->has_term_resize_event = true; - return true; -} - -static inline void tty_done_raw(tty_t* tty) { - ic_unused(tty); -} - -#endif - - diff --git a/vendor/isocline/tty.h b/vendor/isocline/tty.h deleted file mode 100644 index a0062bf3..00000000 --- a/vendor/isocline/tty.h +++ /dev/null @@ -1,160 +0,0 @@ -/* ---------------------------------------------------------------------------- - Copyright (c) 2021, Daan Leijen - This is free software; you can redistribute it and/or modify it - under the terms of the MIT License. A copy of the license can be - found in the "LICENSE" file at the root of this distribution. ------------------------------------------------------------------------------*/ -#pragma once -#ifndef IC_TTY_H -#define IC_TTY_H - -#include "common.h" - -//------------------------------------------------------------- -// TTY/Keyboard input -//------------------------------------------------------------- - -// Key code -typedef uint32_t code_t; - -// TTY interface -struct tty_s; -typedef struct tty_s tty_t; - - -ic_private tty_t* tty_new(alloc_t* mem, int fd_in); -ic_private void tty_free(tty_t* tty); - -ic_private bool tty_is_utf8(const tty_t* tty); -ic_private bool tty_start_raw(tty_t* tty); -ic_private void tty_end_raw(tty_t* tty); -ic_private code_t tty_read(tty_t* tty); -ic_private bool tty_read_timeout(tty_t* tty, long timeout_ms, code_t* c ); - -ic_private void tty_code_pushback( tty_t* tty, code_t c ); -ic_private bool code_is_ascii_char(code_t c, char* chr ); -ic_private bool code_is_unicode(code_t c, unicode_t* uchr); -ic_private bool code_is_virt_key(code_t c ); - -ic_private bool tty_term_resize_event(tty_t* tty); // did the terminal resize? -ic_private bool tty_async_stop(const tty_t* tty); // unblock the read asynchronously -ic_private void tty_set_esc_delay(tty_t* tty, long initial_delay_ms, long followup_delay_ms); - -// shared between tty.c and tty_esc.c: low level character push -ic_private void tty_cpush_char(tty_t* tty, uint8_t c); -ic_private bool tty_cpop(tty_t* tty, uint8_t* c); -ic_private bool tty_readc_noblock(tty_t* tty, uint8_t* c, long timeout_ms); -ic_private code_t tty_read_esc(tty_t* tty, long esc_initial_timeout, long esc_timeout); // in tty_esc.c - -// used by term.c to read back ANSI escape responses -ic_private bool tty_read_esc_response(tty_t* tty, char esc_start, bool final_st, char* buf, ssize_t buflen ); - - -//------------------------------------------------------------- -// Key codes: a code_t is 32 bits. -// we use the bottom 24 (nah, 21) bits for unicode (up to x0010FFFF) -// The codes after x01000000 are for virtual keys -// and events use x02000000. -// The top 4 bits are used for modifiers. -//------------------------------------------------------------- - -static inline code_t key_char( char c ) { - // careful about signed character conversion (negative char ~> 0x80 - 0xFF) - return ((uint8_t)c); -} - -static inline code_t key_unicode( unicode_t u ) { - return u; -} - - -#define KEY_MOD_SHIFT (0x10000000U) -#define KEY_MOD_ALT (0x20000000U) -#define KEY_MOD_CTRL (0x40000000U) - -#define KEY_NO_MODS(k) (k & 0x0FFFFFFFU) -#define KEY_MODS(k) (k & 0xF0000000U) - -#define WITH_SHIFT(x) (x | KEY_MOD_SHIFT) -#define WITH_ALT(x) (x | KEY_MOD_ALT) -#define WITH_CTRL(x) (x | KEY_MOD_CTRL) - -#define KEY_NONE (0) -#define KEY_CTRL_A (1) -#define KEY_CTRL_B (2) -#define KEY_CTRL_C (3) -#define KEY_CTRL_D (4) -#define KEY_CTRL_E (5) -#define KEY_CTRL_F (6) -#define KEY_BELL (7) -#define KEY_BACKSP (8) -#define KEY_TAB (9) -#define KEY_LINEFEED (10) // ctrl/shift + enter is considered KEY_LINEFEED -#define KEY_CTRL_K (11) -#define KEY_CTRL_L (12) -#define KEY_ENTER (13) -#define KEY_CTRL_N (14) -#define KEY_CTRL_O (15) -#define KEY_CTRL_P (16) -#define KEY_CTRL_Q (17) -#define KEY_CTRL_R (18) -#define KEY_CTRL_S (19) -#define KEY_CTRL_T (20) -#define KEY_CTRL_U (21) -#define KEY_CTRL_V (22) -#define KEY_CTRL_W (23) -#define KEY_CTRL_X (24) -#define KEY_CTRL_Y (25) -#define KEY_CTRL_Z (26) -#define KEY_ESC (27) -#define KEY_SPACE (32) -#define KEY_RUBOUT (127) // always translated to KEY_BACKSP -#define KEY_UNICODE_MAX (0x0010FFFFU) - - -#define KEY_VIRT (0x01000000U) -#define KEY_UP (KEY_VIRT+0) -#define KEY_DOWN (KEY_VIRT+1) -#define KEY_LEFT (KEY_VIRT+2) -#define KEY_RIGHT (KEY_VIRT+3) -#define KEY_HOME (KEY_VIRT+4) -#define KEY_END (KEY_VIRT+5) -#define KEY_DEL (KEY_VIRT+6) -#define KEY_PAGEUP (KEY_VIRT+7) -#define KEY_PAGEDOWN (KEY_VIRT+8) -#define KEY_INS (KEY_VIRT+9) - -#define KEY_F1 (KEY_VIRT+11) -#define KEY_F2 (KEY_VIRT+12) -#define KEY_F3 (KEY_VIRT+13) -#define KEY_F4 (KEY_VIRT+14) -#define KEY_F5 (KEY_VIRT+15) -#define KEY_F6 (KEY_VIRT+16) -#define KEY_F7 (KEY_VIRT+17) -#define KEY_F8 (KEY_VIRT+18) -#define KEY_F9 (KEY_VIRT+19) -#define KEY_F10 (KEY_VIRT+20) -#define KEY_F11 (KEY_VIRT+21) -#define KEY_F12 (KEY_VIRT+22) -#define KEY_F(n) (KEY_F1 + (n) - 1) - -#define KEY_EVENT_BASE (0x02000000U) -#define KEY_EVENT_RESIZE (KEY_EVENT_BASE+1) -#define KEY_EVENT_AUTOTAB (KEY_EVENT_BASE+2) -#define KEY_EVENT_STOP (KEY_EVENT_BASE+3) - -// Convenience -#define KEY_CTRL_UP (WITH_CTRL(KEY_UP)) -#define KEY_CTRL_DOWN (WITH_CTRL(KEY_DOWN)) -#define KEY_CTRL_LEFT (WITH_CTRL(KEY_LEFT)) -#define KEY_CTRL_RIGHT (WITH_CTRL(KEY_RIGHT)) -#define KEY_CTRL_HOME (WITH_CTRL(KEY_HOME)) -#define KEY_CTRL_END (WITH_CTRL(KEY_END)) -#define KEY_CTRL_DEL (WITH_CTRL(KEY_DEL)) -#define KEY_CTRL_PAGEUP (WITH_CTRL(KEY_PAGEUP)) -#define KEY_CTRL_PAGEDOWN (WITH_CTRL(KEY_PAGEDOWN))) -#define KEY_CTRL_INS (WITH_CTRL(KEY_INS)) - -#define KEY_SHIFT_TAB (WITH_SHIFT(KEY_TAB)) - -#endif // IC_TTY_H diff --git a/vendor/isocline/tty_esc.c b/vendor/isocline/tty_esc.c deleted file mode 100644 index 0ac8761d..00000000 --- a/vendor/isocline/tty_esc.c +++ /dev/null @@ -1,401 +0,0 @@ -/* ---------------------------------------------------------------------------- - Copyright (c) 2021, Daan Leijen - This is free software; you can redistribute it and/or modify it - under the terms of the MIT License. A copy of the license can be - found in the "LICENSE" file at the root of this distribution. ------------------------------------------------------------------------------*/ -#include -#include "tty.h" - -/*------------------------------------------------------------- -Decoding escape sequences to key codes. -This is a bit tricky there are many variants to encode keys as escape sequences, see for example: -- . -- -- -- -- - -Generally, for our purposes we accept a subset of escape sequences as: - - escseq ::= ESC - | ESC char - | ESC start special? (number (';' modifiers)?)? final - -where: - char ::= [\x00-\xFF] # any character - special ::= [:<=>?] - number ::= [0-9+] - modifiers ::= [1-9] - intermediate ::= [\x20-\x2F] # !"#$%&'()*+,-./ - final ::= [\x40-\x7F] # @A–Z[\]^_`a–z{|}~ - ESC ::= '\x1B' - CSI ::= ESC '[' - SS3 ::= ESC 'O' - -In ECMA48 `special? (number (';' modifiers)?)?` is the more liberal `[\x30-\x3F]*` -but that seems never used for key codes. If the number (vtcode or unicode) or the -modifiers are not given, we assume these are '1'. -We then accept the following key sequences: - - key ::= ESC # lone ESC - | ESC char # Alt+char - | ESC '[' special? vtcode ';' modifiers '~' # vt100 codes - | ESC '[' special? '1' ';' modifiers [A-Z] # xterm codes - | ESC 'O' special? '1' ';' modifiers [A-Za-z] # SS3 codes - | ESC '[' special? unicode ';' modifiers 'u' # direct unicode code - -Moreover, we translate the following special cases that do not fit into the above grammar. -First we translate away special starter sequences: ---------------------------------------------------------------------- - ESC '[' '[' .. ~> ESC '[' .. # Linux sometimes uses extra '[' for CSI - ESC '[' 'O' .. ~> ESC 'O' .. # Linux sometimes uses extra '[' for SS3 - ESC 'o' .. ~> ESC 'O' .. # Eterm: ctrl + SS3 - ESC '?' .. ~> ESC 'O' .. # vt52 treated as SS3 - -And then translate the following special cases into a standard form: ---------------------------------------------------------------------- - ESC '[' .. '@' ~> ESC '[' '3' '~' # Del on Mach - ESC '[' .. '9' ~> ESC '[' '2' '~' # Ins on Mach - ESC .. [^@$] ~> ESC .. '~' # ETerm,xrvt,urxt: ^ = ctrl, $ = shift, @ = alt - ESC '[' [a-d] ~> ESC '[' '1' ';' '2' [A-D] # Eterm shift+ - ESC 'O' [1-9] final ~> ESC 'O' '1' ';' [1-9] final # modifiers as parameter 1 (like on Haiku) - ESC '[' [1-9] [^~u] ~> ESC 'O' '1' ';' [1-9] final # modifiers as parameter 1 - -The modifier keys are encoded as "(modifiers-1) & mask" where the -shift mask is 0x01, alt 0x02 and ctrl 0x04. Therefore: ------------------------------------------------------------- - 1: - 5: ctrl 9: alt (for minicom) - 2: shift 6: shift+ctrl - 3: alt 7: alt+ctrl - 4: shift+alt 8: shift+alt+ctrl - -The different encodings fox vt100, xterm, and SS3 are: - -vt100: ESC [ vtcode ';' modifiers '~' --------------------------------------- - 1: Home 10-15: F1-F5 - 2: Ins 16 : F5 - 3: Del 17-21: F6-F10 - 4: End 23-26: F11-F14 - 5: PageUp 28 : F15 - 6: PageDn 29 : F16 - 7: Home 31-34: F17-F20 - 8: End - -xterm: ESC [ 1 ';' modifiers [A-Z] ------------------------------------ - A: Up N: F2 - B: Down O: F3 - C: Right P: F4 - D: Left Q: F5 - E: '5' R: F6 - F: End S: F7 - G: T: F8 - H: Home U: PageDn - I: PageUp V: PageUp - J: W: F11 - K: X: F12 - L: Ins Y: End - M: F1 Z: shift+Tab - -SS3: ESC 'O' 1 ';' modifiers [A-Za-z] ---------------------------------------- - (normal) (numpad) - A: Up N: a: Up n: - B: Down O: b: Down o: - C: Right P: F1 c: Right p: Ins - D: Left Q: F2 d: Left q: End - E: '5' R: F3 e: r: Down - F: End S: F4 f: s: PageDn - G: T: F5 g: t: Left - H: Home U: F6 h: u: '5' - I: Tab V: F7 i: v: Right - J: W: F8 j: '*' w: Home - K: X: F9 k: '+' x: Up - L: Y: F10 l: ',' y: PageUp - M: \x0A '\n' Z: shift+Tab m: '-' z: - --------------------------------------------------------------*/ - -//------------------------------------------------------------- -// Decode escape sequences -//------------------------------------------------------------- - -static code_t esc_decode_vt(uint32_t vt_code ) { - switch(vt_code) { - case 1: return KEY_HOME; - case 2: return KEY_INS; - case 3: return KEY_DEL; - case 4: return KEY_END; - case 5: return KEY_PAGEUP; - case 6: return KEY_PAGEDOWN; - case 7: return KEY_HOME; - case 8: return KEY_END; - default: - if (vt_code >= 10 && vt_code <= 15) return KEY_F(1 + (vt_code - 10)); - if (vt_code == 16) return KEY_F5; // minicom - if (vt_code >= 17 && vt_code <= 21) return KEY_F(6 + (vt_code - 17)); - if (vt_code >= 23 && vt_code <= 26) return KEY_F(11 + (vt_code - 23)); - if (vt_code >= 28 && vt_code <= 29) return KEY_F(15 + (vt_code - 28)); - if (vt_code >= 31 && vt_code <= 34) return KEY_F(17 + (vt_code - 31)); - } - return KEY_NONE; -} - -static code_t esc_decode_xterm( uint8_t xcode ) { - // ESC [ - switch(xcode) { - case 'A': return KEY_UP; - case 'B': return KEY_DOWN; - case 'C': return KEY_RIGHT; - case 'D': return KEY_LEFT; - case 'E': return '5'; // numpad 5 - case 'F': return KEY_END; - case 'H': return KEY_HOME; - case 'Z': return KEY_TAB | KEY_MOD_SHIFT; - // Freebsd: - case 'I': return KEY_PAGEUP; - case 'L': return KEY_INS; - case 'M': return KEY_F1; - case 'N': return KEY_F2; - case 'O': return KEY_F3; - case 'P': return KEY_F4; // note: differs from - case 'Q': return KEY_F5; - case 'R': return KEY_F6; - case 'S': return KEY_F7; - case 'T': return KEY_F8; - case 'U': return KEY_PAGEDOWN; // Mach - case 'V': return KEY_PAGEUP; // Mach - case 'W': return KEY_F11; - case 'X': return KEY_F12; - case 'Y': return KEY_END; // Mach - } - return KEY_NONE; -} - -static code_t esc_decode_ss3( uint8_t ss3_code ) { - // ESC O - switch(ss3_code) { - case 'A': return KEY_UP; - case 'B': return KEY_DOWN; - case 'C': return KEY_RIGHT; - case 'D': return KEY_LEFT; - case 'E': return '5'; // numpad 5 - case 'F': return KEY_END; - case 'H': return KEY_HOME; - case 'I': return KEY_TAB; - case 'Z': return KEY_TAB | KEY_MOD_SHIFT; - case 'M': return KEY_LINEFEED; - case 'P': return KEY_F1; - case 'Q': return KEY_F2; - case 'R': return KEY_F3; - case 'S': return KEY_F4; - // on Mach - case 'T': return KEY_F5; - case 'U': return KEY_F6; - case 'V': return KEY_F7; - case 'W': return KEY_F8; - case 'X': return KEY_F9; // '=' on vt220 - case 'Y': return KEY_F10; - // numpad - case 'a': return KEY_UP; - case 'b': return KEY_DOWN; - case 'c': return KEY_RIGHT; - case 'd': return KEY_LEFT; - case 'j': return '*'; - case 'k': return '+'; - case 'l': return ','; - case 'm': return '-'; - case 'n': return KEY_DEL; // '.' - case 'o': return '/'; - case 'p': return KEY_INS; - case 'q': return KEY_END; - case 'r': return KEY_DOWN; - case 's': return KEY_PAGEDOWN; - case 't': return KEY_LEFT; - case 'u': return '5'; - case 'v': return KEY_RIGHT; - case 'w': return KEY_HOME; - case 'x': return KEY_UP; - case 'y': return KEY_PAGEUP; - } - return KEY_NONE; -} - -static void tty_read_csi_num(tty_t* tty, uint8_t* ppeek, uint32_t* num, long esc_timeout) { - *num = 1; // default - ssize_t count = 0; - uint32_t i = 0; - while (*ppeek >= '0' && *ppeek <= '9' && count < 16) { - uint8_t digit = *ppeek - '0'; - if (!tty_readc_noblock(tty,ppeek,esc_timeout)) break; // peek is not modified in this case - count++; - i = 10*i + digit; - } - if (count > 0) *num = i; -} - -static code_t tty_read_csi(tty_t* tty, uint8_t c1, uint8_t peek, code_t mods0, long esc_timeout) { - // CSI starts with 0x9b (c1=='[') | ESC [ (c1=='[') | ESC [Oo?] (c1 == 'O') /* = SS3 */ - - // check for extra starter '[' (Linux sends ESC [ [ 15 ~ for F5 for example) - if (c1 == '[' && strchr("[Oo", (char)peek) != NULL) { - uint8_t cx = peek; - if (tty_readc_noblock(tty,&peek,esc_timeout)) { - c1 = cx; - } - } - - // "special" characters ('?' is used for private sequences) - uint8_t special = 0; - if (strchr(":<=>?",(char)peek) != NULL) { - special = peek; - if (!tty_readc_noblock(tty,&peek,esc_timeout)) { - tty_cpush_char(tty,special); // recover - return (key_unicode(c1) | KEY_MOD_ALT); // Alt+ - } - } - - // up to 2 parameters that default to 1 - uint32_t num1 = 1; - uint32_t num2 = 1; - tty_read_csi_num(tty,&peek,&num1,esc_timeout); - if (peek == ';') { - if (!tty_readc_noblock(tty,&peek,esc_timeout)) return KEY_NONE; - tty_read_csi_num(tty,&peek,&num2,esc_timeout); - } - - // the final character (we do not allow 'intermediate characters') - uint8_t final = peek; - code_t modifiers = mods0; - - debug_msg("tty: escape sequence: ESC %c %c %d;%d %c\n", c1, (special == 0 ? '_' : special), num1, num2, final); - - // Adjust special cases into standard ones. - if ((final == '@' || final == '9') && c1 == '[' && num1 == 1) { - // ESC [ @, ESC [ 9 : on Mach - if (final == '@') num1 = 3; // DEL - else if (final == '9') num1 = 2; // INS - final = '~'; - } - else if (final == '^' || final == '$' || final == '@') { - // Eterm/rxvt/urxt - if (final=='^') modifiers |= KEY_MOD_CTRL; - if (final=='$') modifiers |= KEY_MOD_SHIFT; - if (final=='@') modifiers |= KEY_MOD_SHIFT | KEY_MOD_CTRL; - final = '~'; - } - else if (c1 == '[' && final >= 'a' && final <= 'd') { // note: do not catch ESC [ .. u (for unicode) - // ESC [ [a-d] : on Eterm for shift+ cursor - modifiers |= KEY_MOD_SHIFT; - final = 'A' + (final - 'a'); - } - - if (((c1 == 'O') || (c1=='[' && final != '~' && final != 'u')) && - (num2 == 1 && num1 > 1 && num1 <= 8)) - { - // on haiku the modifier can be parameter 1, make it parameter 2 instead - num2 = num1; - num1 = 1; - } - - // parameter 2 determines the modifiers - if (num2 > 1 && num2 <= 9) { - if (num2 == 9) num2 = 3; // iTerm2 in xterm mode - num2--; - if (num2 & 0x1) modifiers |= KEY_MOD_SHIFT; - if (num2 & 0x2) modifiers |= KEY_MOD_ALT; - if (num2 & 0x4) modifiers |= KEY_MOD_CTRL; - } - - // and translate - code_t code = KEY_NONE; - if (final == '~') { - // vt codes - code = esc_decode_vt(num1); - } - else if (c1 == '[' && final == 'u') { - // unicode - code = key_unicode(num1); - } - else if (c1 == 'O' && ((final >= 'A' && final <= 'Z') || (final >= 'a' && final <= 'z'))) { - // ss3 - code = esc_decode_ss3(final); - } - else if (num1 == 1 && final >= 'A' && final <= 'Z') { - // xterm - code = esc_decode_xterm(final); - } - else if (c1 == '[' && final == 'R') { - // cursor position - code = KEY_NONE; - } - - if (code == KEY_NONE && final != 'R') { - debug_msg("tty: ignore escape sequence: ESC %c %zu;%zu %c\n", c1, num1, num2, final); - } - return (code != KEY_NONE ? (code | modifiers) : KEY_NONE); -} - -static code_t tty_read_osc( tty_t* tty, uint8_t* ppeek, long esc_timeout ) { - debug_msg("discard OSC response..\n"); - // keep reading until termination: OSC is terminated by BELL, or ESC \ (ST) (and STX) - while (true) { - uint8_t c = *ppeek; - if (c <= '\x07') { // BELL and anything below (STX, ^C, ^D) - if (c != '\x07') { tty_cpush_char( tty, c ); } - break; - } - else if (c=='\x1B') { - uint8_t c1; - if (!tty_readc_noblock(tty, &c1, esc_timeout)) break; - if (c1=='\\') break; - tty_cpush_char(tty,c1); - } - if (!tty_readc_noblock(tty, ppeek, esc_timeout)) break; - } - return KEY_NONE; -} - -ic_private code_t tty_read_esc(tty_t* tty, long esc_initial_timeout, long esc_timeout) { - code_t mods = 0; - uint8_t peek = 0; - - // lone ESC? - if (!tty_readc_noblock(tty, &peek, esc_initial_timeout)) return KEY_ESC; - - // treat ESC ESC as Alt modifier (macOS sends ESC ESC [ [A-D] for alt-) - if (peek == KEY_ESC) { - if (!tty_readc_noblock(tty, &peek, esc_timeout)) goto alt; - mods |= KEY_MOD_ALT; - } - - // CSI ? - if (peek == '[') { - if (!tty_readc_noblock(tty, &peek, esc_timeout)) goto alt; - return tty_read_csi(tty, '[', peek, mods, esc_timeout); // ESC [ ... - } - - // SS3? - if (peek == 'O' || peek == 'o' || peek == '?' /*vt52*/) { - uint8_t c1 = peek; - if (!tty_readc_noblock(tty, &peek, esc_timeout)) goto alt; - if (c1 == 'o') { - // ETerm uses this for ctrl+ - mods |= KEY_MOD_CTRL; - } - // treat all as standard SS3 'O' - return tty_read_csi(tty,'O',peek,mods, esc_timeout); // ESC [Oo?] ... - } - - // OSC: we may get a delayed query response; ensure it is ignored - if (peek == ']') { - if (!tty_readc_noblock(tty, &peek, esc_timeout)) goto alt; - return tty_read_osc(tty, &peek, esc_timeout); // ESC ] ... - } - -alt: - // Alt+ - return (key_unicode(peek) | KEY_MOD_ALT); // ESC -} diff --git a/vendor/isocline/undo.c b/vendor/isocline/undo.c deleted file mode 100644 index 9bcad6bf..00000000 --- a/vendor/isocline/undo.c +++ /dev/null @@ -1,67 +0,0 @@ -/* ---------------------------------------------------------------------------- - Copyright (c) 2021, Daan Leijen - This is free software; you can redistribute it and/or modify it - under the terms of the MIT License. A copy of the license can be - found in the "LICENSE" file at the root of this distribution. ------------------------------------------------------------------------------*/ -#include -#include - -#include "./isocline.h" -#include "common.h" -#include "env.h" -#include "stringbuf.h" -#include "completions.h" -#include "undo.h" - - - -//------------------------------------------------------------- -// edit state -//------------------------------------------------------------- -struct editstate_s { - struct editstate_s* next; - const char* input; // input - ssize_t pos; // cursor position -}; - -ic_private void editstate_init( editstate_t** es ) { - *es = NULL; -} - -ic_private void editstate_done( alloc_t* mem, editstate_t** es ) { - while (*es != NULL) { - editstate_t* next = (*es)->next; - mem_free(mem, (*es)->input); - mem_free(mem, *es ); - *es = next; - } - *es = NULL; -} - -ic_private void editstate_capture( alloc_t* mem, editstate_t** es, const char* input, ssize_t pos) { - if (input==NULL) input = ""; - // alloc - editstate_t* entry = mem_zalloc_tp(mem, editstate_t); - if (entry == NULL) return; - // initialize - entry->input = mem_strdup( mem, input); - entry->pos = pos; - if (entry->input == NULL) { mem_free(mem, entry); return; } - // and push - entry->next = *es; - *es = entry; -} - -// caller should free *input -ic_private bool editstate_restore( alloc_t* mem, editstate_t** es, const char** input, ssize_t* pos ) { - if (*es == NULL) return false; - // pop - editstate_t* entry = *es; - *es = entry->next; - *input = entry->input; - *pos = entry->pos; - mem_free(mem, entry); - return true; -} - diff --git a/vendor/isocline/undo.h b/vendor/isocline/undo.h deleted file mode 100644 index 576cf977..00000000 --- a/vendor/isocline/undo.h +++ /dev/null @@ -1,24 +0,0 @@ -/* ---------------------------------------------------------------------------- - Copyright (c) 2021, Daan Leijen - This is free software; you can redistribute it and/or modify it - under the terms of the MIT License. A copy of the license can be - found in the "LICENSE" file at the root of this distribution. ------------------------------------------------------------------------------*/ -#pragma once -#ifndef IC_UNDO_H -#define IC_UNDO_H - -#include "common.h" - -//------------------------------------------------------------- -// Edit state -//------------------------------------------------------------- -struct editstate_s; -typedef struct editstate_s editstate_t; - -ic_private void editstate_init( editstate_t** es ); -ic_private void editstate_done( alloc_t* mem, editstate_t** es ); -ic_private void editstate_capture( alloc_t* mem, editstate_t** es, const char* input, ssize_t pos); -ic_private bool editstate_restore( alloc_t* mem, editstate_t** es, const char** input, ssize_t* pos ); // caller needs to free input - -#endif // IC_UNDO_H diff --git a/vendor/isocline/wcwidth.c b/vendor/isocline/wcwidth.c deleted file mode 100644 index 85187d41..00000000 --- a/vendor/isocline/wcwidth.c +++ /dev/null @@ -1,292 +0,0 @@ -// include in "stringbuf.c" -/* - * This is an implementation of wcwidth() and wcswidth() (defined in - * IEEE Std 1002.1-2001) for Unicode. - * - * http://www.opengroup.org/onlinepubs/007904975/functions/wcwidth.html - * http://www.opengroup.org/onlinepubs/007904975/functions/wcswidth.html - * - * In fixed-width output devices, Latin characters all occupy a single - * "cell" position of equal width, whereas ideographic CJK characters - * occupy two such cells. Interoperability between terminal-line - * applications and (teletype-style) character terminals using the - * UTF-8 encoding requires agreement on which character should advance - * the cursor by how many cell positions. No established formal - * standards exist at present on which Unicode character shall occupy - * how many cell positions on character terminals. These routines are - * a first attempt of defining such behavior based on simple rules - * applied to data provided by the Unicode Consortium. - * - * For some graphical characters, the Unicode standard explicitly - * defines a character-cell width via the definition of the East Asian - * FullWidth (F), Wide (W), Half-width (H), and Narrow (Na) classes. - * In all these cases, there is no ambiguity about which width a - * terminal shall use. For characters in the East Asian Ambiguous (A) - * class, the width choice depends purely on a preference of backward - * compatibility with either historic CJK or Western practice. - * Choosing single-width for these characters is easy to justify as - * the appropriate long-term solution, as the CJK practice of - * displaying these characters as double-width comes from historic - * implementation simplicity (8-bit encoded characters were displayed - * single-width and 16-bit ones double-width, even for Greek, - * Cyrillic, etc.) and not any typographic considerations. - * - * Much less clear is the choice of width for the Not East Asian - * (Neutral) class. Existing practice does not dictate a width for any - * of these characters. It would nevertheless make sense - * typographically to allocate two character cells to characters such - * as for instance EM SPACE or VOLUME INTEGRAL, which cannot be - * represented adequately with a single-width glyph. The following - * routines at present merely assign a single-cell width to all - * neutral characters, in the interest of simplicity. This is not - * entirely satisfactory and should be reconsidered before - * establishing a formal standard in this area. At the moment, the - * decision which Not East Asian (Neutral) characters should be - * represented by double-width glyphs cannot yet be answered by - * applying a simple rule from the Unicode database content. Setting - * up a proper standard for the behavior of UTF-8 character terminals - * will require a careful analysis not only of each Unicode character, - * but also of each presentation form, something the author of these - * routines has avoided to do so far. - * - * http://www.unicode.org/unicode/reports/tr11/ - * - * Markus Kuhn -- 2007-05-26 (Unicode 5.0) - * - * Permission to use, copy, modify, and distribute this software - * for any purpose and without fee is hereby granted. The author - * disclaims all warranties with regard to this software. - * - * Latest version: http://www.cl.cam.ac.uk/~mgk25/ucs/wcwidth.c - */ - -#include -#include - -struct interval { - int32_t first; - int32_t last; -}; - -/* auxiliary function for binary search in interval table */ -static int bisearch(int32_t ucs, const struct interval *table, int max) { - int min = 0; - int mid; - - if (ucs < table[0].first || ucs > table[max].last) - return 0; - while (max >= min) { - mid = (min + max) / 2; - if (ucs > table[mid].last) - min = mid + 1; - else if (ucs < table[mid].first) - max = mid - 1; - else - return 1; - } - - return 0; -} - - -/* The following two functions define the column width of an ISO 10646 - * character as follows: - * - * - The null character (U+0000) has a column width of 0. - * - * - Other C0/C1 control characters and DEL will lead to a return - * value of -1. - * - * - Non-spacing and enclosing combining characters (general - * category code Mn or Me in the Unicode database) have a - * column width of 0. - * - * - SOFT HYPHEN (U+00AD) has a column width of 1. - * - * - Other format characters (general category code Cf in the Unicode - * database) and ZERO WIDTH SPACE (U+200B) have a column width of 0. - * - * - Hangul Jamo medial vowels and final consonants (U+1160-U+11FF) - * have a column width of 0. - * - * - Spacing characters in the East Asian Wide (W) or East Asian - * Full-width (F) category as defined in Unicode Technical - * Report #11 have a column width of 2. - * - * - All remaining characters (including all printable - * ISO 8859-1 and WGL4 characters, Unicode control characters, - * etc.) have a column width of 1. - * - * This implementation assumes that wchar_t characters are encoded - * in ISO 10646. - */ - -static int mk_is_wide_char(int32_t ucs) { - static const struct interval wide[] = { - {0x1100, 0x115f}, {0x231a, 0x231b}, {0x2329, 0x232a}, - {0x23e9, 0x23ec}, {0x23f0, 0x23f0}, {0x23f3, 0x23f3}, - {0x25fd, 0x25fe}, {0x2614, 0x2615}, {0x2648, 0x2653}, - {0x267f, 0x267f}, {0x2693, 0x2693}, {0x26a1, 0x26a1}, - {0x26aa, 0x26ab}, {0x26bd, 0x26be}, {0x26c4, 0x26c5}, - {0x26ce, 0x26ce}, {0x26d4, 0x26d4}, {0x26ea, 0x26ea}, - {0x26f2, 0x26f3}, {0x26f5, 0x26f5}, {0x26fa, 0x26fa}, - {0x26fd, 0x26fd}, {0x2705, 0x2705}, {0x270a, 0x270b}, - {0x2728, 0x2728}, {0x274c, 0x274c}, {0x274e, 0x274e}, - {0x2753, 0x2755}, {0x2757, 0x2757}, {0x2795, 0x2797}, - {0x27b0, 0x27b0}, {0x27bf, 0x27bf}, {0x2b1b, 0x2b1c}, - {0x2b50, 0x2b50}, {0x2b55, 0x2b55}, {0x2e80, 0x2fdf}, - {0x2ff0, 0x303e}, {0x3040, 0x3247}, {0x3250, 0x4dbf}, - {0x4e00, 0xa4cf}, {0xa960, 0xa97f}, {0xac00, 0xd7a3}, - {0xf900, 0xfaff}, {0xfe10, 0xfe19}, {0xfe30, 0xfe6f}, - {0xff01, 0xff60}, {0xffe0, 0xffe6}, {0x16fe0, 0x16fe1}, - {0x17000, 0x18aff}, {0x1b000, 0x1b12f}, {0x1b170, 0x1b2ff}, - {0x1f004, 0x1f004}, {0x1f0cf, 0x1f0cf}, {0x1f18e, 0x1f18e}, - {0x1f191, 0x1f19a}, {0x1f200, 0x1f202}, {0x1f210, 0x1f23b}, - {0x1f240, 0x1f248}, {0x1f250, 0x1f251}, {0x1f260, 0x1f265}, - {0x1f300, 0x1f320}, {0x1f32d, 0x1f335}, {0x1f337, 0x1f37c}, - {0x1f37e, 0x1f393}, {0x1f3a0, 0x1f3ca}, {0x1f3cf, 0x1f3d3}, - {0x1f3e0, 0x1f3f0}, {0x1f3f4, 0x1f3f4}, {0x1f3f8, 0x1f43e}, - {0x1f440, 0x1f440}, {0x1f442, 0x1f4fc}, {0x1f4ff, 0x1f53d}, - {0x1f54b, 0x1f54e}, {0x1f550, 0x1f567}, {0x1f57a, 0x1f57a}, - {0x1f595, 0x1f596}, {0x1f5a4, 0x1f5a4}, {0x1f5fb, 0x1f64f}, - {0x1f680, 0x1f6c5}, {0x1f6cc, 0x1f6cc}, {0x1f6d0, 0x1f6d2}, - {0x1f6eb, 0x1f6ec}, {0x1f6f4, 0x1f6f8}, {0x1f910, 0x1f93e}, - {0x1f940, 0x1f94c}, {0x1f950, 0x1f96b}, {0x1f980, 0x1f997}, - {0x1f9c0, 0x1f9c0}, {0x1f9d0, 0x1f9e6}, {0x20000, 0x2fffd}, - {0x30000, 0x3fffd}, - }; - - if ( bisearch(ucs, wide, sizeof(wide) / sizeof(struct interval) - 1) ) { - return 1; - } - - return 0; -} - -static int mk_wcwidth(int32_t ucs) { - /* sorted list of non-overlapping intervals of non-spacing characters */ - /* generated by "uniset +cat=Me +cat=Mn +cat=Cf -00AD +1160-11FF +200B c" */ - static const struct interval combining[] = { - {0x00ad, 0x00ad}, {0x0300, 0x036f}, {0x0483, 0x0489}, - {0x0591, 0x05bd}, {0x05bf, 0x05bf}, {0x05c1, 0x05c2}, - {0x05c4, 0x05c5}, {0x05c7, 0x05c7}, {0x0610, 0x061a}, - {0x061c, 0x061c}, {0x064b, 0x065f}, {0x0670, 0x0670}, - {0x06d6, 0x06dc}, {0x06df, 0x06e4}, {0x06e7, 0x06e8}, - {0x06ea, 0x06ed}, {0x0711, 0x0711}, {0x0730, 0x074a}, - {0x07a6, 0x07b0}, {0x07eb, 0x07f3}, {0x0816, 0x0819}, - {0x081b, 0x0823}, {0x0825, 0x0827}, {0x0829, 0x082d}, - {0x0859, 0x085b}, {0x08d4, 0x08e1}, {0x08e3, 0x0902}, - {0x093a, 0x093a}, {0x093c, 0x093c}, {0x0941, 0x0948}, - {0x094d, 0x094d}, {0x0951, 0x0957}, {0x0962, 0x0963}, - {0x0981, 0x0981}, {0x09bc, 0x09bc}, {0x09c1, 0x09c4}, - {0x09cd, 0x09cd}, {0x09e2, 0x09e3}, {0x0a01, 0x0a02}, - {0x0a3c, 0x0a3c}, {0x0a41, 0x0a42}, {0x0a47, 0x0a48}, - {0x0a4b, 0x0a4d}, {0x0a51, 0x0a51}, {0x0a70, 0x0a71}, - {0x0a75, 0x0a75}, {0x0a81, 0x0a82}, {0x0abc, 0x0abc}, - {0x0ac1, 0x0ac5}, {0x0ac7, 0x0ac8}, {0x0acd, 0x0acd}, - {0x0ae2, 0x0ae3}, {0x0afa, 0x0aff}, {0x0b01, 0x0b01}, - {0x0b3c, 0x0b3c}, {0x0b3f, 0x0b3f}, {0x0b41, 0x0b44}, - {0x0b4d, 0x0b4d}, {0x0b56, 0x0b56}, {0x0b62, 0x0b63}, - {0x0b82, 0x0b82}, {0x0bc0, 0x0bc0}, {0x0bcd, 0x0bcd}, - {0x0c00, 0x0c00}, {0x0c3e, 0x0c40}, {0x0c46, 0x0c48}, - {0x0c4a, 0x0c4d}, {0x0c55, 0x0c56}, {0x0c62, 0x0c63}, - {0x0c81, 0x0c81}, {0x0cbc, 0x0cbc}, {0x0cbf, 0x0cbf}, - {0x0cc6, 0x0cc6}, {0x0ccc, 0x0ccd}, {0x0ce2, 0x0ce3}, - {0x0d00, 0x0d01}, {0x0d3b, 0x0d3c}, {0x0d41, 0x0d44}, - {0x0d4d, 0x0d4d}, {0x0d62, 0x0d63}, {0x0dca, 0x0dca}, - {0x0dd2, 0x0dd4}, {0x0dd6, 0x0dd6}, {0x0e31, 0x0e31}, - {0x0e34, 0x0e3a}, {0x0e47, 0x0e4e}, {0x0eb1, 0x0eb1}, - {0x0eb4, 0x0eb9}, {0x0ebb, 0x0ebc}, {0x0ec8, 0x0ecd}, - {0x0f18, 0x0f19}, {0x0f35, 0x0f35}, {0x0f37, 0x0f37}, - {0x0f39, 0x0f39}, {0x0f71, 0x0f7e}, {0x0f80, 0x0f84}, - {0x0f86, 0x0f87}, {0x0f8d, 0x0f97}, {0x0f99, 0x0fbc}, - {0x0fc6, 0x0fc6}, {0x102d, 0x1030}, {0x1032, 0x1037}, - {0x1039, 0x103a}, {0x103d, 0x103e}, {0x1058, 0x1059}, - {0x105e, 0x1060}, {0x1071, 0x1074}, {0x1082, 0x1082}, - {0x1085, 0x1086}, {0x108d, 0x108d}, {0x109d, 0x109d}, - {0x1160, 0x11ff}, {0x135d, 0x135f}, {0x1712, 0x1714}, - {0x1732, 0x1734}, {0x1752, 0x1753}, {0x1772, 0x1773}, - {0x17b4, 0x17b5}, {0x17b7, 0x17bd}, {0x17c6, 0x17c6}, - {0x17c9, 0x17d3}, {0x17dd, 0x17dd}, {0x180b, 0x180e}, - {0x1885, 0x1886}, {0x18a9, 0x18a9}, {0x1920, 0x1922}, - {0x1927, 0x1928}, {0x1932, 0x1932}, {0x1939, 0x193b}, - {0x1a17, 0x1a18}, {0x1a1b, 0x1a1b}, {0x1a56, 0x1a56}, - {0x1a58, 0x1a5e}, {0x1a60, 0x1a60}, {0x1a62, 0x1a62}, - {0x1a65, 0x1a6c}, {0x1a73, 0x1a7c}, {0x1a7f, 0x1a7f}, - {0x1ab0, 0x1abe}, {0x1b00, 0x1b03}, {0x1b34, 0x1b34}, - {0x1b36, 0x1b3a}, {0x1b3c, 0x1b3c}, {0x1b42, 0x1b42}, - {0x1b6b, 0x1b73}, {0x1b80, 0x1b81}, {0x1ba2, 0x1ba5}, - {0x1ba8, 0x1ba9}, {0x1bab, 0x1bad}, {0x1be6, 0x1be6}, - {0x1be8, 0x1be9}, {0x1bed, 0x1bed}, {0x1bef, 0x1bf1}, - {0x1c2c, 0x1c33}, {0x1c36, 0x1c37}, {0x1cd0, 0x1cd2}, - {0x1cd4, 0x1ce0}, {0x1ce2, 0x1ce8}, {0x1ced, 0x1ced}, - {0x1cf4, 0x1cf4}, {0x1cf8, 0x1cf9}, {0x1dc0, 0x1df9}, - {0x1dfb, 0x1dff}, {0x200b, 0x200f}, {0x202a, 0x202e}, - {0x2060, 0x2064}, {0x2066, 0x206f}, {0x20d0, 0x20f0}, - {0x2cef, 0x2cf1}, {0x2d7f, 0x2d7f}, {0x2de0, 0x2dff}, - {0x302a, 0x302d}, {0x3099, 0x309a}, {0xa66f, 0xa672}, - {0xa674, 0xa67d}, {0xa69e, 0xa69f}, {0xa6f0, 0xa6f1}, - {0xa802, 0xa802}, {0xa806, 0xa806}, {0xa80b, 0xa80b}, - {0xa825, 0xa826}, {0xa8c4, 0xa8c5}, {0xa8e0, 0xa8f1}, - {0xa926, 0xa92d}, {0xa947, 0xa951}, {0xa980, 0xa982}, - {0xa9b3, 0xa9b3}, {0xa9b6, 0xa9b9}, {0xa9bc, 0xa9bc}, - {0xa9e5, 0xa9e5}, {0xaa29, 0xaa2e}, {0xaa31, 0xaa32}, - {0xaa35, 0xaa36}, {0xaa43, 0xaa43}, {0xaa4c, 0xaa4c}, - {0xaa7c, 0xaa7c}, {0xaab0, 0xaab0}, {0xaab2, 0xaab4}, - {0xaab7, 0xaab8}, {0xaabe, 0xaabf}, {0xaac1, 0xaac1}, - {0xaaec, 0xaaed}, {0xaaf6, 0xaaf6}, {0xabe5, 0xabe5}, - {0xabe8, 0xabe8}, {0xabed, 0xabed}, {0xfb1e, 0xfb1e}, - {0xfe00, 0xfe0f}, {0xfe20, 0xfe2f}, {0xfeff, 0xfeff}, - {0xfff9, 0xfffb}, {0x101fd, 0x101fd}, {0x102e0, 0x102e0}, - {0x10376, 0x1037a}, {0x10a01, 0x10a03}, {0x10a05, 0x10a06}, - {0x10a0c, 0x10a0f}, {0x10a38, 0x10a3a}, {0x10a3f, 0x10a3f}, - {0x10ae5, 0x10ae6}, {0x11001, 0x11001}, {0x11038, 0x11046}, - {0x1107f, 0x11081}, {0x110b3, 0x110b6}, {0x110b9, 0x110ba}, - {0x11100, 0x11102}, {0x11127, 0x1112b}, {0x1112d, 0x11134}, - {0x11173, 0x11173}, {0x11180, 0x11181}, {0x111b6, 0x111be}, - {0x111ca, 0x111cc}, {0x1122f, 0x11231}, {0x11234, 0x11234}, - {0x11236, 0x11237}, {0x1123e, 0x1123e}, {0x112df, 0x112df}, - {0x112e3, 0x112ea}, {0x11300, 0x11301}, {0x1133c, 0x1133c}, - {0x11340, 0x11340}, {0x11366, 0x1136c}, {0x11370, 0x11374}, - {0x11438, 0x1143f}, {0x11442, 0x11444}, {0x11446, 0x11446}, - {0x114b3, 0x114b8}, {0x114ba, 0x114ba}, {0x114bf, 0x114c0}, - {0x114c2, 0x114c3}, {0x115b2, 0x115b5}, {0x115bc, 0x115bd}, - {0x115bf, 0x115c0}, {0x115dc, 0x115dd}, {0x11633, 0x1163a}, - {0x1163d, 0x1163d}, {0x1163f, 0x11640}, {0x116ab, 0x116ab}, - {0x116ad, 0x116ad}, {0x116b0, 0x116b5}, {0x116b7, 0x116b7}, - {0x1171d, 0x1171f}, {0x11722, 0x11725}, {0x11727, 0x1172b}, - {0x11a01, 0x11a06}, {0x11a09, 0x11a0a}, {0x11a33, 0x11a38}, - {0x11a3b, 0x11a3e}, {0x11a47, 0x11a47}, {0x11a51, 0x11a56}, - {0x11a59, 0x11a5b}, {0x11a8a, 0x11a96}, {0x11a98, 0x11a99}, - {0x11c30, 0x11c36}, {0x11c38, 0x11c3d}, {0x11c3f, 0x11c3f}, - {0x11c92, 0x11ca7}, {0x11caa, 0x11cb0}, {0x11cb2, 0x11cb3}, - {0x11cb5, 0x11cb6}, {0x11d31, 0x11d36}, {0x11d3a, 0x11d3a}, - {0x11d3c, 0x11d3d}, {0x11d3f, 0x11d45}, {0x11d47, 0x11d47}, - {0x16af0, 0x16af4}, {0x16b30, 0x16b36}, {0x16f8f, 0x16f92}, - {0x1bc9d, 0x1bc9e}, {0x1bca0, 0x1bca3}, {0x1d167, 0x1d169}, - {0x1d173, 0x1d182}, {0x1d185, 0x1d18b}, {0x1d1aa, 0x1d1ad}, - {0x1d242, 0x1d244}, {0x1da00, 0x1da36}, {0x1da3b, 0x1da6c}, - {0x1da75, 0x1da75}, {0x1da84, 0x1da84}, {0x1da9b, 0x1da9f}, - {0x1daa1, 0x1daaf}, {0x1e000, 0x1e006}, {0x1e008, 0x1e018}, - {0x1e01b, 0x1e021}, {0x1e023, 0x1e024}, {0x1e026, 0x1e02a}, - {0x1e8d0, 0x1e8d6}, {0x1e944, 0x1e94a}, {0xe0001, 0xe0001}, - {0xe0020, 0xe007f}, {0xe0100, 0xe01ef}, - }; - - /* test for 8-bit control characters */ - if ( ucs == 0 ) { - return 0; - } - if ( ( ucs < 32 ) || ( ( ucs >= 0x7f ) && ( ucs < 0xa0 ) ) ) { - return -1; - } - - /* binary search in table of non-spacing characters */ - if ( bisearch( ucs, combining, sizeof( combining ) / sizeof( struct interval ) - 1 ) ) { - return 0; - } - - /* if we arrive here, ucs is not a combining or C0/C1 control character */ - return ( mk_is_wide_char( ucs ) ? 2 : 1 ); -} - diff --git a/vendor/tb/.editorconfig b/vendor/tb/.editorconfig deleted file mode 100644 index 70fdd549..00000000 --- a/vendor/tb/.editorconfig +++ /dev/null @@ -1,11 +0,0 @@ -# EditorConfig is awesome: https://EditorConfig.org - -# top-most EditorConfig file -root = true - -# Unix-style newlines with a newline ending every file -[*] -end_of_line = crlf -insert_final_newline = true -indent_style = tab -indent_size = 4 diff --git a/vendor/tb/.gitignore b/vendor/tb/.gitignore deleted file mode 100644 index cca3fcc3..00000000 --- a/vendor/tb/.gitignore +++ /dev/null @@ -1,26 +0,0 @@ -.vs/ -.tup/ -bin/ -negate/ - -tildebackend.lib -tildebackend.a - -*.cache -*.swp -*.out -*.o -*.obj -*.exe -*.pdb - -tup.config -debug.bat -build.bat -run.bat -run4coder.bat -project.4coder -tb.rdbg -build.ninja -.ninja_deps -.ninja_log diff --git a/vendor/tb/IR.txt b/vendor/tb/IR.txt deleted file mode 100644 index 30f86039..00000000 --- a/vendor/tb/IR.txt +++ /dev/null @@ -1,58 +0,0 @@ -# Sea of Nodes (SoN) - - https://www.oracle.com/technetwork/java/javase/tech/c2-ir95-150110.pdf - - SoN is an SSA where ordering is relaxed in the form of explicit dependencies - as opposed to local ordering inside basic blocks, for instance pure operations - like addition will not have an exact placement only the constraint that it must - be resolved after it's inputs. This makes it easier to perform local optimizations - without a care for scheduling, this is especially helpful because of how many - optimizations we've moved to peepholes. - - note: edges going down from A to B means B is dependent on A. - - Reassociation - - x+2+4 - - x 2 2 4 - \ / \ / - + 4 => x + - \ / \ / - + + - - GVN - - A*B + A*B - - A B A B - |\ /| \ / - | X | * - |/ \| => / \ - * * \ / - \ / + - + - Load elimination - - *x = 16 - return *x - - x_ - | \ - | \ - | \ x - | | | - memory | 16 | => memory | 16 - \ | | | \ | / | - Store | Store | - | | / - | / / - | / / - | / / - Load | - | | - V V - - note: we're not showing the control edge memory operations have for simplicit but - both of these are sharing a control edge. Stores produce more memory but don't produce - more control flow and Loads use memory but don't produce more (these are both non-volatile) diff --git a/vendor/tb/LICENSE.txt b/vendor/tb/LICENSE.txt deleted file mode 100644 index b731914f..00000000 --- a/vendor/tb/LICENSE.txt +++ /dev/null @@ -1,21 +0,0 @@ -MIT License - -Copyright (c) 2021 Yasser Arguelles Snape - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. diff --git a/vendor/tb/NOTES.txt b/vendor/tb/NOTES.txt deleted file mode 100644 index cc0208e8..00000000 --- a/vendor/tb/NOTES.txt +++ /dev/null @@ -1,17 +0,0 @@ -# Optimizer crap - - some of the optimizations i should probably worry about are proving when numbers can't - overflow, like induction vars: - -``` -for { - i = phi(0, j) - // even if n is TOP, i must be at least TOP and - // after all additions on the PHI... which means - // no overflow - if i >= n break - ... - // next - j = i + 1 -} -``` diff --git a/vendor/tb/README.txt b/vendor/tb/README.txt deleted file mode 100644 index 7307205f..00000000 --- a/vendor/tb/README.txt +++ /dev/null @@ -1,22 +0,0 @@ -TildeBackend (Tilde or TB for short) - - TB is compiler backend in the form of a reasonable C library. This is built as an alternative to other larger compiler toolchains while providing the optimizations, machine code generation and object file export functionality necessary for the development of compilers. - - # Roadmap - - Code generation: - We're starting with x64 but will be moving focus to Aarch64 soon. - - Optimizer: - It's almost complete with all the -O1 level passes (mostly missing inlining). - After that we can move towards -O2 level stuff (the goal is to compete with - LLVM so we need to be a bit ambitious). - - Debug info: - Codeview support and DWARF has not yet been started, there's plans on making a - new debug info format eventually. - - Output targets: - We currently have basic ELF64, COFF64, some current work is being done for - PE and Macho-O. We got exporting object files but wanna go further because - linkers ain't supposed to be separate programs. diff --git a/vendor/tb/include/sdg.h b/vendor/tb/include/sdg.h deleted file mode 100644 index 2c7aba7f..00000000 --- a/vendor/tb/include/sdg.h +++ /dev/null @@ -1,111 +0,0 @@ -#include - -// builtin primitives (the custom types start at 0x100) -// -// if the top bit is set, we're using a pointer to these -// types rather than a direct type. -typedef enum { - SDG_PRIM_VOID, - - // builtin bools - SDG_PRIM_BOOL8, SDG_PRIM_BOOL16, SDG_PRIM_BOOL32, SDG_PRIM_BOOL64, - - // builtin char - SDG_PRIM_CHAR8, SDG_PRIM_CHAR16, SDG_PRIM_CHAR32, - - // builtin integers - SDG_PRIM_INT8, SDG_PRIM_UINT8, - SDG_PRIM_INT16, SDG_PRIM_UINT16, - SDG_PRIM_INT32, SDG_PRIM_UINT32, - SDG_PRIM_INT64, SDG_PRIM_UINT64, - - // builtin floats - SDG_PRIM_FLOAT, SDG_PRIM_DOUBLE, SDG_PRIM_LONG_DOUBLE, - - // NOTE(NeGate): if set, the type is now a pointer to the described type. - SDG_PRIM_POINTER = 0x80, - - CUSTOM_TYPE_START = 0x100, -} SDG_TypeIndex; - -typedef struct { - enum { - SDG_TYPE_PTR, - SDG_TYPE_FUNC, - } tag; - - int arg_count; - SDG_TypeIndex base; - SDG_TypeIndex args[]; -} SDG_Type; - -// symbol table -typedef enum { - // normal symbols - SDG_SYMBOL_PROC, - SDG_SYMBOL_GLOBAL, - - // magic symbols - SDG_SYMBOL_FILE, - SDG_SYMBOL_MODULE, -} SDG_SymbolTag; - -typedef struct { - uint8_t tag; - // number of bytes to skip to reach the first element in the body of the symbol. - // this only applies for procedures because they have nested elements. - uint16_t content_ptr; - uint32_t kid_count; - // 0 if doesn't have one - uint32_t next; -} SDG_Symbol; - -typedef struct { - SDG_Symbol super; - // type - SDG_TypeIndex type; - // in program's memory - uint32_t rva, size; - char name[]; -} SDG_NormalSymbol; - -typedef struct { - SDG_Symbol super; - // type - SDG_TypeIndex type; - // locals consist of pieces (for now, eventually - // some location -> pieces) - char name[]; -} SDG_Local; - -// Each piece can do a limited amount of processing -// on a variable to get it copied into the logical -// view of the variable. -typedef struct { - // place to copy into the logical view. - uint64_t offset; - - // indirection: - // - int32_t disp; - uint8_t base; - - // bit extraction: - // (x >> bit_offset) & (~0 >> (64 - bit_len)) - // - uint8_t bit_offset, bit_len; -} SDG_Piece; - -typedef struct { - SDG_Symbol super; - // used to track changes - uint32_t last_write; - char name[]; -} SDG_File; - -typedef struct { - SDG_Symbol super; - uint32_t type_table; - uint32_t type_count; - char name[]; -} SDG_Module; diff --git a/vendor/tb/include/tb.h b/vendor/tb/include/tb.h deleted file mode 100644 index 0bf7c34e..00000000 --- a/vendor/tb/include/tb.h +++ /dev/null @@ -1,1468 +0,0 @@ -// Glossary (because i don't know where else to put it) -// IR - intermediate representation -// SoN - sea of nodes (https://www.oracle.com/technetwork/java/javase/tech/c2-ir95-150110.pdf) -// SSA - single static assignment -// GVN - global value numbering -// CSE - common subexpression elimination -// CFG - control flow graph -// DSE - dead store elimination -// GCM - global code motion -// SROA - scalar replacement of aggregates -// SCCP - sparse conditional constant propagation -// RPO - reverse postorder -// BB - basic block -// ZTC - zero trip count -// SCC - strongly connected components -#ifndef TB_CORE_H -#define TB_CORE_H - -#include -#include -#include -#include -#include -#include -#include -#include - -#define TB_VERSION_MAJOR 0 -#define TB_VERSION_MINOR 3 -#define TB_VERSION_PATCH 0 - -#ifndef TB_API -# ifdef __cplusplus -# define TB_EXTERN extern "C" -# else -# define TB_EXTERN -# endif -# ifdef TB_DLL -# ifdef TB_IMPORT_DLL -# define TB_API TB_EXTERN __declspec(dllimport) -# else -# define TB_API TB_EXTERN __declspec(dllexport) -# endif -# else -# define TB_API TB_EXTERN -# endif -#endif - -// These are flags -typedef enum TB_ArithmeticBehavior { - TB_ARITHMATIC_NONE = 0, - TB_ARITHMATIC_NSW = 1, - TB_ARITHMATIC_NUW = 2, -} TB_ArithmeticBehavior; - -typedef enum TB_DebugFormat { - TB_DEBUGFMT_NONE, - - TB_DEBUGFMT_DWARF, - TB_DEBUGFMT_CODEVIEW, - - TB_DEBUGFMT_SDG, -} TB_DebugFormat; - -typedef enum TB_Arch { - TB_ARCH_UNKNOWN, - - TB_ARCH_X86_64, - TB_ARCH_AARCH64, - - // they're almost identical so might as well do both. - TB_ARCH_MIPS32, - TB_ARCH_MIPS64, - - TB_ARCH_WASM32, -} TB_Arch; - -typedef enum TB_System { - TB_SYSTEM_WINDOWS, - TB_SYSTEM_LINUX, - TB_SYSTEM_MACOS, - TB_SYSTEM_ANDROID, // Not supported yet - TB_SYSTEM_WASM, - - TB_SYSTEM_MAX, -} TB_System; - -typedef enum TB_WindowsSubsystem { - TB_WIN_SUBSYSTEM_UNKNOWN, - - TB_WIN_SUBSYSTEM_WINDOWS, - TB_WIN_SUBSYSTEM_CONSOLE, - TB_WIN_SUBSYSTEM_EFI_APP, -} TB_WindowsSubsystem; - -typedef enum TB_ABI { - // Used on 64bit Windows platforms - TB_ABI_WIN64, - - // Used on Mac, BSD and Linux platforms - TB_ABI_SYSTEMV, -} TB_ABI; - -typedef enum TB_OutputFlavor { - TB_FLAVOR_OBJECT, // .o .obj - TB_FLAVOR_SHARED, // .so .dll - TB_FLAVOR_STATIC, // .a .lib - TB_FLAVOR_EXECUTABLE, // .exe -} TB_OutputFlavor; - -typedef enum TB_CallingConv { - TB_CDECL, - TB_STDCALL -} TB_CallingConv; - -typedef enum TB_FeatureSet_X64 { - TB_FEATURE_X64_SSE2 = (1u << 0u), - TB_FEATURE_X64_SSE3 = (1u << 1u), - TB_FEATURE_X64_SSE41 = (1u << 2u), - TB_FEATURE_X64_SSE42 = (1u << 3u), - - TB_FEATURE_X64_POPCNT = (1u << 4u), - TB_FEATURE_X64_LZCNT = (1u << 5u), - - TB_FEATURE_X64_CLMUL = (1u << 6u), - TB_FEATURE_X64_F16C = (1u << 7u), - - TB_FEATURE_X64_BMI1 = (1u << 8u), - TB_FEATURE_X64_BMI2 = (1u << 9u), - - TB_FEATURE_X64_AVX = (1u << 10u), - TB_FEATURE_X64_AVX2 = (1u << 11u), -} TB_FeatureSet_X64; - -typedef enum TB_FeatureSet_Generic { - TB_FEATURE_FRAME_PTR = (1u << 0u), -} TB_FeatureSet_Generic; - -typedef struct TB_FeatureSet { - TB_FeatureSet_Generic gen; - TB_FeatureSet_X64 x64; -} TB_FeatureSet; - -typedef enum TB_Linkage { - TB_LINKAGE_PUBLIC, - TB_LINKAGE_PRIVATE -} TB_Linkage; - -typedef enum { - TB_COMDAT_NONE, - - TB_COMDAT_MATCH_ANY, -} TB_ComdatType; - -typedef enum TB_MemoryOrder { - TB_MEM_ORDER_RELAXED, - TB_MEM_ORDER_CONSUME, - TB_MEM_ORDER_ACQUIRE, - TB_MEM_ORDER_RELEASE, - TB_MEM_ORDER_ACQ_REL, - TB_MEM_ORDER_SEQ_CST, -} TB_MemoryOrder; - -typedef enum TB_DataTypeEnum { - // Integers, note void is an i0 and bool is an i1 - // i(0-64) - TB_INT, - // Floating point numbers - // f{32,64} - TB_FLOAT, - // Pointers - TB_PTR, - // represents control flow for REGION, BRANCH - TB_CONTROL, - // represents memory (and I/O) - TB_MEMORY, - // Tuples, these cannot be used in memory ops, just accessed via projections - TB_TUPLE, -} TB_DataTypeEnum; - -typedef enum TB_FloatFormat { - // IEEE 754 floats - TB_FLT_32, TB_FLT_64 -} TB_FloatFormat; - -typedef union TB_DataType { - struct { - uint16_t type : 4; - // for integers it's the bitwidth - uint16_t data : 12; - }; - uint16_t raw; -} TB_DataType; -static_assert(sizeof(TB_DataType) == 2, "im expecting this to be a uint16_t"); - -// classify data types -#define TB_IS_VOID_TYPE(x) ((x).type == TB_INT && (x).data == 0) -#define TB_IS_BOOL_TYPE(x) ((x).type == TB_INT && (x).data == 1) -#define TB_IS_INTEGER_TYPE(x) ((x).type == TB_INT) -#define TB_IS_FLOAT_TYPE(x) ((x).type == TB_FLOAT) -#define TB_IS_POINTER_TYPE(x) ((x).type == TB_PTR) - -// accessors -#define TB_GET_INT_BITWIDTH(x) ((x).data) -#define TB_GET_FLOAT_FORMAT(x) ((x).data) -#define TB_GET_PTR_ADDRSPACE(x) ((x).data) - -//////////////////////////////// -// ANNOTATIONS -//////////////////////////////// -// -// (A, B) -> (C, D) -// -// node takes A and B, produces C, D. if there's multiple -// results we need to use projections and the indices are -// based on the order seen here, proj0 is C, proj1 is D. -// -// (A, B) & C -> Int -// -// nodes takes A and B along with C in it's extra data. this is -// where non-node inputs fit. -// -typedef enum TB_NodeTypeEnum { - TB_NULL = 0, - - //////////////////////////////// - // CONSTANTS - //////////////////////////////// - TB_INTEGER_CONST, - TB_FLOAT32_CONST, - TB_FLOAT64_CONST, - - //////////////////////////////// - // MISCELLANEOUS - //////////////////////////////// - // this is an unspecified value, usually generated by the optimizer - // when malformed input is folded into an operation. - TB_POISON, // () -> Any - // projections just extract a single field of a tuple - TB_PROJ, // Tuple & Int -> Any - // this is a simple way to embed machine code into the code - TB_INLINE_ASM, // (Control, Memory) & InlineAsm -> (Control, Memory) - // reads the TSC on x64 - TB_CYCLE_COUNTER, // (Control) -> Int64 - // prefetches data for reading. The number next to the - // - // 0 is temporal - // 1-3 are just cache levels - TB_PREFETCH, // (Memory, Ptr) & Int -> Memory - - //////////////////////////////// - // CONTROL - //////////////////////////////// - // there's only one ROOT per function, it's inputs are the return values, it's - // outputs are the initial params. - TB_ROOT, // (Callgraph, Exits...) -> (Control, Memory, RPC, Data...) - // return nodes feed into ROOT, jumps through the RPC out of this stack frame. - TB_RETURN, // (Control, Memory, RPC, Data...) -> () - // regions are used to represent paths which have multiple entries. - // each input is a predecessor. - TB_REGION, // (Control...) -> (Control) - // a natural loop header has the first edge be the dominating predecessor, every other edge - // is a backedge. - TB_NATURAL_LOOP, // (Control...) -> (Control) - // a natural loop header (thus also a region) of an affine loop (one where the induction - // var is an affine y=mx+b kind of function) - TB_AFFINE_LOOP, // (Control...) -> (Control) - // phi nodes work the same as in SSA CFG, the value is based on which predecessor was taken. - // each input lines up with the regions such that region.in[i] will use phi.in[i+1] as the - // subsequent data. - TB_PHI, // (Control, Data...) -> Data - // branch is used to implement most control flow, it acts like a switch - // statement in C usually. they take a key and match against some cases, - // if they match, it'll jump to that successor, if none match it'll take - // the default successor. - // - // if (cond) { A; } else { B; } is just switch (cond) { case 0: B; default: A; } - // - // it's possible to not pass a key and the default successor is always called, this is - // a GOTO. tb_inst_goto, tb_inst_if can handle common cases for you. - TB_BRANCH, // (Control, Data?) -> (Control...) - // debugbreak will trap in a continuable manner. - TB_DEBUGBREAK, // (Control, Memory) -> (Control) - // trap will not be continuable but will stop execution. - TB_TRAP, // (Control) -> (Control) - // unreachable means it won't trap or be continuable. - TB_UNREACHABLE, // (Control) -> (Control) - // all dead paths are stitched here - TB_DEAD, // (Control) -> (Control) - - //////////////////////////////// - // CONTROL + MEMORY - //////////////////////////////// - // nothing special, it's just a function call, 3rd argument here is the - // target pointer (or syscall number) and the rest are just data args. - TB_CALL, // (Control, Memory, Data, Data...) -> (Control, Memory, Data) - TB_SYSCALL, // (Control, Memory, Data, Data...) -> (Control, Memory, Data) - // performs call while recycling the stack frame somewhat - TB_TAILCALL, // (Control, Memory, RPC, Data, Data...) -> () - // safepoint polls are the same except they only trigger if the poll site - // says to (platform specific but almost always just the page being made - // unmapped/guard), 3rd argument is the poll site. - TB_SAFEPOINT_POLL, // (Control, Memory, Ptr?, Data...) -> (Control) - // this special op tracks calls such that we can produce our cool call graph, there's - // one call graph node per function that never moves. - TB_CALLGRAPH, // (Call...) -> Void - - //////////////////////////////// - // MEMORY - //////////////////////////////// - // produces a set of non-aliasing memory effects - TB_SPLITMEM, // (Memory) -> (Memory...) - // MERGEMEM will join multiple non-aliasing memory effects, because - // they don't alias there's no ordering guarentee. - TB_MERGEMEM, // (Split, Memory...) -> Memory - // LOAD and STORE are standard memory accesses, they can be folded away. - TB_LOAD, // (Control?, Memory, Ptr) -> Data - TB_STORE, // (Control, Memory, Ptr, Data) -> Memory - // bulk memory ops. - TB_MEMCPY, // (Control, Memory, Ptr, Ptr, Size) -> Memory - TB_MEMSET, // (Control, Memory, Ptr, Int8, Size) -> Memory - // these memory accesses represent "volatile" which means - // they may produce side effects and thus cannot be eliminated. - TB_READ, // (Control, Memory, Ptr) -> (Memory, Data) - TB_WRITE, // (Control, Memory, Ptr, Data) -> (Memory, Data) - // atomics have multiple observers (if not they wouldn't need to - // be atomic) and thus produce side effects everywhere just like - // volatiles except they have synchronization guarentees. the atomic - // data ops will return the value before the operation is performed. - // Atomic CAS return the old value and a boolean for success (true if - // the value was changed) - TB_ATOMIC_LOAD, // (Control, Memory, Ptr) -> (Memory, Data) - TB_ATOMIC_XCHG, // (Control, Memory, Ptr, Data) -> (Memory, Data) - TB_ATOMIC_ADD, // (Control, Memory, Ptr, Data) -> (Memory, Data) - TB_ATOMIC_SUB, // (Control, Memory, Ptr, Data) -> (Memory, Data) - TB_ATOMIC_AND, // (Control, Memory, Ptr, Data) -> (Memory, Data) - TB_ATOMIC_XOR, // (Control, Memory, Ptr, Data) -> (Memory, Data) - TB_ATOMIC_OR, // (Control, Memory, Ptr, Data) -> (Memory, Data) - TB_ATOMIC_CAS, // (Control, Memory, Data, Data) -> (Memory, Data, Bool) - - // like a multi-way branch but without the control flow aspect, but for data. - TB_LOOKUP, - - //////////////////////////////// - // POINTERS - //////////////////////////////// - // LOCAL will statically allocate stack space - TB_LOCAL, // () & (Int, Int) -> Ptr - // SYMBOL will return a pointer to a TB_Symbol - TB_SYMBOL, // () & TB_Symbol* -> Ptr - // offsets pointer by constant value - TB_MEMBER_ACCESS, // Ptr & Int -> Ptr - // arguments represent base, index, and stride respectively - // and will perform `base + index*stride` - TB_ARRAY_ACCESS, // (Ptr, Int) & Int -> Ptr - - // Conversions - TB_TRUNCATE, - TB_FLOAT_EXT, - TB_SIGN_EXT, - TB_ZERO_EXT, - TB_UINT2FLOAT, - TB_FLOAT2UINT, - TB_INT2FLOAT, - TB_FLOAT2INT, - TB_BITCAST, - - // Select - TB_SELECT, - - // Bitmagic - TB_BSWAP, - TB_CLZ, - TB_CTZ, - TB_POPCNT, - - // Unary operations - TB_NOT, - TB_NEG, - - // Integer arithmatic - TB_AND, - TB_OR, - TB_XOR, - TB_ADD, - TB_SUB, - TB_MUL, - - TB_SHL, - TB_SHR, - TB_SAR, - TB_ROL, - TB_ROR, - TB_UDIV, - TB_SDIV, - TB_UMOD, - TB_SMOD, - - // Float arithmatic - TB_FADD, - TB_FSUB, - TB_FMUL, - TB_FDIV, - TB_FMIN, - TB_FMAX, - - // Comparisons - TB_CMP_EQ, - TB_CMP_NE, - TB_CMP_ULT, - TB_CMP_ULE, - TB_CMP_SLT, - TB_CMP_SLE, - TB_CMP_FLT, - TB_CMP_FLE, - - // Special ops - // add with carry - TB_ADC, // (Int, Int, Bool?) -> (Int, Bool) - // division and modulo - TB_UDIVMOD, // (Int, Int) -> (Int, Int) - TB_SDIVMOD, // (Int, Int) -> (Int, Int) - // does full multiplication (64x64=128 and so on) returning - // the low and high values in separate projections - TB_MULPAIR, - - // variadic - TB_VA_START, - - // x86 intrinsics - TB_X86INTRIN_LDMXCSR, - TB_X86INTRIN_STMXCSR, - TB_X86INTRIN_SQRT, - TB_X86INTRIN_RSQRT, - - TB_NODE_TYPE_MAX, -} TB_NodeTypeEnum; -typedef uint8_t TB_NodeType; - -// just represents some region of bytes, usually in file parsing crap -typedef struct { - const uint8_t* data; - size_t length; -} TB_Slice; - -// represents byte counts -typedef uint32_t TB_CharUnits; - -// will get interned so each TB_Module has a unique identifier for the source file -typedef struct { - // used by the debug info export - int id; - - size_t len; - uint8_t path[]; -} TB_SourceFile; - -typedef struct TB_Location { - TB_SourceFile* file; - int line, column; - uint32_t pos; -} TB_Location; - -// SO refers to shared objects which mean either shared libraries (.so or .dll) -// or executables (.exe or ELF executables) -typedef enum { - // exports to the rest of the shared object - TB_EXTERNAL_SO_LOCAL, - - // exports outside of the shared object - TB_EXTERNAL_SO_EXPORT, -} TB_ExternalType; - -typedef struct TB_Global TB_Global; -typedef struct TB_External TB_External; -typedef struct TB_Function TB_Function; - -typedef struct TB_Module TB_Module; -typedef struct TB_DebugType TB_DebugType; -typedef struct TB_ModuleSection TB_ModuleSection; -typedef struct TB_FunctionPrototype TB_FunctionPrototype; - -enum { TB_MODULE_SECTION_NONE = -1 }; -typedef int32_t TB_ModuleSectionHandle; -typedef struct TB_Attrib TB_Attrib; - -// target-specific, just a unique ID for the registers -typedef int TB_PhysicalReg; - -// Thread local module state -typedef struct TB_ThreadInfo TB_ThreadInfo; - -typedef enum { - TB_SYMBOL_NONE, - TB_SYMBOL_EXTERNAL, - TB_SYMBOL_GLOBAL, - TB_SYMBOL_FUNCTION, - TB_SYMBOL_MAX, -} TB_SymbolTag; - -// Refers generically to objects within a module -// -// TB_Function, TB_Global, and TB_External are all subtypes of TB_Symbol -// and thus are safely allowed to cast into a symbol for operations. -typedef struct TB_Symbol { - #ifdef __cplusplus - TB_SymbolTag tag; - #else - _Atomic TB_SymbolTag tag; - #endif - - // which thread info it's tied to (we may need to remove it, this - // is used for that) - TB_ThreadInfo* info; - char* name; - - // It's kinda a weird circular reference but yea - TB_Module* module; - - // helpful for sorting and getting consistent builds - uint64_t ordinal; - - union { - // if we're JITing then this maps to the address of the symbol - void* address; - size_t symbol_id; - }; - - // after this point it's tag-specific storage -} TB_Symbol; - -// associated to nodes for debug locations -typedef struct { - TB_SourceFile* file; - int line, column; -} TB_NodeLocation; - -typedef struct TB_Node TB_Node; -typedef struct User User; -struct User { - User* next; - TB_Node* n; - int slot; -}; - -struct TB_Node { - TB_NodeType type; - - uint16_t input_cap; - uint16_t input_count; - - TB_DataType dt; - - // makes it easier to track in graph walks - uint32_t gvn; - - // only value while inside of a TB_Passes, - // these are unordered and usually just - // help perform certain transformations or - // analysis (not necessarily semantics) - User* users; - - // ordered def-use edges, jolly ol' semantics - TB_Node** inputs; - - char extra[]; -}; - -// These are the extra data in specific nodes -#define TB_NODE_GET_EXTRA(n) ((void*) n->extra) -#define TB_NODE_GET_EXTRA_T(n, T) ((T*) (n)->extra) -#define TB_NODE_SET_EXTRA(n, T, ...) (*((T*) (n)->extra) = (T){ __VA_ARGS__ }) - -typedef struct { // TB_BRANCH - uint64_t taken; - int64_t key; -} TB_BranchKey; - -// this represents switch (many targets), if (one target) -typedef struct { // TB_BRANCH - uint64_t total_hits; - size_t succ_count; - TB_BranchKey keys[]; -} TB_NodeBranch; - -typedef struct { // TB_PROJ - int index; -} TB_NodeProj; - -typedef struct { // TB_INTEGER_CONST - uint64_t value; -} TB_NodeInt; - -typedef struct { // any compare operator - TB_DataType cmp_dt; -} TB_NodeCompare; - -typedef struct { // any integer binary operator - TB_ArithmeticBehavior ab; -} TB_NodeBinopInt; - -typedef struct { - TB_CharUnits align; -} TB_NodeMemAccess; - -typedef struct { - int level; -} TB_NodePrefetch; - -typedef struct { - TB_CharUnits size, align; - int alias_index; // 0 if local is used beyond direct memops, 1...n as a unique alias name - - // dbg info - char* name; - TB_DebugType* type; -} TB_NodeLocal; - -typedef struct { - float value; -} TB_NodeFloat32; - -typedef struct { - double value; -} TB_NodeFloat64; - -typedef struct { - int64_t stride; -} TB_NodeArray; - -typedef struct { - int64_t offset; -} TB_NodeMember; - -typedef struct { - int alias_cnt; - int alias_idx[]; -} TB_NodeMemSplit; - -typedef struct { - TB_Symbol* sym; -} TB_NodeSymbol; - -typedef struct { - TB_MemoryOrder order; - TB_MemoryOrder order2; -} TB_NodeAtomic; - -typedef struct { - TB_FunctionPrototype* proto; - int proj_count; - TB_Node* projs[]; -} TB_NodeCall; - -typedef struct { - TB_FunctionPrototype* proto; -} TB_NodeTailcall; - -typedef struct { - void* tag; - int param_start; -} TB_NodeSafepoint; - -typedef struct { - const char* tag; - - // used for IR building - TB_Node *mem_in; -} TB_NodeRegion; - -typedef struct { - int64_t key; - uint64_t val; -} TB_LookupEntry; - -typedef struct { - size_t entry_count; - TB_LookupEntry entries[]; -} TB_NodeLookup; - -typedef struct TB_MultiOutput { - size_t count; - union { - // count = 1 - TB_Node* single; - // count > 1 - TB_Node** multiple; - }; -} TB_MultiOutput; -#define TB_MULTI_OUTPUT(o) ((o).count > 1 ? (o).multiple : &(o).single) - -typedef struct { - int64_t key; - TB_Node* value; -} TB_SwitchEntry; - -typedef enum { - TB_EXECUTABLE_UNKNOWN, - TB_EXECUTABLE_PE, - TB_EXECUTABLE_ELF, -} TB_ExecutableType; - -typedef struct { - TB_Node* node; // type == TB_SAFEPOINT - void* userdata; - - uint32_t ip; // relative to the function body. - uint32_t count; // same as node->input_count - int32_t values[]; -} TB_Safepoint; - -typedef enum { - TB_MODULE_SECTION_WRITE = 1, - TB_MODULE_SECTION_EXEC = 2, - TB_MODULE_SECTION_TLS = 4, -} TB_ModuleSectionFlags; - -typedef void (*TB_InlineAsmRA)(TB_Node* n, void* ctx); - -// This is the function that'll emit bytes from a TB_INLINE_ASM node -typedef size_t (*TB_InlineAsmEmit)(TB_Node* n, void* ctx, size_t out_cap, uint8_t* out); - -typedef struct { - void* ctx; - TB_InlineAsmRA ra; - TB_InlineAsmEmit emit; -} TB_NodeInlineAsm; - -// ******************************* -// Public macros -// ******************************* -#ifdef __cplusplus - -#define TB_TYPE_TUPLE TB_DataType{ { TB_TUPLE } } -#define TB_TYPE_CONTROL TB_DataType{ { TB_CONTROL } } -#define TB_TYPE_VOID TB_DataType{ { TB_INT, 0 } } -#define TB_TYPE_I8 TB_DataType{ { TB_INT, 8 } } -#define TB_TYPE_I16 TB_DataType{ { TB_INT, 16 } } -#define TB_TYPE_I32 TB_DataType{ { TB_INT, 32 } } -#define TB_TYPE_I64 TB_DataType{ { TB_INT, 64 } } -#define TB_TYPE_F32 TB_DataType{ { TB_FLOAT, TB_FLT_32 } } -#define TB_TYPE_F64 TB_DataType{ { TB_FLOAT, TB_FLT_64 } } -#define TB_TYPE_BOOL TB_DataType{ { TB_INT, 1 } } -#define TB_TYPE_PTR TB_DataType{ { TB_PTR, 0 } } -#define TB_TYPE_MEMORY TB_DataType{ { TB_MEMORY,0 } } -#define TB_TYPE_INTN(N) TB_DataType{ { TB_INT, (N) } } -#define TB_TYPE_PTRN(N) TB_DataType{ { TB_PTR, (N) } } - -#else - -#define TB_TYPE_TUPLE (TB_DataType){ { TB_TUPLE } } -#define TB_TYPE_CONTROL (TB_DataType){ { TB_CONTROL } } -#define TB_TYPE_VOID (TB_DataType){ { TB_INT, 0 } } -#define TB_TYPE_I8 (TB_DataType){ { TB_INT, 8 } } -#define TB_TYPE_I16 (TB_DataType){ { TB_INT, 16 } } -#define TB_TYPE_I32 (TB_DataType){ { TB_INT, 32 } } -#define TB_TYPE_I64 (TB_DataType){ { TB_INT, 64 } } -#define TB_TYPE_F32 (TB_DataType){ { TB_FLOAT, TB_FLT_32 } } -#define TB_TYPE_F64 (TB_DataType){ { TB_FLOAT, TB_FLT_64 } } -#define TB_TYPE_BOOL (TB_DataType){ { TB_INT, 1 } } -#define TB_TYPE_PTR (TB_DataType){ { TB_PTR, 0 } } -#define TB_TYPE_MEMORY (TB_DataType){ { TB_MEMORY,0 } } -#define TB_TYPE_INTN(N) (TB_DataType){ { TB_INT, (N) } } -#define TB_TYPE_PTRN(N) (TB_DataType){ { TB_PTR, (N) } } - -#endif - -typedef void (*TB_PrintCallback)(void* user_data, const char* fmt, ...); - -// defined in common/arena.h -typedef struct TB_Arena TB_Arena; - -// 0 for default -TB_API TB_Arena* tb_arena_create(size_t chunk_size); -TB_API void tb_arena_destroy(TB_Arena* restrict arena); -TB_API bool tb_arena_is_empty(TB_Arena* restrict arena); -TB_API void tb_arena_clear(TB_Arena* restrict arena); - -//////////////////////////////// -// Module management -//////////////////////////////// -// Creates a module with the correct target and settings -TB_API TB_Module* tb_module_create(TB_Arch arch, TB_System sys, bool is_jit); - -// Creates a module but defaults on the architecture and system based on the host machine -TB_API TB_Module* tb_module_create_for_host(bool is_jit); - -// Frees all resources for the TB_Module and it's functions, globals and -// compiled code. -TB_API void tb_module_destroy(TB_Module* m); - -// When targetting windows & thread local storage, you'll need to bind a tls index -// which is usually just a global that the runtime support has initialized, if you -// dont and the tls_index is used, it'll crash -TB_API void tb_module_set_tls_index(TB_Module* m, ptrdiff_t len, const char* name); - -TB_API TB_ModuleSectionHandle tb_module_create_section(TB_Module* m, ptrdiff_t len, const char* name, TB_ModuleSectionFlags flags, TB_ComdatType comdat); - -typedef struct { - TB_ThreadInfo* info; - size_t i; -} TB_SymbolIter; - -// Lovely iterator for all the symbols... it's probably not "fast" -TB_SymbolIter tb_symbol_iter(TB_Module* mod); -TB_Symbol* tb_symbol_iter_next(TB_SymbolIter* iter); - -//////////////////////////////// -// Compiled code introspection -//////////////////////////////// -enum { TB_ASSEMBLY_CHUNK_CAP = 4*1024 - sizeof(size_t[2]) }; - -typedef struct TB_Assembly TB_Assembly; -struct TB_Assembly { - TB_Assembly* next; - - // nice chunk of text here - size_t length; - char data[]; -}; - -// this is where the machine code and other relevant pieces go. -typedef struct TB_FunctionOutput TB_FunctionOutput; - -TB_API void tb_output_print_asm(TB_FunctionOutput* out, FILE* fp); - -TB_API uint8_t* tb_output_get_code(TB_FunctionOutput* out, size_t* out_length); - -// returns NULL if there's no line info -TB_API TB_Location* tb_output_get_locations(TB_FunctionOutput* out, size_t* out_count); - -// returns NULL if no assembly was generated -TB_API TB_Assembly* tb_output_get_asm(TB_FunctionOutput* out); - -// this is relative to the start of the function (the start of the prologue) -TB_API TB_Safepoint* tb_safepoint_get(TB_Function* f, uint32_t relative_ip); - -//////////////////////////////// -// JIT compilation -//////////////////////////////// -typedef struct TB_JIT TB_JIT; -typedef struct TB_CPUContext TB_CPUContext; - -// passing 0 to jit_heap_capacity will default to 4MiB -TB_API TB_JIT* tb_jit_begin(TB_Module* m, size_t jit_heap_capacity); -TB_API void* tb_jit_place_function(TB_JIT* jit, TB_Function* f); -TB_API void* tb_jit_place_global(TB_JIT* jit, TB_Global* g); -TB_API void* tb_jit_alloc_obj(TB_JIT* jit, size_t size, size_t align); -TB_API void tb_jit_free_obj(TB_JIT* jit, void* ptr); -TB_API void tb_jit_dump_heap(TB_JIT* jit); -TB_API void tb_jit_end(TB_JIT* jit); - -typedef struct { - void* tag; - uint32_t offset; -} TB_ResolvedAddr; - -TB_API void* tb_jit_resolve_addr(TB_JIT* jit, void* ptr, uint32_t* offset); -TB_API void* tb_jit_get_code_ptr(TB_Function* f); - -// you can take an tag an allocation, fresh space for random userdata :) -TB_API void tb_jit_tag_object(TB_JIT* jit, void* ptr, void* tag); - -// Debugger stuff -// creates a new context we can run JIT code in, you don't -// technically need this but it's a nice helper for writing -// JITs especially when it comes to breakpoints (and eventually -// safepoints) -TB_API TB_CPUContext* tb_jit_thread_create(TB_JIT* jit, size_t ud_size); -TB_API void* tb_jit_thread_get_userdata(TB_CPUContext* cpu); -TB_API void tb_jit_breakpoint(TB_JIT* jit, void* addr); - -// offsetof pollsite in the CPUContext -TB_API size_t tb_jit_thread_pollsite(void); - -// Only relevant when you're pausing the thread -TB_API void* tb_jit_thread_pc(TB_CPUContext* cpu); -TB_API void* tb_jit_thread_sp(TB_CPUContext* cpu); - -TB_API bool tb_jit_thread_call(TB_CPUContext* cpu, void* pc, uint64_t* ret, size_t arg_count, void** args); - -// returns true if we stepped off the end and returned through the trampoline -TB_API bool tb_jit_thread_step(TB_CPUContext* cpu, uint64_t* ret, uintptr_t pc_start, uintptr_t pc_end); - -//////////////////////////////// -// Disassembler -//////////////////////////////// -TB_API ptrdiff_t tb_print_disassembly_inst(TB_Arch arch, size_t length, const void* ptr); - -//////////////////////////////// -// Exporter -//////////////////////////////// -// Export buffers are generated in chunks because it's easier, usually the -// chunks are "massive" (representing some connected piece of the buffer) -// but they don't have to be. -typedef struct TB_ExportChunk TB_ExportChunk; -struct TB_ExportChunk { - TB_ExportChunk* next; - size_t pos, size; - uint8_t data[]; -}; - -typedef struct { - size_t total; - TB_ExportChunk *head, *tail; -} TB_ExportBuffer; - -TB_API TB_ExportBuffer tb_module_object_export(TB_Module* m, TB_Arena* dst_arena, TB_DebugFormat debug_fmt); -TB_API bool tb_export_buffer_to_file(TB_ExportBuffer buffer, const char* path); - -//////////////////////////////// -// Linker exporter -//////////////////////////////// -// This is used to export shared objects or executables -typedef struct TB_Linker TB_Linker; -typedef struct TB_LinkerSection TB_LinkerSection; -typedef struct TB_LinkerSectionPiece TB_LinkerSectionPiece; - -typedef struct { - enum { - TB_LINKER_MSG_NULL, - - // pragma comment(lib, "blah") - TB_LINKER_MSG_IMPORT, - } tag; - union { - // pragma lib request - TB_Slice import_path; - }; -} TB_LinkerMsg; - -TB_API TB_ExecutableType tb_system_executable_format(TB_System s); - -TB_API TB_Linker* tb_linker_create(TB_ExecutableType type, TB_Arch arch); -TB_API TB_ExportBuffer tb_linker_export(TB_Linker* l, TB_Arena* arena); -TB_API void tb_linker_destroy(TB_Linker* l); - -TB_API bool tb_linker_get_msg(TB_Linker* l, TB_LinkerMsg* msg); - -// windows only -TB_API void tb_linker_set_subsystem(TB_Linker* l, TB_WindowsSubsystem subsystem); - -TB_API void tb_linker_set_entrypoint(TB_Linker* l, const char* name); - -// Links compiled module into output -TB_API void tb_linker_append_module(TB_Linker* l, TB_Module* m); - -// Adds object file to output -TB_API void tb_linker_append_object(TB_Linker* l, TB_Slice obj_name, TB_Slice content); - -// Adds static library to output -// this can include imports (wrappers for DLL symbols) along with -// normal sections. -TB_API void tb_linker_append_library(TB_Linker* l, TB_Slice ar_name, TB_Slice content); - -//////////////////////////////// -// Symbols -//////////////////////////////// -TB_API bool tb_extern_resolve(TB_External* e, TB_Symbol* sym); -TB_API TB_External* tb_extern_create(TB_Module* m, ptrdiff_t len, const char* name, TB_ExternalType type); - -TB_API TB_SourceFile* tb_get_source_file(TB_Module* m, ptrdiff_t len, const char* path); - -// Called once you're done with TB operations on a thread (or i guess when it's -// about to be killed :p), not calling it can only result in leaks on that thread -// and calling it too early will result in TB potentially reallocating it but there's -// should be no crashes from this, just potential slowdown or higher than expected memory -// usage. -TB_API void tb_free_thread_resources(void); - -//////////////////////////////// -// Function Prototypes -//////////////////////////////// -typedef struct TB_PrototypeParam { - TB_DataType dt; - TB_DebugType* debug_type; - - // does not apply for returns - const char* name; -} TB_PrototypeParam; - -struct TB_FunctionPrototype { - // header - TB_CallingConv call_conv; - uint16_t return_count, param_count; - bool has_varargs; - - // params are directly followed by returns - TB_PrototypeParam params[]; -}; -#define TB_PROTOTYPE_RETURNS(p) ((p)->params + (p)->param_count) - -// creates a function prototype used to define a function's parameters and returns. -// -// function prototypes do not get freed individually and last for the entire run -// of the backend, they can also be reused for multiple functions which have -// matching signatures. -TB_API TB_FunctionPrototype* tb_prototype_create(TB_Module* m, TB_CallingConv cc, size_t param_count, const TB_PrototypeParam* params, size_t return_count, const TB_PrototypeParam* returns, bool has_varargs); - -// same as tb_function_set_prototype except it will handle lowering from types like the TB_DebugType -// into the correct ABI and exposing sane looking nodes to the parameters. -// -// returns the parameters -TB_API TB_Node** tb_function_set_prototype_from_dbg(TB_Function* f, TB_ModuleSectionHandle section, TB_DebugType* dbg, TB_Arena* arena, size_t* out_param_count); -TB_API TB_FunctionPrototype* tb_prototype_from_dbg(TB_Module* m, TB_DebugType* dbg); - -// used for ABI parameter passing -typedef enum { - // needs a direct value - TB_PASSING_DIRECT, - - // needs an address to the value - TB_PASSING_INDIRECT, - - // doesn't use this parameter - TB_PASSING_IGNORE, -} TB_PassingRule; - -TB_API TB_PassingRule tb_get_passing_rule_from_dbg(TB_Module* mod, TB_DebugType* param_type, bool is_return); - -//////////////////////////////// -// Globals -//////////////////////////////// -TB_API TB_Global* tb_global_create(TB_Module* m, ptrdiff_t len, const char* name, TB_DebugType* dbg_type, TB_Linkage linkage); - -// allocate space for the global -TB_API void tb_global_set_storage(TB_Module* m, TB_ModuleSectionHandle section, TB_Global* global, size_t size, size_t align, size_t max_objects); - -// returns a buffer which the user can fill to then have represented in the initializer -TB_API void* tb_global_add_region(TB_Module* m, TB_Global* global, size_t offset, size_t size); - -// places a relocation for a global at offset, the size of the relocation -// depends on the pointer size -TB_API void tb_global_add_symbol_reloc(TB_Module* m, TB_Global* global, size_t offset, TB_Symbol* symbol); - -TB_API TB_ModuleSectionHandle tb_module_get_text(TB_Module* m); -TB_API TB_ModuleSectionHandle tb_module_get_rdata(TB_Module* m); -TB_API TB_ModuleSectionHandle tb_module_get_data(TB_Module* m); -TB_API TB_ModuleSectionHandle tb_module_get_tls(TB_Module* m); - -//////////////////////////////// -// Function Attributes -//////////////////////////////// -// These are parts of a function that describe metadata for instructions -TB_API void tb_function_attrib_variable(TB_Function* f, TB_Node* n, TB_Node* parent, ptrdiff_t len, const char* name, TB_DebugType* type); -TB_API void tb_function_attrib_scope(TB_Function* f, TB_Node* n, TB_Node* parent); - -//////////////////////////////// -// Debug info Generation -//////////////////////////////// -TB_API TB_DebugType* tb_debug_get_void(TB_Module* m); -TB_API TB_DebugType* tb_debug_get_bool(TB_Module* m); -TB_API TB_DebugType* tb_debug_get_integer(TB_Module* m, bool is_signed, int bits); -TB_API TB_DebugType* tb_debug_get_float(TB_Module* m, TB_FloatFormat fmt); -TB_API TB_DebugType* tb_debug_create_ptr(TB_Module* m, TB_DebugType* base); -TB_API TB_DebugType* tb_debug_create_array(TB_Module* m, TB_DebugType* base, size_t count); -TB_API TB_DebugType* tb_debug_create_alias(TB_Module* m, TB_DebugType* base, ptrdiff_t len, const char* tag); -TB_API TB_DebugType* tb_debug_create_struct(TB_Module* m, ptrdiff_t len, const char* tag); -TB_API TB_DebugType* tb_debug_create_union(TB_Module* m, ptrdiff_t len, const char* tag); -TB_API TB_DebugType* tb_debug_create_field(TB_Module* m, TB_DebugType* type, ptrdiff_t len, const char* name, TB_CharUnits offset); - -// returns the array you need to fill with fields -TB_API TB_DebugType** tb_debug_record_begin(TB_Module* m, TB_DebugType* type, size_t count); -TB_API void tb_debug_record_end(TB_DebugType* type, TB_CharUnits size, TB_CharUnits align); - -TB_API TB_DebugType* tb_debug_create_func(TB_Module* m, TB_CallingConv cc, size_t param_count, size_t return_count, bool has_varargs); - -TB_API TB_DebugType* tb_debug_field_type(TB_DebugType* type); - -TB_API size_t tb_debug_func_return_count(TB_DebugType* type); -TB_API size_t tb_debug_func_param_count(TB_DebugType* type); - -// you'll need to fill these if you make a function -TB_API TB_DebugType** tb_debug_func_params(TB_DebugType* type); -TB_API TB_DebugType** tb_debug_func_returns(TB_DebugType* type); - -//////////////////////////////// -// Symbols -//////////////////////////////// -// returns NULL if the tag doesn't match -TB_API TB_Function* tb_symbol_as_function(TB_Symbol* s); -TB_API TB_External* tb_symbol_as_external(TB_Symbol* s); -TB_API TB_Global* tb_symbol_as_global(TB_Symbol* s); - -//////////////////////////////// -// Function IR Generation -//////////////////////////////// -TB_API void tb_get_data_type_size(TB_Module* mod, TB_DataType dt, size_t* size, size_t* align); - -TB_API void tb_inst_location(TB_Function* f, TB_SourceFile* file, int line, int column); - -// this is where the STOP will be -TB_API void tb_inst_set_exit_location(TB_Function* f, TB_SourceFile* file, int line, int column); - -// if section is NULL, default to .text -TB_API TB_Function* tb_function_create(TB_Module* m, ptrdiff_t len, const char* name, TB_Linkage linkage); - -TB_API TB_Arena* tb_function_get_arena(TB_Function* f); - -// if len is -1, it's null terminated -TB_API void tb_symbol_set_name(TB_Symbol* s, ptrdiff_t len, const char* name); - -TB_API void tb_symbol_bind_ptr(TB_Symbol* s, void* ptr); -TB_API const char* tb_symbol_get_name(TB_Symbol* s); - -// if arena is NULL, defaults to module arena which is freed on tb_free_thread_resources -TB_API void tb_function_set_prototype(TB_Function* f, TB_ModuleSectionHandle section, TB_FunctionPrototype* p, TB_Arena* arena); -TB_API TB_FunctionPrototype* tb_function_get_prototype(TB_Function* f); - -// if len is -1, it's null terminated -TB_API void tb_inst_set_region_name(TB_Function* f, TB_Node* n, ptrdiff_t len, const char* name); - -TB_API TB_Node* tb_inst_poison(TB_Function* f, TB_DataType dt); - -TB_API TB_Node* tb_inst_root_node(TB_Function* f); -TB_API TB_Node* tb_inst_param(TB_Function* f, int param_id); - -TB_API TB_Node* tb_inst_fpxt(TB_Function* f, TB_Node* src, TB_DataType dt); -TB_API TB_Node* tb_inst_sxt(TB_Function* f, TB_Node* src, TB_DataType dt); -TB_API TB_Node* tb_inst_zxt(TB_Function* f, TB_Node* src, TB_DataType dt); -TB_API TB_Node* tb_inst_trunc(TB_Function* f, TB_Node* src, TB_DataType dt); -TB_API TB_Node* tb_inst_int2ptr(TB_Function* f, TB_Node* src); -TB_API TB_Node* tb_inst_ptr2int(TB_Function* f, TB_Node* src, TB_DataType dt); -TB_API TB_Node* tb_inst_int2float(TB_Function* f, TB_Node* src, TB_DataType dt, bool is_signed); -TB_API TB_Node* tb_inst_float2int(TB_Function* f, TB_Node* src, TB_DataType dt, bool is_signed); -TB_API TB_Node* tb_inst_bitcast(TB_Function* f, TB_Node* src, TB_DataType dt); - -TB_API TB_Node* tb_inst_local(TB_Function* f, TB_CharUnits size, TB_CharUnits align); - -TB_API TB_Node* tb_inst_load(TB_Function* f, TB_DataType dt, TB_Node* addr, TB_CharUnits align, bool is_volatile); -TB_API void tb_inst_store(TB_Function* f, TB_DataType dt, TB_Node* addr, TB_Node* val, TB_CharUnits align, bool is_volatile); - -TB_API void tb_inst_safepoint_poll(TB_Function* f, void* tag, TB_Node* addr, int input_count, TB_Node** inputs); - -TB_API TB_Node* tb_inst_bool(TB_Function* f, bool imm); -TB_API TB_Node* tb_inst_sint(TB_Function* f, TB_DataType dt, int64_t imm); -TB_API TB_Node* tb_inst_uint(TB_Function* f, TB_DataType dt, uint64_t imm); -TB_API TB_Node* tb_inst_float32(TB_Function* f, float imm); -TB_API TB_Node* tb_inst_float64(TB_Function* f, double imm); -TB_API TB_Node* tb_inst_cstring(TB_Function* f, const char* str); -TB_API TB_Node* tb_inst_string(TB_Function* f, size_t len, const char* str); - -// write 'val' over 'count' bytes on 'dst' -TB_API void tb_inst_memset(TB_Function* f, TB_Node* dst, TB_Node* val, TB_Node* count, TB_CharUnits align); - -// zero 'count' bytes on 'dst' -TB_API void tb_inst_memzero(TB_Function* f, TB_Node* dst, TB_Node* count, TB_CharUnits align); - -// performs a copy of 'count' elements from one memory location to another -// both locations cannot overlap. -TB_API void tb_inst_memcpy(TB_Function* f, TB_Node* dst, TB_Node* src, TB_Node* count, TB_CharUnits align); - -// result = base + (index * stride) -TB_API TB_Node* tb_inst_array_access(TB_Function* f, TB_Node* base, TB_Node* index, int64_t stride); - -// result = base + offset -// where base is a pointer -TB_API TB_Node* tb_inst_member_access(TB_Function* f, TB_Node* base, int64_t offset); - -TB_API TB_Node* tb_inst_get_symbol_address(TB_Function* f, TB_Symbol* target); - -// Performs a conditional select between two values, if the operation is -// performed wide then the cond is expected to be the same type as a and b where -// the condition is resolved as true if the MSB (per component) is 1. -// -// result = cond ? a : b -// a, b must match in type -TB_API TB_Node* tb_inst_select(TB_Function* f, TB_Node* cond, TB_Node* a, TB_Node* b); - -// Integer arithmatic -TB_API TB_Node* tb_inst_add(TB_Function* f, TB_Node* a, TB_Node* b, TB_ArithmeticBehavior arith_behavior); -TB_API TB_Node* tb_inst_sub(TB_Function* f, TB_Node* a, TB_Node* b, TB_ArithmeticBehavior arith_behavior); -TB_API TB_Node* tb_inst_mul(TB_Function* f, TB_Node* a, TB_Node* b, TB_ArithmeticBehavior arith_behavior); -TB_API TB_Node* tb_inst_div(TB_Function* f, TB_Node* a, TB_Node* b, bool signedness); -TB_API TB_Node* tb_inst_mod(TB_Function* f, TB_Node* a, TB_Node* b, bool signedness); - -// Bitmagic operations -TB_API TB_Node* tb_inst_bswap(TB_Function* f, TB_Node* n); -TB_API TB_Node* tb_inst_clz(TB_Function* f, TB_Node* n); -TB_API TB_Node* tb_inst_ctz(TB_Function* f, TB_Node* n); -TB_API TB_Node* tb_inst_popcount(TB_Function* f, TB_Node* n); - -// Bitwise operations -TB_API TB_Node* tb_inst_not(TB_Function* f, TB_Node* n); -TB_API TB_Node* tb_inst_neg(TB_Function* f, TB_Node* n); -TB_API TB_Node* tb_inst_and(TB_Function* f, TB_Node* a, TB_Node* b); -TB_API TB_Node* tb_inst_or(TB_Function* f, TB_Node* a, TB_Node* b); -TB_API TB_Node* tb_inst_xor(TB_Function* f, TB_Node* a, TB_Node* b); -TB_API TB_Node* tb_inst_sar(TB_Function* f, TB_Node* a, TB_Node* b); -TB_API TB_Node* tb_inst_shl(TB_Function* f, TB_Node* a, TB_Node* b, TB_ArithmeticBehavior arith_behavior); -TB_API TB_Node* tb_inst_shr(TB_Function* f, TB_Node* a, TB_Node* b); -TB_API TB_Node* tb_inst_rol(TB_Function* f, TB_Node* a, TB_Node* b); -TB_API TB_Node* tb_inst_ror(TB_Function* f, TB_Node* a, TB_Node* b); - -// Atomics -// By default you can use TB_MEM_ORDER_SEQ_CST for the memory order to get -// correct but possibly slower results on certain platforms (those with relaxed -// memory models). - -// Must be aligned to the natural alignment of dt -TB_API TB_Node* tb_inst_atomic_load(TB_Function* f, TB_Node* addr, TB_DataType dt, TB_MemoryOrder order); - -// All atomic operations here return the old value and the operations are -// performed in the same data type as 'src' with alignment of 'addr' being -// the natural alignment of 'src' -TB_API TB_Node* tb_inst_atomic_xchg(TB_Function* f, TB_Node* addr, TB_Node* src, TB_MemoryOrder order); -TB_API TB_Node* tb_inst_atomic_add(TB_Function* f, TB_Node* addr, TB_Node* src, TB_MemoryOrder order); -TB_API TB_Node* tb_inst_atomic_sub(TB_Function* f, TB_Node* addr, TB_Node* src, TB_MemoryOrder order); -TB_API TB_Node* tb_inst_atomic_and(TB_Function* f, TB_Node* addr, TB_Node* src, TB_MemoryOrder order); -TB_API TB_Node* tb_inst_atomic_xor(TB_Function* f, TB_Node* addr, TB_Node* src, TB_MemoryOrder order); -TB_API TB_Node* tb_inst_atomic_or(TB_Function* f, TB_Node* addr, TB_Node* src, TB_MemoryOrder order); - -// returns old_value from *addr -TB_API TB_Node* tb_inst_atomic_cmpxchg(TB_Function* f, TB_Node* addr, TB_Node* expected, TB_Node* desired, TB_MemoryOrder succ, TB_MemoryOrder fail); - -// Float math -TB_API TB_Node* tb_inst_fadd(TB_Function* f, TB_Node* a, TB_Node* b); -TB_API TB_Node* tb_inst_fsub(TB_Function* f, TB_Node* a, TB_Node* b); -TB_API TB_Node* tb_inst_fmul(TB_Function* f, TB_Node* a, TB_Node* b); -TB_API TB_Node* tb_inst_fdiv(TB_Function* f, TB_Node* a, TB_Node* b); - -// Comparisons -TB_API TB_Node* tb_inst_cmp_eq(TB_Function* f, TB_Node* a, TB_Node* b); -TB_API TB_Node* tb_inst_cmp_ne(TB_Function* f, TB_Node* a, TB_Node* b); - -TB_API TB_Node* tb_inst_cmp_ilt(TB_Function* f, TB_Node* a, TB_Node* b, bool signedness); -TB_API TB_Node* tb_inst_cmp_ile(TB_Function* f, TB_Node* a, TB_Node* b, bool signedness); -TB_API TB_Node* tb_inst_cmp_igt(TB_Function* f, TB_Node* a, TB_Node* b, bool signedness); -TB_API TB_Node* tb_inst_cmp_ige(TB_Function* f, TB_Node* a, TB_Node* b, bool signedness); - -TB_API TB_Node* tb_inst_cmp_flt(TB_Function* f, TB_Node* a, TB_Node* b); -TB_API TB_Node* tb_inst_cmp_fle(TB_Function* f, TB_Node* a, TB_Node* b); -TB_API TB_Node* tb_inst_cmp_fgt(TB_Function* f, TB_Node* a, TB_Node* b); -TB_API TB_Node* tb_inst_cmp_fge(TB_Function* f, TB_Node* a, TB_Node* b); - -// General intrinsics -TB_API TB_Node* tb_inst_va_start(TB_Function* f, TB_Node* a); -TB_API TB_Node* tb_inst_cycle_counter(TB_Function* f); -TB_API TB_Node* tb_inst_prefetch(TB_Function* f, TB_Node* addr, int level); - -// x86 Intrinsics -TB_API TB_Node* tb_inst_x86_ldmxcsr(TB_Function* f, TB_Node* a); -TB_API TB_Node* tb_inst_x86_stmxcsr(TB_Function* f); -TB_API TB_Node* tb_inst_x86_sqrt(TB_Function* f, TB_Node* a); -TB_API TB_Node* tb_inst_x86_rsqrt(TB_Function* f, TB_Node* a); - -// Control flow -// trace is a single-entry piece of IR. -typedef struct { - TB_Node* top_ctrl; - TB_Node* bot_ctrl; - - // latest memory effect, for now there's - // only one stream going at a time but that'll - // have to change for some of the interesting - // langs later. - TB_Node* mem; -} TB_Trace; - -// Old-style uses regions for all control flow similar to how people use basic blocks -TB_API TB_Node* tb_inst_region(TB_Function* f); -TB_API void tb_inst_set_control(TB_Function* f, TB_Node* region); -TB_API TB_Node* tb_inst_get_control(TB_Function* f); - -// But since regions aren't basic blocks (they only guarentee single entry, not single exit) -// the new-style is built for that. -TB_API TB_Trace tb_inst_new_trace(TB_Function* f); -TB_API void tb_inst_set_trace(TB_Function* f, TB_Trace trace); -TB_API TB_Trace tb_inst_get_trace(TB_Function* f); - -// only works on regions which haven't been constructed yet -TB_API TB_Trace tb_inst_trace_from_region(TB_Function* f, TB_Node* region); -TB_API TB_Node* tb_inst_region_mem_in(TB_Function* f, TB_Node* region); - -TB_API TB_Node* tb_inst_syscall(TB_Function* f, TB_DataType dt, TB_Node* syscall_num, size_t param_count, TB_Node** params); -TB_API TB_MultiOutput tb_inst_call(TB_Function* f, TB_FunctionPrototype* proto, TB_Node* target, size_t param_count, TB_Node** params); -TB_API void tb_inst_tailcall(TB_Function* f, TB_FunctionPrototype* proto, TB_Node* target, size_t param_count, TB_Node** params); - -TB_API TB_Node* tb_inst_safepoint(TB_Function* f, TB_Node* poke_site, size_t param_count, TB_Node** params); - -TB_API TB_Node* tb_inst_incomplete_phi(TB_Function* f, TB_DataType dt, TB_Node* region, size_t preds); -TB_API bool tb_inst_add_phi_operand(TB_Function* f, TB_Node* phi, TB_Node* region, TB_Node* val); - -TB_API TB_Node* tb_inst_phi2(TB_Function* f, TB_Node* region, TB_Node* a, TB_Node* b); -TB_API void tb_inst_goto(TB_Function* f, TB_Node* target); -TB_API TB_Node* tb_inst_if(TB_Function* f, TB_Node* cond, TB_Node* true_case, TB_Node* false_case); -TB_API TB_Node* tb_inst_branch(TB_Function* f, TB_DataType dt, TB_Node* key, TB_Node* default_case, size_t entry_count, const TB_SwitchEntry* keys); -TB_API void tb_inst_unreachable(TB_Function* f); -TB_API void tb_inst_debugbreak(TB_Function* f); -TB_API void tb_inst_trap(TB_Function* f); - -// revised API for if, this one returns the control projections such that a target is not necessary while building -// projs[0] is the true case, projs[1] is false. -TB_API TB_Node* tb_inst_if2(TB_Function* f, TB_Node* cond, TB_Node* projs[2]); - -// n is a TB_BRANCH with two successors, taken is the number of times it's true -TB_API void tb_inst_set_branch_freq(TB_Function* f, TB_Node* n, int total_hits, int taken); - -TB_API void tb_inst_ret(TB_Function* f, size_t count, TB_Node** values); - -//////////////////////////////// -// Cooler IR building -//////////////////////////////// -typedef struct TB_GraphBuilder TB_GraphBuilder; -enum { TB_GRAPH_BUILDER_PARAMS = 0 }; - -// arena isn't for the function, it's for the builder -TB_API TB_GraphBuilder* tb_builder_enter(TB_Function* f, TB_Arena* arena); -TB_API void tb_builder_exit(TB_GraphBuilder* g); - -// sometimes you wanna make a scope and have some shit in there... use this -TB_API int tb_builder_save(TB_GraphBuilder* g); -TB_API void tb_builder_restore(TB_GraphBuilder* g, int v); - -TB_API void tb_builder_push(TB_GraphBuilder* g, TB_Node* n); -TB_API TB_Node* tb_builder_pop(TB_GraphBuilder* g); - -TB_API void tb_builder_debugbreak(TB_GraphBuilder* g); - -// ( -- a ) -TB_API void tb_builder_uint(TB_GraphBuilder* g, TB_DataType dt, uint64_t x); -TB_API void tb_builder_sint(TB_GraphBuilder* g, TB_DataType dt, int64_t x); -TB_API void tb_builder_float32(TB_GraphBuilder* g, float imm); -TB_API void tb_builder_float64(TB_GraphBuilder* g, double imm); -TB_API void tb_builder_string(TB_GraphBuilder* g, ptrdiff_t len, const char* str); - -// ( a b -- c ) -// -// works with type: AND, OR, XOR, ADD, SUB, MUL, SHL, SHR, SAR, ROL, ROR, UDIV, SDIV, UMOD, SMOD. -// note that arithmetic behavior is irrelevant for some of the operations (but 0 is always a good default). -TB_API void tb_builder_binop_int(TB_GraphBuilder* g, int type, TB_ArithmeticBehavior ab); - -TB_API void tb_builder_cast(TB_GraphBuilder* g, TB_DataType dt, int type); - -// ( a b -- c ) -TB_API void tb_builder_cmp(TB_GraphBuilder* g, int type, bool flip, TB_DataType dt); - -// pointer arithmetic -// ( a b -- c ) => a + b*stride -TB_API void tb_builder_array(TB_GraphBuilder* g, int64_t stride); -// ( a -- c ) => a + offset -TB_API void tb_builder_member(TB_GraphBuilder* g, int64_t offset); - -// memory -// ( addr -- val ) -TB_API void tb_builder_load(TB_GraphBuilder* g, int mem_var, bool ctrl_dep, TB_DataType dt, int32_t offset, TB_CharUnits align); -// ( addr val -- ) -TB_API void tb_builder_store(TB_GraphBuilder* g, int mem_var, int32_t offset, TB_CharUnits align); - -// function call -// ( ... -- ... ) -TB_API void tb_builder_static_call(TB_GraphBuilder* g, TB_FunctionPrototype* proto, TB_Symbol* target, int mem_var, int nargs); - -// locals (variables but as stack vars) -TB_API TB_Node* tb_builder_local(TB_GraphBuilder* g, TB_CharUnits size, TB_CharUnits align); -TB_API void tb_builder_push(TB_GraphBuilder* g, TB_Node* n); -TB_API TB_Node* tb_builder_pop(TB_GraphBuilder* g); - -// variables (these are just named stack slots) -// ( a -- ) -// -// we take the top item in the stack and treat it as a -// variable we'll might later fiddle with, the control -// flow primitives will diff changes to insert phis. -TB_API int tb_builder_decl(TB_GraphBuilder* g); -// ( -- a ) -TB_API void tb_builder_get_var(TB_GraphBuilder* g, int id); -// ( a -- ) -TB_API void tb_builder_set_var(TB_GraphBuilder* g, int id); - -// control flow primitives -// ( a -- ) -// -// taken is the number of times we do the true case -TB_API void tb_builder_if(TB_GraphBuilder* g, int total_hits, int taken); -// ( -- ) -TB_API void tb_builder_else(TB_GraphBuilder* g); -// ( -- ) -TB_API void tb_builder_endif(TB_GraphBuilder* g); -// ( ... -- ) -// -// technically TB has multiple returns, in practice it's like 2 regs before -// ABI runs out of shit. -TB_API void tb_builder_ret(TB_GraphBuilder* g, int count); -// ( a -- ) -TB_API void tb_builder_br(TB_GraphBuilder* g, int depth); -// ( a -- ) -TB_API void tb_builder_br_if(TB_GraphBuilder* g, int depth); -// ( -- ) -TB_API void tb_builder_loop(TB_GraphBuilder* g); -// ( -- ) -TB_API void tb_builder_endloop(TB_GraphBuilder* g); -// ( -- ) -TB_API void tb_builder_block(TB_GraphBuilder* g); -// ( -- ) -TB_API void tb_builder_endblock(TB_GraphBuilder* g); - -//////////////////////////////// -// Passes -//////////////////////////////// -// Function analysis, optimizations, and codegen are all part of this -typedef struct TB_Passes TB_Passes; - -// the arena is used to allocate the nodes while passes are being done. -TB_API TB_Passes* tb_pass_enter(TB_Function* f, TB_Arena* arena); -TB_API void tb_pass_exit(TB_Passes* p); - -// allocates peephole datastructures, necessarily if you wanna run the peephole optimizer -// during IR construction. -TB_API void tb_pass_prep(TB_Passes* p); - -// this is the peephole optimizer in a form you can run during IR construction. -TB_API TB_Node* tb_pass_peephole_node(TB_Passes* p, TB_Node* n); - -// returns GVN on a new node, returning either the same node or a duplicate node 'k'. -// it deletes 'n' if it's a duplicate btw. -TB_API TB_Node* tb_pass_gvn_node(TB_Function* f, TB_Node* n); - -// transformation passes: -// peephole: 90% of the optimizer, i'm sea of nodes pilled so i -// break down most optimizations into local rewrites, it's -// also incremental so spamming it doesn't really do anything and -// if placing all the nodes back onto the worklist caused progress -// that's a bug that should be reported. -TB_API void tb_pass_peephole(TB_Passes* p); -TB_API void tb_pass_locals(TB_Passes* p); -TB_API void tb_pass_loop(TB_Passes* p); - -// this just runs the optimizer in the default configuration -TB_API void tb_pass_optimize(TB_Passes* p); - -// analysis -// print: prints IR in a flattened text form. -TB_API void tb_pass_print(TB_Passes* opt); -TB_API char *tb_pass_c_prelude(TB_Module *mod); -TB_API char *tb_pass_c_fmt(TB_Passes* opt); -// print-dot: prints IR as DOT -TB_API void tb_pass_print_dot(TB_Passes* opt, TB_PrintCallback callback, void* user_data); - -// codegen: -// output goes at the top of the code_arena, feel free to place multiple functions -// into the same code arena (although arenas aren't thread-safe you'll want one per thread -// at least) -TB_API TB_FunctionOutput* tb_pass_codegen(TB_Passes* opt, TB_Arena* code_arena, const TB_FeatureSet* features, bool emit_asm); - -// interprocedural optimizer iter -TB_API bool tb_module_ipo(TB_Module* m); - -TB_API void tb_pass_kill_node(TB_Function* f, TB_Node* n); -TB_API void tb_pass_mark(TB_Passes* opt, TB_Node* n); -TB_API void tb_pass_mark_users(TB_Passes* opt, TB_Node* n); - -//////////////////////////////// -// Helpers -//////////////////////////////// -// the user_data is expected to be a valid FILE* -TB_API void tb_default_print_callback(void* user_data, const char* fmt, ...); - -// p is optional -TB_API void tb_dumb_print(TB_Function* f, TB_Passes* p); - -//////////////////////////////// -// IR access -//////////////////////////////// -TB_API bool tb_node_is_constant_non_zero(TB_Node* n); -TB_API bool tb_node_is_constant_zero(TB_Node* n); - -#endif /* TB_CORE_H */ diff --git a/vendor/tb/include/tb_coff.h b/vendor/tb/include/tb_coff.h deleted file mode 100644 index 5492dfc2..00000000 --- a/vendor/tb/include/tb_coff.h +++ /dev/null @@ -1,334 +0,0 @@ -// PE/COFF is the executable/object format used by Microsoft. -#ifndef TB_COFF_H -#define TB_COFF_H - -#include "tb_formats.h" - -#define IMAGE_SCN_LNK_NRELOC_OVFL 0x01000000 - -#define IMAGE_SYM_CLASS_EXTERNAL 0x0002 -#define IMAGE_SYM_CLASS_STATIC 0x0003 -#define IMAGE_SYM_CLASS_LABEL 0x0006 -#define IMAGE_SYM_CLASS_FILE 0x0067 -#define IMAGE_SYM_CLASS_SECTION 0x0068 -#define IMAGE_SYM_CLASS_WEAK_EXTERNAL 0x0069 - -#define IMAGE_FILE_LINE_NUMS_STRIPPED 0x0004 - -#define IMAGE_REL_AMD64_ADDR64 0x0001 -#define IMAGE_REL_AMD64_ADDR32 0x0002 -#define IMAGE_REL_AMD64_ADDR32NB 0x0003 -#define IMAGE_REL_AMD64_REL32 0x0004 -#define IMAGE_REL_AMD64_REL32_1 0x0005 -#define IMAGE_REL_AMD64_REL32_2 0x0006 -#define IMAGE_REL_AMD64_REL32_3 0x0007 -#define IMAGE_REL_AMD64_REL32_4 0x0008 -#define IMAGE_REL_AMD64_REL32_5 0x0009 -#define IMAGE_REL_AMD64_SECTION 0x000A -#define IMAGE_REL_AMD64_SECREL 0x000B - -#define IMAGE_SCN_LNK_REMOVE 0x00000800 -#define IMAGE_SCN_LNK_COMDAT 0x00001000 -#define IMAGE_SCN_MEM_DISCARDABLE 0x02000000 -#define IMAGE_SCN_MEM_EXECUTE 0x20000000 -#define IMAGE_SCN_MEM_READ 0x40000000 -#define IMAGE_SCN_MEM_WRITE 0x80000000 - -#define IMAGE_SCN_CNT_CODE 0x00000020 /* Section contains code. */ -#define IMAGE_SCN_CNT_INITIALIZED_DATA 0x00000040 /* Section contains initialized data. */ -#define IMAGE_SCN_CNT_UNINITIALIZED_DATA 0x00000080 /* Section contains uninitialized data. */ - -#define IMAGE_DIRECTORY_ENTRY_EXPORT 0 // Export Directory -#define IMAGE_DIRECTORY_ENTRY_IMPORT 1 // Import Directory -#define IMAGE_DIRECTORY_ENTRY_RESOURCE 2 // Resource Directory -#define IMAGE_DIRECTORY_ENTRY_EXCEPTION 3 // Exception Directory -#define IMAGE_DIRECTORY_ENTRY_SECURITY 4 // Security Directory -#define IMAGE_DIRECTORY_ENTRY_BASERELOC 5 // Base Relocation Table -#define IMAGE_DIRECTORY_ENTRY_DEBUG 6 // Debug Directory -#define IMAGE_DIRECTORY_ENTRY_ARCHITECTURE 7 // Architecture Specific Data -#define IMAGE_DIRECTORY_ENTRY_GLOBALPTR 8 // RVA of GP -#define IMAGE_DIRECTORY_ENTRY_TLS 9 // TLS Directory -#define IMAGE_DIRECTORY_ENTRY_LOAD_CONFIG 10 // Load Configuration Directory -#define IMAGE_DIRECTORY_ENTRY_BOUND_IMPORT 11 // Bound Import Directory in headers -#define IMAGE_DIRECTORY_ENTRY_IAT 12 // Import Address Table -#define IMAGE_DIRECTORY_ENTRY_DELAY_IMPORT 13 // Delay Load Import Descriptors -#define IMAGE_DIRECTORY_ENTRY_COM_DESCRIPTOR 14 // COM Runtime descriptor - -#define IMAGE_SUBSYSTEM_WINDOWS_GUI 2 -#define IMAGE_SUBSYSTEM_WINDOWS_CUI 3 -#define IMAGE_SUBSYSTEM_EFI_APPLICATION 10 - -typedef enum { - TB_COFF_SECTION_NO_PAD = 0x00000008, - TB_COFF_SECTION_CODE = 0x00000020, - TB_COFF_SECTION_INIT = 0x00000040, - TB_COFF_SECTION_UNINIT = 0x00000080, - TB_COFF_SECTION_OTHER = 0x00000100, - TB_COFF_SECTION_INFO = 0x00000200, - TB_COFF_SECTION_REMOVE = 0x00000800, - TB_COFF_SECTION_COMDAT = 0x00001000, - - // this is actually a 4bit field - TB_COFF_SECTION_ALIGN = 0x00F00000, - - // if we have more than 65535 relocations we do this - TB_COFF_SECTION_RELOC_OVR = 0x00F00000, - - // memory flags - TB_COFF_SECTION_DISCARDABLE = 0x02000000, - TB_COFF_SECTION_NOT_CACHED = 0x04000000, - TB_COFF_SECTION_NOT_PAGED = 0x08000000, - TB_COFF_SECTION_SHARED = 0x10000000, - TB_COFF_SECTION_EXECUTE = 0x20000000, - TB_COFF_SECTION_READ = 0x40000000, - TB_COFF_SECTION_WRITE = 0x80000000, -} TB_COFF_SectionFlags; - -typedef struct TB_COFF_Parser { - // inputs - TB_Slice name, file; - - // results - size_t section_count; - size_t symbol_table, symbol_count; - - // private - TB_Slice string_table; -} TB_COFF_Parser; - -// fills the parser with results from the COFF header -bool tb_coff_parse_init(TB_COFF_Parser* restrict parser); -bool tb_coff_parse_section(TB_COFF_Parser* restrict parser, size_t i, TB_ObjectSection* out_sec); - -// how many symbols does this one symbol take up (basically 1 + aux symbols). -// returns 0 if error. -size_t tb_coff_parse_symbol(TB_COFF_Parser* restrict parser, size_t i, TB_ObjectSymbol* restrict out_sym); - -#endif // TB_COFF_H - -#ifdef TB_COFF_IMPL -#include - -#pragma pack(push, 2) -typedef struct COFF_SectionHeader { - char name[8]; - union { - uint32_t physical_address; - uint32_t virtual_size; - } misc; - uint32_t virtual_address; - uint32_t raw_data_size; - uint32_t raw_data_pos; - uint32_t pointer_to_reloc; - uint32_t pointer_to_lineno; - uint16_t num_reloc; - uint16_t num_lineno; - uint32_t characteristics; -} COFF_SectionHeader; - -typedef struct COFF_FileHeader { - uint16_t machine; - uint16_t section_count; - uint32_t timestamp; - uint32_t symbol_table; - uint32_t symbol_count; - uint16_t optional_header_size; - uint16_t flags; -} COFF_FileHeader; - -typedef struct COFF_Symbol { - union { - uint8_t short_name[8]; - uint32_t long_name[2]; - }; - uint32_t value; - int16_t section_number; - uint16_t type; - uint8_t storage_class; - uint8_t aux_symbols_count; -} COFF_Symbol; - -typedef struct COFF_ImageReloc { - union { - uint32_t VirtualAddress; - uint32_t RelocCount; - }; - uint32_t SymbolTableIndex; - uint16_t Type; -} COFF_ImageReloc; -#pragma pack(pop) - -// sanity checks -static_assert(sizeof(COFF_SectionHeader) == 40, "COFF Section header size != 40 bytes"); -static_assert(sizeof(COFF_ImageReloc) == 10, "COFF Image Relocation size != 10 bytes"); -static_assert(sizeof(COFF_FileHeader) == 20, "COFF File header size != 20 bytes"); -static_assert(sizeof(COFF_Symbol) == 18, "COFF Symbol size != 18 bytes"); - -bool tb_coff_parse_init(TB_COFF_Parser* restrict parser) { - TB_Slice file = parser->file; - - if (file.length < sizeof(COFF_FileHeader)) return false; - COFF_FileHeader* header = (COFF_FileHeader*) &parser->file.data[0]; - - // locate string table (it spans until the end of the file) - size_t string_table_pos = header->symbol_table + (header->symbol_count * sizeof(COFF_Symbol)); - if (file.length < string_table_pos) return false; - - parser->symbol_count = header->symbol_count; - parser->symbol_table = header->symbol_table; - parser->section_count = header->section_count; - parser->string_table = (TB_Slice){ - .length = file.length - string_table_pos, - .data = &file.data[string_table_pos] - }; - - return true; -} - -static long long tb__parse_decimal_int(size_t n, const char* str) { - const char* end = &str[n]; - - int result = 0; - while (str != end) { - if (*str < '0' || *str > '9') break; - - result *= 10; - result += *str - '0'; - str++; - } - - return result; -} - -bool tb_coff_parse_section(TB_COFF_Parser* restrict parser, size_t i, TB_ObjectSection* restrict out_sec) { - TB_Slice file = parser->file; - size_t section_offset = sizeof(COFF_FileHeader) + (i * sizeof(COFF_SectionHeader)); - - if (file.length < section_offset + sizeof(COFF_SectionHeader)) { - return false; - } - - COFF_SectionHeader* sec = (COFF_SectionHeader*) &file.data[section_offset]; - *out_sec = (TB_ObjectSection) { .flags = sec->characteristics }; - - // Parse string table name stuff - if (sec->name[0] == '/') { - // string table access - int offset = tb__parse_decimal_int(7, &sec->name[1]); - if (file.length > offset) { - return false; - } - - const uint8_t* data = &parser->string_table.data[offset]; - out_sec->name = (TB_Slice){ data, strlen((const char*) data) }; - } else { - // normal inplace string - size_t len = strlen(sec->name); - out_sec->name = (TB_Slice){ (uint8_t*) sec->name, len }; - } - - // Parse relocations - if (sec->num_reloc > 0) { - out_sec->relocation_count = sec->num_reloc; - COFF_ImageReloc* src_relocs = (COFF_ImageReloc*) &file.data[sec->pointer_to_reloc]; - - TB_ObjectReloc* dst_relocs = tb_platform_heap_alloc(sec->num_reloc * sizeof(TB_ObjectReloc)); - FOREACH_N(j, 0, sec->num_reloc) { - dst_relocs[j] = (TB_ObjectReloc){ 0 }; - switch (src_relocs[j].Type) { - case IMAGE_REL_AMD64_ADDR32NB: dst_relocs[j].type = TB_OBJECT_RELOC_ADDR32NB; break; - case IMAGE_REL_AMD64_ADDR32: dst_relocs[j].type = TB_OBJECT_RELOC_ADDR32; break; - case IMAGE_REL_AMD64_ADDR64: dst_relocs[j].type = TB_OBJECT_RELOC_ADDR64; break; - case IMAGE_REL_AMD64_SECREL: dst_relocs[j].type = TB_OBJECT_RELOC_SECREL; break; - case IMAGE_REL_AMD64_SECTION: dst_relocs[j].type = TB_OBJECT_RELOC_SECTION; break; - - case IMAGE_REL_AMD64_REL32: - case IMAGE_REL_AMD64_REL32_1: - case IMAGE_REL_AMD64_REL32_2: - case IMAGE_REL_AMD64_REL32_3: - case IMAGE_REL_AMD64_REL32_4: - case IMAGE_REL_AMD64_REL32_5: - dst_relocs[j].type = TB_OBJECT_RELOC_REL32; - break; - - default: tb_todo(); - } - - if (src_relocs[j].Type >= IMAGE_REL_AMD64_REL32 && src_relocs[j].Type <= IMAGE_REL_AMD64_REL32_5) { - dst_relocs[j].addend = 4 + (src_relocs[j].Type - IMAGE_REL_AMD64_REL32); - } - - dst_relocs[j].symbol_index = src_relocs[j].SymbolTableIndex; - dst_relocs[j].virtual_address = src_relocs[j].VirtualAddress; - } - - out_sec->relocations = dst_relocs; - } - - // Parse virtual region - out_sec->virtual_address = sec->virtual_address; - out_sec->virtual_size = sec->misc.virtual_size; - - // Read raw data (if applies) - if (sec->raw_data_size) { - assert(sec->raw_data_pos + sec->raw_data_size < file.length); - out_sec->raw_data = (TB_Slice){ &file.data[sec->raw_data_pos], sec->raw_data_size }; - } - - return true; -} - -TB_ObjectSymbolType classify_symbol_type(uint16_t st_class) { - switch (st_class) { - case 2: return TB_OBJECT_SYMBOL_EXTERN; - case 3: return TB_OBJECT_SYMBOL_STATIC; - case 6: return TB_OBJECT_SYMBOL_STATIC; - case 0x68: return TB_OBJECT_SYMBOL_SECTION; - case 0x69: return TB_OBJECT_SYMBOL_WEAK_EXTERN; - default: return TB_OBJECT_SYMBOL_UNKNOWN; - } -} - -size_t tb_coff_parse_symbol(TB_COFF_Parser* restrict parser, size_t i, TB_ObjectSymbol* restrict out_sym) { - TB_Slice file = parser->file; - size_t symbol_offset = parser->symbol_table + (i * sizeof(COFF_Symbol)); - - if (file.length < symbol_offset + sizeof(COFF_Symbol)) { - return 0; - } - - COFF_Symbol* sym = (COFF_Symbol*) &file.data[symbol_offset]; - *out_sym = (TB_ObjectSymbol) { - .ordinal = i, - .type = classify_symbol_type(sym->storage_class), - .section_num = sym->section_number, - .value = sym->value - }; - - // Parse string table name stuff - if (sym->long_name[0] == 0) { - // string table access (read a cstring) - // TODO(NeGate): bounds check this - const uint8_t* data = &parser->string_table.data[sym->long_name[1]]; - out_sym->name = (TB_Slice){ data, strlen((const char*) data) }; - } else { - // normal inplace string - size_t len = 1; - const char* name = (const char*) sym->short_name; - while (len < 8 && name[len] != 0) { - len++; - } - out_sym->name = (TB_Slice){ sym->short_name, len }; - } - - // TODO(NeGate): Process aux symbols - if (sym->aux_symbols_count) { - out_sym->extra = &sym[1]; - - // FOREACH_N(j, 0, sym->aux_symbols_count) {} - } - - return sym->aux_symbols_count + 1; -} - -#endif // TB_COFF_IMPL diff --git a/vendor/tb/include/tb_elf.h b/vendor/tb/include/tb_elf.h deleted file mode 100644 index c29b3fc3..00000000 --- a/vendor/tb/include/tb_elf.h +++ /dev/null @@ -1,171 +0,0 @@ -#ifndef TB_ELF_H -#define TB_ELF_H - -#include - -#define TB_EI_MAG0 0 -#define TB_EI_MAG1 1 -#define TB_EI_MAG2 2 -#define TB_EI_MAG3 3 -#define TB_EI_CLASS 4 /* Class of machine. */ -#define TB_EI_DATA 5 /* Data format. */ -#define TB_EI_VERSION 6 /* ELF format version. */ -#define TB_EI_OSABI 7 /* Operating system / ABI identification */ -#define TB_EI_ABIVERSION 8 /* ABI version */ -#define TB_OLD_EI_BRAND 8 /* Start of architecture identification. */ -#define TB_EI_PAD 9 /* Start of padding (per SVR4 ABI). */ -#define TB_EI_NIDENT 16 /* Size of e_ident array. */ - -/* Values for e_type. */ -#define TB_ET_NONE 0 /* Unknown type. */ -#define TB_ET_REL 1 /* Relocatable. */ -#define TB_ET_EXEC 2 /* Executable. */ -#define TB_ET_DYN 3 /* Shared object. */ -#define TB_ET_CORE 4 /* Core file. */ -#define TB_ET_LOOS 0xfe00 /* First operating system specific. */ -#define TB_ET_HIOS 0xfeff /* Last operating system-specific. */ -#define TB_ET_LOPROC 0xff00 /* First processor-specific. */ -#define TB_ET_HIPROC 0xffff /* Last processor-specific. */ - -/* Values for e_machine. */ -#define TB_EM_NONE 0 /* Unknown machine. */ -#define TB_EM_MIPS 8 /* Mips */ -#define TB_EM_X86_64 62 /* Advanced Micro Devices x86-64 */ -#define TB_EM_AARCH64 183 /* AArch64 (64-bit ARM) */ - -/* sh_type */ -#define TB_SHT_NULL 0 /* inactive */ -#define TB_SHT_PROGBITS 1 /* program defined information */ -#define TB_SHT_SYMTAB 2 /* symbol table section */ -#define TB_SHT_STRTAB 3 /* string table section */ -#define TB_SHT_RELA 4 /* relocation section with addends */ -#define TB_SHT_NOBITS 8 /* no space section */ - -/* Flags for sh_flags. */ -#define TB_SHF_WRITE 0x1 /* Section contains writable data. */ -#define TB_SHF_ALLOC 0x2 /* Section occupies memory. */ -#define TB_SHF_EXECINSTR 0x4 /* Section contains instructions. */ -#define TB_SHF_MERGE 0x10 /* Section may be merged. */ -#define TB_SHF_STRINGS 0x20 /* Section contains strings. */ -#define TB_SHF_INFO_LINK 0x40 /* sh_info holds section index. */ -#define TB_SHF_LINK_ORDER 0x80 /* Special ordering requirements. */ -#define TB_SHF_OS_NONCONFORMING 0x100 /* OS-specific processing required. */ -#define TB_SHF_GROUP 0x200 /* Member of section group. */ -#define TB_SHF_TLS 0x400 /* Section contains TLS data. */ -#define TB_SHF_MASKOS 0x0ff00000 /* OS-specific semantics. */ -#define TB_SHF_MASKPROC 0xf0000000 /* Processor-specific semantics. */ - -/* Values for p_flags. */ -#define TB_PF_X 0x1 /* Executable. */ -#define TB_PF_W 0x2 /* Writable. */ -#define TB_PF_R 0x4 /* Readable. */ -#define TB_PF_MASKOS 0x0ff00000 /* Operating system-specific. */ -#define TB_PF_MASKPROC 0xf0000000 /* Processor-specific. */ - -/* Values for p_type. */ -#define TB_PT_NULL 0 /* Unused entry. */ -#define TB_PT_LOAD 1 /* Loadable segment. */ -#define TB_PT_DYNAMIC 2 /* Dynamic linking information segment. */ -#define TB_PT_INTERP 3 /* Pathname of interpreter. */ -#define TB_PT_NOTE 4 /* Auxiliary information. */ -#define TB_PT_SHLIB 5 /* Reserved (not used). */ -#define TB_PT_PHDR 6 /* Location of program header itself. */ -#define TB_PT_TLS 7 /* Thread local storage segment */ - -/* Values for relocation */ -typedef enum { - TB_ELF_X86_64_NONE = 0, - TB_ELF_X86_64_64 = 1, - TB_ELF_X86_64_PC32 = 2, - TB_ELF_X86_64_GOT32 = 3, - TB_ELF_X86_64_PLT32 = 4, - TB_ELF_X86_64_GOTPCREL = 9, -} TB_ELF_RelocType; - -// ST_TYPE -#define TB_ELF64_STT_NOTYPE 0 -#define TB_ELF64_STT_OBJECT 1 -#define TB_ELF64_STT_FUNC 2 -#define TB_ELF64_STT_SECTION 3 - -// ST_INFO -#define TB_ELF64_STB_LOCAL 0 -#define TB_ELF64_STB_GLOBAL 1 -#define TB_ELF64_STB_WEAK 2 - -/* Macros for accessing the fields of st_info. */ -#define TB_ELF64_ST_BIND(info) ((info) >> 4) -#define TB_ELF64_ST_TYPE(info) ((info) & 0xf) - -#define TB_ELF64_ST_INFO(b, t) (((b) << 4) | ((t) & 0xF)) - -#define TB_ELF64_R_SYM(i) ((i) >> 32u) -#define TB_ELF64_R_TYPE(i) ((i)&0xffffffffULL) -#define TB_ELF64_R_INFO(s, t) (((uint64_t)(s) << 32ULL) + ((uint64_t)(t) & 0xffffffffULL)) - -// http://web.mit.edu/freebsd/head/sys/sys/elf64.h -// https://cirosantilli.com/elf-hello-world#minimal-elf-file -// https://en.wikipedia.org/wiki/Executable_and_Linkable_Format -typedef struct { - uint8_t ident[16]; - uint16_t type; - uint16_t machine; - uint32_t version; - uint64_t entry; - uint64_t phoff; - uint64_t shoff; - uint32_t flags; - uint16_t ehsize; - uint16_t phentsize; - uint16_t phnum; - uint16_t shentsize; - uint16_t shnum; - uint16_t shstrndx; -} TB_Elf64_Ehdr; - -typedef struct { - uint32_t name; - uint32_t type; - uint64_t flags; - uint64_t addr; - uint64_t offset; - uint64_t size; - uint32_t link; - uint32_t info; - uint64_t addralign; - uint64_t entsize; -} TB_Elf64_Shdr; - -// Segment header for ELF64. -typedef struct { - uint32_t type; // Type of segment - uint32_t flags; // Segment flags - uint64_t offset; // File offset where segment is located, in bytes - uint64_t vaddr; // Virtual address of beginning of segment - uint64_t paddr; // Physical addr of beginning of segment (OS-specific) - uint64_t filesz; // Num. of bytes in file image of segment (may be zero) - uint64_t memsz; // Num. of bytes in mem image of segment (may be zero) - uint64_t align; // Segment alignment constraint -} TB_Elf64_Phdr; - -typedef struct { - uint32_t name; - uint8_t info; - uint8_t other; - uint16_t shndx; - uint64_t value; - uint64_t size; -} TB_Elf64_Sym; - -typedef struct { - uint64_t offset; - uint64_t info; - int64_t addend; -} TB_Elf64_Rela; - -typedef struct { - uint64_t offset; - uint64_t info; -} TB_Elf64_Rel; - -#endif /* TB_ELF_H */ diff --git a/vendor/tb/include/tb_formats.h b/vendor/tb/include/tb_formats.h deleted file mode 100644 index 1975d584..00000000 --- a/vendor/tb/include/tb_formats.h +++ /dev/null @@ -1,132 +0,0 @@ -// This handles the generalized executable/object format parsing stuff -#ifndef TB_OBJECT_H -#define TB_OBJECT_H - -#include -#include - -typedef enum { - TB_OBJECT_RELOC_NONE, // how? - - // Target independent - TB_OBJECT_RELOC_ADDR32, - TB_OBJECT_RELOC_ADDR64, // unsupported on 32bit platforms - TB_OBJECT_RELOC_SECREL, - TB_OBJECT_RELOC_SECTION, - - // COFF only - TB_OBJECT_RELOC_ADDR32NB, // Relative virtual address - - // x64 only - TB_OBJECT_RELOC_REL32, // relative 32bit displacement - - // Aarch64 only - TB_OBJECT_RELOC_BRANCH26, // 26bit displacement for B and BL instructions - TB_OBJECT_RELOC_REL21, // for ADR instructions - - // TODO(NeGate): fill in the rest of this later -} TB_ObjectRelocType; - -typedef struct { - TB_ObjectRelocType type; - uint32_t symbol_index; - size_t virtual_address; - size_t addend; -} TB_ObjectReloc; - -typedef enum { - TB_OBJECT_SYMBOL_UNKNOWN, - TB_OBJECT_SYMBOL_EXTERN, // exported - TB_OBJECT_SYMBOL_WEAK_EXTERN, // weak - TB_OBJECT_SYMBOL_IMPORT, // forward decl - TB_OBJECT_SYMBOL_STATIC, // local - TB_OBJECT_SYMBOL_SECTION, // local -} TB_ObjectSymbolType; - -typedef struct { - TB_ObjectSymbolType type; - int section_num; - - uint32_t ordinal; - uint32_t value; - - TB_Slice name; - - // for COFF, this is the auxillary - void* extra; - - // this is zeroed out by the loader and left for the user to do crap with - void* user_data; -} TB_ObjectSymbol; - -typedef struct { - TB_Slice name; - uint32_t flags; - - size_t virtual_address; - size_t virtual_size; - - // You can have a virtual size without having a raw - // data size, that's how the BSS section works - TB_Slice raw_data; - - size_t relocation_count; - TB_ObjectReloc* relocations; - - // this is zeroed out by the loader and left for the user to do crap with - void* user_data; -} TB_ObjectSection; - -typedef enum { - TB_OBJECT_FILE_UNKNOWN, - - TB_OBJECT_FILE_COFF, - TB_OBJECT_FILE_ELF64 -} TB_ObjectFileType; - -typedef struct { - TB_ObjectFileType type; - TB_Arch arch; - - TB_Slice name; - TB_Slice ar_name; - - size_t symbol_count; - TB_ObjectSymbol* symbols; - - size_t section_count; - TB_ObjectSection sections[]; -} TB_ObjectFile; - -//////////////////////////////// -// Archive parser -//////////////////////////////// -typedef struct { - TB_Slice name; - - // if import_name is empty, we're dealing with an object file - TB_Slice import_name; - uint16_t ordinal; - - TB_Slice content; -} TB_ArchiveEntry; - -typedef struct { - TB_Slice file; - size_t pos; - - size_t member_count; - uint32_t* members; - - size_t symbol_count; - uint16_t* symbols; - - TB_Slice strtbl; -} TB_ArchiveFileParser; - -// We do this to parse the header -bool tb_archive_parse(TB_Slice file, TB_ArchiveFileParser* restrict out_parser); -// After that we can enumerate any symbol entries to resolve imports -size_t tb_archive_parse_entries(TB_ArchiveFileParser* restrict parser, size_t i, size_t count, TB_ArchiveEntry* out_entry); - -#endif // TB_OBJECT_H diff --git a/vendor/tb/include/tb_x64.h b/vendor/tb/include/tb_x64.h deleted file mode 100644 index 9da719e0..00000000 --- a/vendor/tb/include/tb_x64.h +++ /dev/null @@ -1,89 +0,0 @@ -#ifndef TB_X64_H -#define TB_X64_H - -#include -#include - -typedef enum { - // r/m is a memory operand - TB_X86_INSTR_INDIRECT = (1u << 0u), - - // LOCK prefix is present - TB_X86_INSTR_LOCK = (1u << 1u), - - // uses an immediate - TB_X86_INSTR_IMMEDIATE = (1u << 2u), - - // set if the r/m can be found on the right hand side - TB_X86_INSTR_DIRECTION = (1u << 3u), - - // REP prefix is present - TB_X86_INSTR_REP = (1u << 5u), - - // REPNE prefix is present - TB_X86_INSTR_REPNE = (1u << 6u), -} TB_X86_InstFlags; - -typedef enum { - TB_X86_RAX, TB_X86_RCX, TB_X86_RDX, TB_X86_RBX, TB_X86_RSP, TB_X86_RBP, TB_X86_RSI, TB_X86_RDI, - TB_X86_R8, TB_X86_R9, TB_X86_R10, TB_X86_R11, TB_X86_R12, TB_X86_R13, TB_X86_R14, TB_X86_R15, -} TB_X86_GPR; - -typedef enum { - TB_X86_SEGMENT_DEFAULT = 0, - - TB_X86_SEGMENT_ES, TB_X86_SEGMENT_CS, - TB_X86_SEGMENT_SS, TB_X86_SEGMENT_DS, - TB_X86_SEGMENT_GS, TB_X86_SEGMENT_FS, -} TB_X86_Segment; - -typedef enum { - TB_X86_TYPE_NONE = 0, - - TB_X86_TYPE_BYTE, // 1 - TB_X86_TYPE_WORD, // 2 - TB_X86_TYPE_DWORD, // 4 - TB_X86_TYPE_QWORD, // 8 - - TB_X86_TYPE_PBYTE, // int8 x 16 = 16 - TB_X86_TYPE_PWORD, // int16 x 8 = 16 - TB_X86_TYPE_PDWORD, // int32 x 4 = 16 - TB_X86_TYPE_PQWORD, // int64 x 2 = 16 - - TB_X86_TYPE_SSE_SS, // float32 x 1 = 4 - TB_X86_TYPE_SSE_SD, // float64 x 1 = 8 - TB_X86_TYPE_SSE_PS, // float32 x 4 = 16 - TB_X86_TYPE_SSE_PD, // float64 x 2 = 16 - - TB_X86_TYPE_XMMWORD, // the generic idea of them -} TB_X86_DataType; - -typedef struct { - uint16_t opcode; - - // packed 16bits - uint16_t scale : 2; - uint16_t flags : 6; - uint16_t dt : 4; - uint16_t dt2 : 4; - - // each 8bits are a different reg (lowest bits to higher): - // base, index, rx, extra - uint32_t regs; - int32_t disp; - - // bitpacking amirite - TB_X86_Segment segment : 4; - uint8_t length : 4; - - // immediate operand - int64_t imm; -} TB_X86_Inst; - -void tb_x86_print_inst(FILE* fp, TB_X86_Inst* inst); -bool tb_x86_disasm(TB_X86_Inst* restrict inst, size_t length, const uint8_t* data); -const char* tb_x86_reg_name(int8_t reg, TB_X86_DataType dt); -const char* tb_x86_type_name(TB_X86_DataType dt); -const char* tb_x86_mnemonic(TB_X86_Inst* inst); - -#endif /* TB_X64_H */ diff --git a/vendor/tb/src/aarch64/aarch64.c b/vendor/tb/src/aarch64/aarch64.c deleted file mode 100644 index 255a4e1c..00000000 --- a/vendor/tb/src/aarch64/aarch64.c +++ /dev/null @@ -1,218 +0,0 @@ -#if 0 -// NOTE(NeGate): THIS IS VERY INCOMPLETE -#include "../tb_internal.h" -#include "../codegen/emitter.h" - -typedef struct Ctx Ctx; - -enum { - AARCH64_REG_CLASS_GPR -}; - -// Xn refers to the 64bit variants of the registers, -// usually the 32bit aliases are Wn (we don't have enums -// for them because it's not that important, they're equal) -typedef enum { - X0, X1, X2, X3, X4, X5, X6, X7, - X8, X9, X10, X11, X12, X13, X14, X15, - X16, X17, X18, X19, X20, X21, X22, X23, - X24, X25, X26, X27, X28, X29, X30, - - // It's context specific because ARM lmao - ZR = 0x1F, SP = 0x1F, - - // not a real gpr - GPR_NONE = -1, -} GPR; - -// refers to the data processing immediate operand. -// Aarch64 has a bunch of weird immediate fields so -// we might wanna rename this later. -typedef struct { - uint16_t imm; - uint8_t shift; -} Immediate; - -typedef enum ValType { - VAL_NONE, - VAL_FLAGS, - VAL_GPR, - VAL_FPR, - VAL_IMM, - VAL_MEM, - VAL_GLOBAL, -} ValType; - -typedef struct { - int8_t type; - - // if VAL_MEM then this is the base - int8_t reg; - - uint8_t base; - - union { - - int reg; - GPR gpr; - uint8_t fpr; - struct { - bool is_rvalue; - int32_t disp; - } mem; - - int32_t disp; - uint64_t num; - }; -} Val; - -#include "../x64/generic_cg.h" -#include "aarch64_emitter.h" - -static int classify_reg_class(TB_DataType dt) { - -} - -static int isel(Ctx* restrict ctx, Sequence* restrict seq, TB_Node* n) { - /*TB_Node* restrict n = &f->nodes[r]; - TB_NodeTypeEnum type = n->type; - - switch (type) { - case TB_INTEGER_CONST: { - assert(n->integer.num_words == 1); - GAD_VAL dst = GAD_FN(regalloc)(ctx, f, r, AARCH64_REG_CLASS_GPR); - - uint64_t x = n->integer.single_word; - emit_movz(&ctx->emit, dst.reg, x & 0xFFFF, 0, true); - - uint8_t shift = 16; - x >>= 16; - - while (x & 0xFFFF) { - emit_movk(&ctx->emit, dst.reg, x & 0xFFFF, shift, true); - x >>= 16, shift += 16; - } - return dst; - } - - case TB_ADD: { - GAD_VAL a = ctx->values[n->i_arith.a]; - GAD_VAL b = ctx->values[n->i_arith.b]; - GAD_VAL dst = GAD_FN(regalloc)(ctx, f, r, AARCH64_REG_CLASS_GPR); - - bool is_64bit = n->dt.type == TB_PTR || (n->dt.type == TB_INT && n->dt.data == 64); - emit_dp_r(&ctx->emit, ADD, dst.reg, a.reg, b.reg, 0, 0, is_64bit); - return dst; - } - - default: - tb_todo(); - return (GAD_VAL){ 0 }; - }*/ -} - -static void emit_sequence(Ctx* restrict ctx, Sequence* restrict seq, TB_Node* n) { - assert(seq->inst_count == 0 && "TODO"); -} - -static void patch_local_labels(Ctx* restrict ctx) { - -} - -static void resolve_stack_usage(Ctx* restrict ctx, size_t caller_usage) { - tb_todo(); -} - -static void copy_value(Ctx* restrict ctx, Sequence* seq, TB_Node* phi, int dst, TB_Node* src, TB_DataType dt) { - tb_todo(); -} - -static size_t aarch64_emit_call_patches(TB_Module* restrict m) { - return 0; -} - -static void aarch64_return(Ctx* restrict ctx, TB_Function* f, TB_Node* restrict n) { - GAD_VAL* src = &ctx->values[n->ret.value]; - - if (src->type != VAL_GPR || src->gpr != X0) { - emit_mov(&ctx->emit, X0, src->gpr, true); - } -} - -static void aarch64_goto(Ctx* restrict ctx, TB_Label label) { - // jmp(&ctx->emit, label); - tb_todo(); -} - -static void aarch64_ret_jmp(Ctx* restrict ctx) { - ctx->emit.ret_patches[ctx->emit.ret_patch_count++] = GET_CODE_POS(&ctx->emit); - EMIT4(&ctx->emit, (0b101 << 25)); -} - -static size_t aarch64_emit_prologue(uint8_t* out, uint64_t saved, uint64_t stack_usage) { - return 0; -} - -static size_t aarch64_emit_epilogue(uint8_t* out, uint64_t saved, uint64_t stack_usage) { - // RET - memcpy(out, &(uint32_t){ 0xD65F03C0 }, 4); - return 4; -} - -static void aarch64_initial_reg_alloc(Ctx* restrict ctx) { - ctx->callee_saved[0] = 0; - - // SP/ZR is the only reserved register - ctx->free_regs[0] = set_create(32); - set_put(&ctx->free_regs[0], SP); - ctx->active[0] = 0, ctx->active_count += 1; -} - -static int get_data_type_size(const TB_DataType dt) { - assert(dt.width <= 2 && "Vector width too big!"); - - switch (dt.type) { - case TB_INT: { - // round up bits to a byte - bool is_big_int = dt.data > 64; - int bits = is_big_int ? ((dt.data + 7) / 8) : tb_next_pow2(dt.data); - - return ((bits+7) / 8) << dt.width; - } - case TB_FLOAT: { - int s = 0; - if (dt.data == TB_FLT_32) s = 4; - else if (dt.data == TB_FLT_64) s = 8; - else tb_unreachable(); - - return s << dt.width; - } - case TB_PTR: { - return 8; - } - default: { - tb_unreachable(); - return 0; - } - } -} - -#if _MSC_VER -_Pragma("warning (push)") _Pragma("warning (disable: 4028)") -#endif -ICodeGen tb__aarch64_codegen = { - .minimum_addressable_size = 8, - .pointer_size = 64, - - .get_data_type_size = aarch64_get_data_type_size, - .emit_call_patches = aarch64_emit_call_patches, - .emit_prologue = aarch64_emit_prologue, - .emit_epilogue = aarch64_emit_epilogue, - - .fast_path = aarch64_compile_function, - //.complex_path = x64_complex_compile_function -}; -#if _MSC_VER -_Pragma("warning (pop)") -#endif -#endif \ No newline at end of file diff --git a/vendor/tb/src/aarch64/aarch64_emitter.h b/vendor/tb/src/aarch64/aarch64_emitter.h deleted file mode 100644 index 34b93ed6..00000000 --- a/vendor/tb/src/aarch64/aarch64_emitter.h +++ /dev/null @@ -1,186 +0,0 @@ - -// Xn refers to the 64bit variants of the registers, -// usually the 32bit aliases are Wn (we don't have enums -// for them because it's not that important, they're equal) -typedef enum { - X0, X1, X2, X3, X4, X5, X6, X7, - X8, X9, X10, X11, X12, X13, X14, X15, - X16, X17, X18, X19, X20, X21, X22, X23, - X24, X25, X26, X27, X28, X29, X30, - - // frame pointer - FP = 29, - // link register is basically just the RPC - LR = 30, - // It's context specific because ARM lmao - ZR = 31, SP = 31, - - // not a real gpr - GPR_NONE = -1, -} GPR; - -// refers to the data processing immediate operand. -// Aarch64 has a bunch of weird immediate fields so -// we might wanna rename this later. -typedef struct { - uint16_t imm; - uint8_t shift; -} Immediate; - -typedef enum { - ADD, - SUB, - EOR, - UDIV, - SDIV, -} DPOpcode; - -enum { - SHIFT_LSL, - SHIFT_LSR, - SHIFT_ASR, - SHIFT_RESERVED, -}; - -typedef struct { - uint32_t r, i; -} DPInst; - -// op0 30-29 -// op1 28 -// 101 27-25 -// op2 24-21 -// op3 15-10 -#define DPR(op0, op1, op2, op3) ((op0 << 29u) | (op1 << 28) | (0b101 << 25) | (op2 << 21) | (op3 << 10)) -// op 30 -// 100 28-26 -// op0 25-23 -#define DPI(op, op0) ((op << 30u) | (0b100 << 26u) | (op0 << 23u)) - -#define DP3(op0, op1, op2) ((op0 << 30u) | (op1 << 28u) | (0b101 << 25u) | (op2 << 21u)) - -static const DPInst inst_table[] = { - // register immediate - [ADD] = { DPR(0, 0, 0b1000, 0), DPI(0, 0b010) }, - [SUB] = { DPR(2, 0, 0b1000, 0), DPI(1, 0b010) }, - [EOR] = { 0b01001010000 << 21u, 0 }, - - [UDIV] = { DPR(0, 1, 0b0110, 0b000010) }, - [SDIV] = { DPR(0, 1, 0b0110, 0b000011) }, -}; - -enum { - UBFM = DPI(0, 0b110) | (0b10 << 29u), - - // op0 - // V - MADD = 0b00011011000000000000000000000000, - MSUB = 0b00011011000000001000000000000000, -}; - -static void emit_ret(TB_CGEmitter* restrict e, GPR rn) { - // 1101 0110 0101 1111 0000 00NN NNN0 0000 - // - // 'ret rn' just does 'mov pc, rn', although in practice - // we only pass the link reg to it. - uint32_t inst = 0b11010110010111110000000000000000; - inst |= (rn & 0b11111) << 5u; - EMIT4(e, inst); -} - -// OP Rd, Rn, Rm, Ra -static void emit_dp3(TB_CGEmitter* restrict e, uint32_t inst, GPR d, GPR n, GPR m, GPR a, bool _64bit) { - inst |= (_64bit ? (1u << 31u) : 0); - inst |= (m & 0x1F) << 16u; - inst |= (a & 0x1F) << 10u; - inst |= (n & 0x1F) << 5u; - inst |= (d & 0x1F) << 0u; - EMIT4(e, inst); -} - -// data processing instruction -// OP dst, src, imm -// -// 0000 0000 0000 0000 0000 0000 0000 0000 -// AOOO OOOO SSII IIII IIII IINN NNND DDDD -// -// A - set when we're doing the 64bit variant of the instruction -// O - is the opcode -// S - shift -// I - immediate -// N - source -// D - destination -static void emit_dp_imm(TB_CGEmitter* restrict e, DPOpcode op, GPR dst, GPR src, uint16_t imm, uint8_t shift, bool _64bit) { - uint32_t inst = inst_table[op].i | (_64bit ? (1u << 31u) : 0); - - if (op == ADD || op == SUB) { - assert(shift == 0 || shift == 12); - inst |= (1 << 22u); - } else if (op == UBFM) { - inst |= (1 << 22u); - } - - inst |= (imm & 0xFFF) << 10u; - inst |= (src & 0x1F) << 5u; - inst |= (dst & 0x1F) << 0u; - EMIT4(e, inst); -} - -// bitfield -static void emit_bitfield(TB_CGEmitter* restrict e, uint32_t op, GPR dst, GPR src, uint8_t immr, uint8_t imms, bool _64bit) { - uint32_t inst = op | (_64bit ? (1u << 31u) : 0); - inst |= (immr & 0b111111) << 16u; - inst |= (imms & 0b111111) << 10u; - inst |= (src & 0b11111) << 5u; - inst |= (dst & 0b11111) << 0u; - EMIT4(e, inst); -} - -// data processing instruction -// OP dst, a, b -// -// 0000 0000 0000 0000 0000 0000 0000 0000 -// AOOO OOOO SS_M MMMM IIII IINN NNND DDDD -// x101 1110 xx1m mmmm x000 01nn nnnd dddd - add Sd Sn Sm -static void emit_dp_r(TB_CGEmitter* restrict e, DPOpcode op, GPR dst, GPR a, GPR b, uint16_t imm, uint8_t shift, bool _64bit) { - uint32_t inst = inst_table[op].r | (_64bit ? (1u << 31u) : 0); - - if (op == ADD || op == SUB) { - assert(shift == 0 || shift == 12); - inst |= (1 << 22u); - } else if (op == UBFM) { - if (shift) inst |= (1 << 22u); - } - - inst |= (b & 0b11111) << 16u; - inst |= (imm & 0xFFF) << 10u; - inst |= (a & 0b11111) << 5u; - inst |= (dst & 0b11111) << 0u; - EMIT4(e, inst); -} - -static void emit_mov(TB_CGEmitter* restrict e, uint8_t dst, uint8_t src, bool _64bit) { - uint32_t inst = (_64bit ? (1 << 31u) : 0); - - inst |= (0b00101010 << 24u); - inst |= (src & 0b11111) << 16u; - inst |= (0b11111) << 5u; - inst |= (dst & 0b11111) << 0u; - EMIT4(e, inst); -} - -// clear means movz, else movk -static void emit_movimm(TB_CGEmitter* restrict e, uint8_t dst, uint16_t imm, uint8_t shift, bool _64bit, bool clear) { - uint32_t inst = (_64bit ? (1 << 31u) : 0); - - if (clear) { - inst |= (0b010100101 << 23u); - } else { - inst |= (0b011100101 << 23u); - } - inst |= shift << 21u; - inst |= imm << 5u; - inst |= (dst & 0b11111) << 0u; - EMIT4(e, inst); -} - diff --git a/vendor/tb/src/aarch64/aarch64_target.c b/vendor/tb/src/aarch64/aarch64_target.c deleted file mode 100644 index bdb60660..00000000 --- a/vendor/tb/src/aarch64/aarch64_target.c +++ /dev/null @@ -1,481 +0,0 @@ -#include "../tb_internal.h" - -#ifdef TB_HAS_AARCH64 -// NOTE(NeGate): THIS IS VERY INCOMPLETE -#include "../emitter.h" -#include "aarch64_emitter.h" - -enum { - // register classes - REG_CLASS_GPR = 1, - REG_CLASS_COUNT, -}; - -enum { - ALL_GPRS = 0xFFFFFFFF & ~((1 << SP) | (1 << LR)), -}; - -#include "../codegen_impl.h" - -enum { - // OP reg, imm - TILE_HAS_IMM = 1 -}; - -// true for 64bit -static bool legalize_int(TB_DataType dt) { return dt.type == TB_PTR || (dt.type == TB_INT && dt.data > 32); } - -static bool try_for_imm12(int bits, TB_Node* n, int32_t* out_x) { - if (n->type != TB_INTEGER_CONST) { - return false; - } - - TB_NodeInt* i = TB_NODE_GET_EXTRA(n); - if (bits > 12) { - bool sign = (i->value >> 11ull) & 1; - uint64_t top = i->value >> 12ull; - - // if the sign matches the rest of the top bits, we can sign extend just fine - if (top != (sign ? 0xFFFFFFFFFFFFF : 0)) { - return false; - } - } - - *out_x = i->value; - return true; -} - -static bool _2addr(TB_Node* n) { return false; } -static void init_ctx(Ctx* restrict ctx, TB_ABI abi) { - ctx->sched = greedy_scheduler; - ctx->_2addr = _2addr; - ctx->regalloc = tb__lsra; - - uint32_t all_gprs = UINT32_MAX & ~(1u << SP); - if (ctx->features.gen & TB_FEATURE_FRAME_PTR) { - all_gprs &= ~(1u << FP); - } - ctx->num_regs[1] = 32; - ctx->normie_mask[1] = REGMASK(GPR, all_gprs); - - // x19 - x29 are callee saved - ctx->callee_saved[0] = ((1u << 10u) - 1) << 19u; -} - -static RegMask isel_node(Ctx* restrict ctx, Tile* dst, TB_Node* n) { - switch (n->type) { - case TB_ROOT: { - TileInput* ins = tile_set_ins(ctx, dst, n, 4, n->input_count); - int rets = n->input_count - 4; - - assert(rets <= 2 && "At most 2 return values :("); - FOREACH_N(i, 0, rets) { - ins[i].mask = REGMASK(GPR, 1 << i); - } - return REGEMPTY; - } - - case TB_PROJ: - int i = TB_NODE_GET_EXTRA_T(n, TB_NodeProj)->index; - if (n->inputs[0]->type == TB_ROOT) { - return REGMASK(GPR, i >= 3 ? (1u << (i - 3)) : 0); - } else if (n->inputs[0]->type == TB_CALL) { - if (i == 2) return REGMASK(GPR, 1 << X0); - else if (i == 3) return REGMASK(GPR, 1 << X1); - else return REGEMPTY; - } else if (n->inputs[0]->type == TB_BRANCH) { - return REGEMPTY; - } else { - tb_todo(); - } - - case TB_ARRAY_ACCESS: - tile_broadcast_ins(ctx, dst, n, 1, n->input_count, ctx->normie_mask[1]); - return ctx->normie_mask[1]; - - case TB_INTEGER_CONST: - return ctx->normie_mask[1]; - - case TB_MUL: - case TB_UDIV: - case TB_SDIV: - case TB_XOR: - tile_broadcast_ins(ctx, dst, n, 1, n->input_count, ctx->normie_mask[1]); - return ctx->normie_mask[1]; - - // binary ops - case TB_SHL: - case TB_SHR: - case TB_ADD: - case TB_SUB: { - int32_t x; - if (try_for_imm12(n->dt.data, n->inputs[2], &x)) { - tile_broadcast_ins(ctx, dst, n, 1, 2, ctx->normie_mask[1]); - dst->flags |= TILE_HAS_IMM; - } else { - tile_broadcast_ins(ctx, dst, n, 1, n->input_count, ctx->normie_mask[1]); - } - return ctx->normie_mask[1]; - } - - default: - tb_todo(); - } -} - -static void pre_emit(Ctx* restrict ctx, TB_CGEmitter* e, TB_Node* n) { - // TODO(NeGate): optionally preserve the frame pointer - // TODO(NeGate): stack allocation - ctx->prologue_length = ctx->emit.count; -} - -static GPR gpr_at(LiveInterval* l) { assert(l->class == REG_CLASS_GPR); return l->assigned; } -static void emit_tile(Ctx* restrict ctx, TB_CGEmitter* e, Tile* t) { - if (t->tag == TILE_SPILL_MOVE) { - GPR dst = gpr_at(t->interval); - GPR src = gpr_at(t->ins[0].src); - if (dst != src) { - emit_mov(e, dst, src, true); - } - } else { - TB_Node* n = t->n; - switch (n->type) { - // projections don't manage their own work, that's the - // TUPLE node's job. - case TB_PROJ: break; - - case TB_ROOT: { - emit_ret(e, LR); - break; - } - - case TB_INTEGER_CONST: { - bool is_64bit = false; - GPR dst = gpr_at(t->interval); - uint64_t imm = TB_NODE_GET_EXTRA_T(n, TB_NodeInt)->value; - - int shift = 0; bool first = true; - while (imm) { - if (imm & 0xFFFF) { - emit_movimm(e, dst, imm & 0xFFFF, shift / 16, is_64bit, first); - first = false; - } - imm >>= 16, shift += 16; - } - break; - } - - case TB_SDIV: { - bool is_64bit = legalize_int(n->dt); - GPR dst = gpr_at(t->interval); - GPR lhs = gpr_at(t->ins[0].src); - GPR rhs = gpr_at(t->ins[1].src); - emit_dp_r(e, SDIV, dst, lhs, rhs, 0, 0, is_64bit); - break; - } - - case TB_SHL: - case TB_SHR: - case TB_SAR: { - bool is_64bit = legalize_int(n->dt); - GPR dst = gpr_at(t->interval); - GPR lhs = gpr_at(t->ins[0].src); - if (t->flags & TILE_HAS_IMM) { - assert(n->inputs[2]->type == TB_INTEGER_CONST); - TB_NodeInt* i = TB_NODE_GET_EXTRA(n->inputs[2]); - - int amt = 31 - i->value; - emit_bitfield(e, UBFM, dst, lhs, amt + 1, amt, is_64bit); - } else { - // GPR rhs = gpr_at(t->ins[1].src); - tb_todo(); - } - break; - } - - case TB_XOR: - case TB_ADD: - case TB_SUB: { - const static int ops[] = { -1, -1, EOR, ADD, SUB }; - int op = ops[n->type - TB_AND]; - - bool is_64bit = legalize_int(n->dt); - GPR dst = gpr_at(t->interval); - GPR lhs = gpr_at(t->ins[0].src); - - if (t->flags & TILE_HAS_IMM) { - assert(n->inputs[2]->type == TB_INTEGER_CONST); - TB_NodeInt* i = TB_NODE_GET_EXTRA(n->inputs[2]); - emit_dp_imm(e, op, dst, lhs, i->value, 0, is_64bit); - } else { - GPR rhs = gpr_at(t->ins[1].src); - emit_dp_r(e, op, dst, lhs, rhs, 0, 0, is_64bit); - } - break; - } - - case TB_MUL: { - GPR dst = gpr_at(t->interval); - GPR lhs = gpr_at(t->ins[0].src); - GPR rhs = gpr_at(t->ins[1].src); - - emit_dp3(e, MADD, dst, lhs, rhs, ZR, legalize_int(n->dt)); - break; - } - - default: tb_todo(); - } - } -} - -static void post_emit(Ctx* restrict ctx, TB_CGEmitter* e) { -} - -#define E(fmt, ...) tb_asm_print(e, fmt, ## __VA_ARGS__) -static void print_gpr_sp(TB_CGEmitter* e, int i, bool is_64bit) { - if (i == ZR) { - E("sp"); - } else if (i == LR) { - E("lr"); - } else { - E("%c%d", is_64bit["wx"], i); - } -} - -static void print_gpr(TB_CGEmitter* e, int i, bool is_64bit) { - if (i == ZR) { - E("zr"); - } else if (i == LR) { - E("lr"); - } else { - E("%c%d", is_64bit["wx"], i); - } -} - -static void disassemble(TB_CGEmitter* e, Disasm* restrict d, int bb, size_t pos, size_t end) { - if (bb >= 0) { - E(".bb%d:\n", bb); - } - - static const char* dp_strs[] = { "add", "adds", "sub", "subs" }; - static const char* sh_strs[] = { "lsl", "lsr", "asr", "ror" }; - - while (pos < end) { - uint32_t inst = *((uint32_t*) &e->data[pos]); - pos += 4; - - bool is_64bit = inst >> 31u; - uint32_t family = (inst >> 25u) & 0xF; - switch (family) { - // Data processing (immediate) - case 0b1000: - case 0b1001: { - uint32_t op1 = (inst >> 23u) & 0b111; - uint32_t opc = (inst >> 29u) & 0b11; - - if (op1 == 2) { - uint32_t imm = (inst >> 10u) & 0xFFF; - uint32_t rn = (inst >> 5u) & 0x1F; - uint32_t rd = (inst >> 0u) & 0x1F; - - E(" %s ", dp_strs[opc]); - print_gpr(e, rd, is_64bit); - E(", "); - print_gpr(e, rn, is_64bit); - E(", #%#x", imm); - } else if (op1 == 5) { - // Move wide - if (opc == 1) { - E(" ERROR "); - } else { - const char* str = "n_zk"; - uint32_t hw = (inst >> 21u) & 3; - uint32_t imm = (inst >> 5u) & 0xFFFF; - uint32_t rn = inst & 0x1F; - - E(" mov%c ", str[opc]); - print_gpr(e, rn, is_64bit); - E(", #%#x", imm); - if (hw) { - E(", lsl #%d", hw*16); - } - } - } else if (op1 == 6) { - // Bitfield - static const char* strs[] = { "sbfm", "bfm", "ubfm", "error" }; - uint32_t immr = (inst >> 16u) & 0b111111; - uint32_t imms = (inst >> 10u) & 0b111111; - uint32_t rn = (inst >> 5u) & 0x1F; - uint32_t rd = (inst >> 0u) & 0x1F; - - if (opc == 2 && immr == imms + 1) { - // lsl - int amt = (is_64bit ? 63 : 31) - imms; - E(" lsl "); - print_gpr(e, rd, is_64bit); - E(", #%#x", amt); - } else { - E(" %s ", strs[opc]); - print_gpr(e, rd, is_64bit); - E(", "); - print_gpr(e, rn, is_64bit); - E(", #%#x, #%#x", immr, imms); - } - } else { - E(" DATA IMM "); - } - break; - } - - // Data processing (register) - case 0b0101: - case 0b1101: { - uint32_t sf = (inst >> 31u) & 1; - uint32_t op0 = (inst >> 30u) & 1; - uint32_t op1 = (inst >> 28u) & 1; - uint32_t op2 = (inst >> 21u) & 0xF; - uint32_t rm = (inst >> 16u) & 0x1F; - uint32_t rn = (inst >> 5u) & 0x1F; - uint32_t rd = (inst >> 0u) & 0x1F; - - __debugbreak(); - - const char* mnemonic = NULL; - if (op1) { // op1=1 - if (op2 == 0b0110) { - - } - - // Data processing (3 source) - uint32_t ra = (inst >> 10u) & 0x1F; - - if ((inst >> 29u) & 3) { - E("ERROR"); - } else { - uint32_t opc = (((inst >> 21u) & 7) << 1) | ((inst >> 15u) & 1); - switch (opc) { - case 0b0000: E(" madd "); break; - case 0b0001: E(" msub "); break; - default: tb_todo(); - } - } - print_gpr(e, rd, is_64bit); - E(", "); - print_gpr(e, rn, is_64bit); - E(", "); - print_gpr(e, rm, is_64bit); - E(", "); - print_gpr(e, ra, is_64bit); - - /*if (op0 == 0 && op2 == 0b0110) { - // uint32_t S = (inst >> 29u) & 1; - uint32_t opc = (inst >> 10u) & 0x3F; - - if (opc == 2) { - E(" udiv "); - } else if (opc == 3) { - E(" sdiv "); - } else { - E("ERROR "); - } - print_gpr(e, rd, is_64bit); - E(", "); - print_gpr(e, rn, is_64bit); - E(", "); - print_gpr(e, rm, is_64bit); - } else if (op2 >> 3) { - } else { - E(" DATA REG "); - }*/ - } else { // op1=0 - uint32_t imm = (inst >> 10u) & 0x3F; - uint32_t sh = (inst >> 22u) & 3; - - if (op2 < 0b1000) { // op2=0xxx - static const char* ops[] = { - // Logical (shifted register) - // n=0 n=1 opc - "and", "bic", // 00 - "orr", "orn", // 01 - "eor", "eon", // 10 - "ands", "bics" // 11 - }; - - uint32_t opc = (((inst >> 29u) & 3) << 1) | ((inst >> 21) & 1); - mnemonic = ops[opc]; - } else if (op2 & 1) { // op2=1xx1 - // Add/subtract (extended register) - tb_todo(); - } else { // op2=1xx0 - // Add/subtract (shifted register) - static const char* ops[] = { - "add", "adds", - "sub", "subs" - }; - uint32_t opc = (inst >> 29u) & 3; - mnemonic = ops[opc]; - } - - E(" %s ", mnemonic); - print_gpr(e, rd, is_64bit); - E(", "); - print_gpr(e, rn, is_64bit); - E(", "); - print_gpr(e, rm, is_64bit); - if (imm) { - E(", %s #%#x", sh_strs[sh], imm); - } - } - break; - } - - // Branches, Exceptions, Syscalls - case 0b1010: - case 0b1011: { - uint32_t op0 = (inst >> 29u) & 0b111; - uint32_t op1 = (inst >> 12u) & 0b11111111111111; - - if (op0 == 0b110 && op1 >> 13u) { - // Unconditional - // - // opc op2 op3 op4 - // 0010 11111 000000 00000 RET - uint32_t opc = (inst >> 21u) & 0xF; - uint32_t op2 = (inst >> 16u) & 0x1F; - uint32_t op3 = (inst >> 10u) & 0x1F; - uint32_t rn = (inst >> 5u) & 0x1F; - uint32_t op4 = (inst >> 0u) & 0x1F; - if (opc == 2 && op2 == 0x1F && op3 == 0 && op4 == 0) { - if (rn == LR) { E(" ret"); } - else { E(" ret v%d", rn); } - } else { - tb_todo(); - } - } else { - E(" BRANCH(op1=%#x) ", op1); - } - break; - } - - default: tb_todo(); - } - E("\n"); - } -} -#undef E - -static size_t emit_call_patches(TB_Module* restrict m, TB_FunctionOutput* out_f) { - return 0; -} - -ICodeGen tb__aarch64_codegen = { - .minimum_addressable_size = 8, - .pointer_size = 64, - .emit_win64eh_unwind_info = NULL, - .emit_call_patches = emit_call_patches, - .get_data_type_size = get_data_type_size, - .compile_function = compile_function, -}; -#else -ICodeGen tb__aarch64_codegen; -#endif diff --git a/vendor/tb/src/abi.c b/vendor/tb/src/abi.c deleted file mode 100644 index 36ebb72d..00000000 --- a/vendor/tb/src/abi.c +++ /dev/null @@ -1,247 +0,0 @@ -// This is gonna get complicated but we can push through :p - -//////////////////////////////// -// x86-64 -//////////////////////////////// -// Our two ABIs are System-V and Win64, important to note that -// returns go through 0=RAX, 1=RDX so references to these in terms -// of returns will mean that. - -// we finna retrofit systemv terms onto -// windows, it's not too big of a deal. -typedef enum { - RG_NONE, - // GPRs - RG_INTEGER, - // vector registers - RG_SSE, RG_SSEUP, - // stack slot - RG_MEMORY, -} RegClass; - -static int debug_type_size(TB_ABI abi, TB_DebugType* t) { - while (t->tag == TB_DEBUG_TYPE_ALIAS) { - t = t->alias.type; - } - - switch (t->tag) { - case TB_DEBUG_TYPE_VOID: return 0; - case TB_DEBUG_TYPE_BOOL: return 1; - case TB_DEBUG_TYPE_UINT: return (t->int_bits + 7) / 8; - case TB_DEBUG_TYPE_INT: return (t->int_bits + 7) / 8; - - case TB_DEBUG_TYPE_FUNCTION: return 8; - case TB_DEBUG_TYPE_ARRAY: return 8; - case TB_DEBUG_TYPE_POINTER: return 8; - - case TB_DEBUG_TYPE_FLOAT: { - switch (t->float_fmt) { - case TB_FLT_32: return 4; - case TB_FLT_64: return 8; - } - } - - case TB_DEBUG_TYPE_STRUCT: - case TB_DEBUG_TYPE_UNION: - return t->record.size; - - default: tb_todo(); - } - return 0; -} - -static int debug_type_align(TB_ABI abi, TB_DebugType* t) { - while (t->tag == TB_DEBUG_TYPE_ALIAS) { - t = t->alias.type; - } - - if (t->tag == TB_DEBUG_TYPE_STRUCT || t->tag == TB_DEBUG_TYPE_UNION) { - return t->record.align; - } - - return debug_type_size(abi, t); -} - -static RegClass classify_reg(TB_ABI abi, TB_DebugType* t) { - switch (abi) { - // [https://learn.microsoft.com/en-us/cpp/build/x64-calling-convention] - // Any argument that doesn't fit in 8 bytes, or isn't 1, 2, 4, or 8 bytes, - // must be passed by reference. A single argument is never spread across - // multiple registers. - case TB_ABI_WIN64: { - int s = debug_type_size(abi, t); - if (s == 1 || s == 2 || s == 4 || s == 8) { - return t->tag == TB_DEBUG_TYPE_FLOAT ? RG_SSE : RG_INTEGER; - } - - return RG_MEMORY; - } - - case TB_ABI_SYSTEMV: { - int s = debug_type_size(abi, t); - if (s <= 8) { - return t->tag == TB_DEBUG_TYPE_FLOAT ? RG_SSE : RG_INTEGER; - } - - return RG_MEMORY; - } - - default: tb_todo(); - } -} - -static TB_DataType debug_type_to_tb(TB_DebugType* t) { - switch (t->tag) { - case TB_DEBUG_TYPE_VOID: return TB_TYPE_VOID; - case TB_DEBUG_TYPE_BOOL: return TB_TYPE_I8; - case TB_DEBUG_TYPE_UINT: return TB_TYPE_INTN(t->int_bits); - case TB_DEBUG_TYPE_INT: return TB_TYPE_INTN(t->int_bits); - - case TB_DEBUG_TYPE_FUNCTION: return TB_TYPE_PTR; - case TB_DEBUG_TYPE_ARRAY: return TB_TYPE_PTR; - case TB_DEBUG_TYPE_POINTER: return TB_TYPE_PTR; - - case TB_DEBUG_TYPE_FLOAT: return (TB_DataType){ { TB_FLOAT, t->float_fmt } }; - - default: tb_assert(0, "todo"); return TB_TYPE_VOID; - } -} - -static TB_DataType reg_class_to_tb(TB_ABI abi, RegClass rg, TB_DebugType* type) { - switch (rg) { - case RG_MEMORY: return TB_TYPE_PTR; - case RG_INTEGER: { - if (type->tag == TB_DEBUG_TYPE_POINTER) return TB_TYPE_PTR; - return TB_TYPE_INTN(debug_type_size(abi, type) * 8); - } - - case RG_SSE: { - assert(type->tag == TB_DEBUG_TYPE_FLOAT); - return (TB_DataType){ { TB_FLOAT, type->float_fmt } }; - } - - default: tb_assert(0, "todo"); return TB_TYPE_VOID; - } -} - -TB_API TB_DebugType* tb_debug_field_type(TB_DebugType* type) { - assert(type->tag == TB_DEBUG_TYPE_FIELD); - return type->field.type; -} - -TB_PassingRule tb_get_passing_rule_from_dbg(TB_Module* mod, TB_DebugType* param_type, bool is_return) { - return classify_reg(mod->target_abi, param_type) == RG_MEMORY ? TB_PASSING_INDIRECT : TB_PASSING_DIRECT; -} - -TB_Node** tb_function_set_prototype_from_dbg(TB_Function* f, TB_ModuleSectionHandle section, TB_DebugType* dbg, TB_Arena* arena, size_t* out_param_count) { - tb_assert(dbg->tag == TB_DEBUG_TYPE_FUNCTION, "type has to be a function"); - tb_assert(dbg->func.return_count <= 1, "C can't do multiple returns and thus we can't lower it into C from here, try tb_function_set_prototype and do it manually"); - - TB_ABI abi = f->super.module->target_abi; - TB_FunctionPrototype* p = tb_prototype_from_dbg(f->super.module, dbg); - - // apply prototype - tb_function_set_prototype(f, section, p, arena); - - size_t param_count = dbg->func.param_count; - TB_DebugType** param_list = dbg->func.params; - - // reassemble values - TB_Node** params = NULL; - if (dbg->func.param_count > 0) { - params = tb_arena_alloc(f->arena, sizeof(TB_Node*) * param_count); - - bool has_aggregate_return = dbg->func.return_count > 0 && classify_reg(abi, dbg->func.returns[0]) == RG_MEMORY; - FOREACH_N(i, 0, param_count) { - TB_DebugType* type = param_list[i]->field.type; - const char* name = param_list[i]->field.name; - size_t name_len = param_list[i]->field.len; - - int size = debug_type_size(abi, type); - int align = debug_type_align(abi, type); - - // place values into memory - TB_Node* v = tb_inst_param(f, i + has_aggregate_return); - - RegClass rg = classify_reg(abi, type); - if (rg == RG_MEMORY) { - params[i] = v; - } else { - TB_Node* slot = tb_inst_local(f, size, align); - tb_inst_store(f, p->params[i + has_aggregate_return].dt, slot, v, align, false); - params[i] = slot; - } - - // mark debug info - tb_function_attrib_variable(f, params[i], NULL, name_len, name, type); - } - } - - *out_param_count = param_count; - return params; -} - -TB_API TB_FunctionPrototype* tb_prototype_from_dbg(TB_Module* m, TB_DebugType* dbg) { - TB_ABI abi = m->target_abi; - - // aggregate return means the first parameter will be used for - // a pointer to where the output should be written. - // - // TODO(NeGate): it's uninitialized by default but we don't communicate - // this to the IR yet. - RegClass return_rg = RG_NONE; - TB_PrototypeParam ret = { TB_TYPE_VOID }; - if (dbg->func.return_count == 1) { - return_rg = classify_reg(abi, dbg->func.returns[0]); - - ret.debug_type = dbg->func.returns[0]; - ret.dt = reg_class_to_tb(abi, return_rg, dbg->func.returns[0]); - ret.name = "$ret"; - } - - bool has_aggregate_return = return_rg == RG_MEMORY; - - // estimate the number of parameters: - // * in win64 this is easy, parameters don't split. - // * in sysv this is a nightmare, structs are usually - // the culprit because they can be split up. - size_t param_count = dbg->func.param_count; - TB_DebugType** param_list = dbg->func.params; - if (abi == TB_ABI_SYSTEMV) { - // tb_todo(); - } - - // build up prototype param types - size_t return_count = dbg->func.return_count; - size_t size = sizeof(TB_FunctionPrototype) + ((param_count + has_aggregate_return + return_count) * sizeof(TB_PrototypeParam)); - TB_FunctionPrototype* p = tb_arena_alloc(get_permanent_arena(m), size); - p->call_conv = dbg->func.cc; - p->has_varargs = dbg->func.has_varargs; - p->return_count = return_count; - p->param_count = has_aggregate_return + param_count; - - if (dbg->func.param_count > 0) { - FOREACH_N(i, 0, dbg->func.param_count) { - TB_DebugType* type = param_list[i]->field.type; - RegClass rg = classify_reg(abi, type); - - TB_PrototypeParam param = { - .name = param_list[i]->field.name, - .debug_type = type, - .dt = reg_class_to_tb(abi, rg, type), - }; - - p->params[has_aggregate_return + i] = param; - } - } - - if (p->return_count == 1) { - if (has_aggregate_return) { - p->params[0] = ret; - } - - p->params[p->param_count] = ret; - } - - return p; -} diff --git a/vendor/tb/src/builtins.h b/vendor/tb/src/builtins.h deleted file mode 100644 index 8c9c2b5b..00000000 --- a/vendor/tb/src/builtins.h +++ /dev/null @@ -1,108 +0,0 @@ -#pragma once -#include - -typedef struct TB_MultiplyResult { - uint64_t lo; - uint64_t hi; -} TB_MultiplyResult; - -#if defined(_MSC_VER) && !defined(__clang__) -static int tb_clz64(uint64_t x) { - return _lzcnt_u64(x); -} - -static int tb_ffs(uint32_t x) { - unsigned long index; - return _BitScanForward(&index, x) ? 1 + index : 0; -} - -static int tb_ffs64(uint64_t x) { - unsigned long index; - return _BitScanForward64(&index, x) ? 1 + index : 0; -} - -static int tb_popcount(uint32_t x) { - return __popcnt(x); -} - -static int tb_popcount64(uint64_t x) { - return __popcnt64(x); -} - -static uint64_t tb_next_pow2(uint64_t x) { - return x == 1 ? 1 : 1 << (64 - _lzcnt_u64(x - 1)); -} - -static bool tb_add_overflow(uint64_t a, uint64_t b, uint64_t* result) { - uint64_t c = a + b; - *result = c; - return c < a; -} - -static bool tb_sub_overflow(uint64_t a, uint64_t b, uint64_t* result) { - uint64_t c = a - b; - *result = c; - return c > a; -} - -#pragma intrinsic(_udiv128) -static uint64_t tb_div128(uint64_t ahi, uint64_t alo, uint64_t b) { - uint64_t rem; - return _udiv128(ahi, alo, b, &rem); -} - -#pragma intrinsic(_umul128) -static TB_MultiplyResult tb_mul64x128(uint64_t a, uint64_t b) { - uint64_t hi; - uint64_t lo = _umul128(a, b, &hi); - return (TB_MultiplyResult) { lo, hi }; -} -#else -static int tb_clz64(uint64_t x) { - return __builtin_clzll(x); -} - -static int tb_ffs(uint32_t x) { - return __builtin_ffs(x); -} - -static int tb_ffs64(uint64_t x) { - return __builtin_ffsll(x); -} - -static int tb_popcount(uint32_t x) { - return __builtin_popcount(x); -} - -static int tb_popcount64(uint64_t x) { - return __builtin_popcountll(x); -} - -static uint64_t tb_next_pow2(uint64_t x) { - return x == 1 ? 1 : 1 << (64 - __builtin_clzll(x - 1)); -} - -static bool tb_add_overflow(uint64_t a, uint64_t b, uint64_t* result) { - return __builtin_add_overflow(a, b, result); -} - -static bool tb_sub_overflow(uint64_t a, uint64_t b, uint64_t* result) { - return __builtin_sub_overflow(a, b, result); -} - -static TB_MultiplyResult tb_mul64x128(uint64_t a, uint64_t b) { - __uint128_t product = (__uint128_t)a * (__uint128_t)b; - - return (TB_MultiplyResult) { (uint64_t)(product & 0xFFFFFFFFFFFFFFFF), (uint64_t)(product >> 64) }; -} - -static uint64_t tb_div128(uint64_t ahi, uint64_t alo, uint64_t b) { - // We don't want 128 bit software division - uint64_t d, e; - __asm__("divq %[b]" - : "=a"(d), "=d"(e) - : [b] "r"(b), "a"(alo), "d"(ahi) - ); - return d; -} -#endif diff --git a/vendor/tb/src/chaitin.c b/vendor/tb/src/chaitin.c deleted file mode 100644 index ae0533e4..00000000 --- a/vendor/tb/src/chaitin.c +++ /dev/null @@ -1,268 +0,0 @@ -// TODO(NeGate): implement Chaitin-Briggs, if you wanna contribute this would be cool to work -// with someone else on. -#include "codegen.h" -#include - -typedef struct { - Ctx* ctx; - TB_Arena* arena; - - int spills; - - int num_classes; - int* num_regs; - uint64_t* callee_saved; - RegMask* normie_mask; - - size_t ifg_stride; - size_t ifg_len; - uint64_t* ifg; - int* degree; -} Chaitin; - -static void ifg_edge(Chaitin* ra, int i, int j) { - ra->ifg[i*ra->ifg_stride + j/64] |= 1ull << (j % 64); -} - -static bool ifg_test(Chaitin* ra, int i, int j) { - return ra->ifg[i*ra->ifg_stride + j/64] & (1ull << (j % 64)); -} - -static void ifg_remove(Chaitin* ra, int i, int j) { - printf("remove v%d -- v%d\n", i, j); - assert(ifg_test(ra, i, j)); - ra->ifg[i*ra->ifg_stride + j/64] &= ~(1ull << (j % 64)); -} - -static void ifg_remove_edges(Chaitin* ra, int i) { - FOREACH_N(j, 0, ra->ifg_stride) { - uint64_t bits = ra->ifg[i*ra->ifg_stride + j]; - if (bits == 0) continue; - - FOREACH_N(k, 0, 64) if ((bits >> k) & 1) { - ifg_remove(ra, j*64 + k, i); - ra->degree[j*64 + k] -= 1; - } - - // reset all the bits - ra->ifg[i*ra->ifg_stride + j] = 0; - } - ra->degree[i] = 0; -} - -static bool ifg_empty(Chaitin* ra) { - FOREACH_N(i, 0, ra->ifg_len) { - if (ra->degree[i]) return false; - } - - return true; -} - -static bool reg_mask_intersect(RegMask a, RegMask b) { - if (a.class > b.class) { - SWAP(RegMask, a, b); - } - - if (a.class == REG_CLASS_STK && a.mask == 0) { - return b.may_spill || (b.class == REG_CLASS_STK && b.mask == 0); - } else { - return a.class == b.class && (a.mask & b.mask) != 0; - } -} - -static bool reg_mask_may_stack(RegMask a) { - return a.class == REG_CLASS_STK || a.may_spill; -} - -static void build_ifg(Ctx* restrict ctx, TB_Arena* arena, Chaitin* ra) { - size_t normie_nodes = ctx->interval_count - ctx->num_fixed; - - ra->ifg_stride = (normie_nodes + 63) / 64; - ra->ifg_len = normie_nodes; - ra->ifg = tb_arena_alloc(arena, ra->ifg_len * ra->ifg_stride * sizeof(uint64_t)); - ra->degree = tb_arena_alloc(arena, ra->ifg_len * sizeof(int)); - - memset(ra->ifg, 0, ra->ifg_len * ra->ifg_stride * sizeof(uint64_t)); - memset(ra->degree, 0, ra->ifg_len * sizeof(int)); - - Set live = set_create_in_arena(arena, ctx->interval_count); - FOREACH_REVERSE_N(i, 0, ctx->bb_count) { - MachineBB* mbb = &ctx->machine_bbs[i]; - - Set* live_out = &mbb->live_out; - set_copy(&live, live_out); - - for (Tile* t = mbb->end; t; t = t->prev) { - FOREACH_N(j, 0, t->out_count) { - LiveInterval* interval = t->outs[j]; - set_remove(&live, interval->id); - - // interfere - FOREACH_N(k, 0, ra->ifg_len) { - if (!set_get(&live, k)) continue; - - LiveInterval* other = ctx->id2interval[k]; - if (!reg_mask_intersect(interval->mask, other->mask)) continue; - - printf("v%d -- v%td\n", interval->id, k); - ifg_edge(ra, interval->id, k); - ifg_edge(ra, k, interval->id); - } - - // 2 address ops will interfere with their own inputs (except for - // shared dst/src) - if (t->n && ctx->_2addr(t->n)) { - FOREACH_N(k, 1, t->in_count) { - LiveInterval* in_def = t->ins[k].src; - if (in_def == NULL) continue; - if (!reg_mask_intersect(interval->mask, in_def->mask)) continue; - - printf("v%d -- v%d\n", interval->id, in_def->id); - ifg_edge(ra, interval->id, in_def->id); - ifg_edge(ra, in_def->id, interval->id); - } - } - } - - // uses are live now - FOREACH_N(j, 0, t->in_count) { - LiveInterval* in_def = t->ins[j].src; - if (in_def) set_put(&live, in_def->id); - } - } - } - - // compute degree - FOREACH_N(i, 0, ra->ifg_len) { - int sum = 0; - FOREACH_N(j, 0, ra->ifg_stride) { - sum += tb_popcount64(ra->ifg[i*ra->ifg_stride + j]); - } - ra->degree[i] = sum; - } -} - -void tb__chaitin(Ctx* restrict ctx, TB_Arena* arena) { - Chaitin ra = { .ctx = ctx, .arena = arena, .spills = ctx->num_regs[0] }; - - CUIK_TIMED_BLOCK("build IFG") { - // simplify/select stack - int cnt, cap; - LiveInterval** stk; - - bool has_spills; - for (;;) { - TB_ArenaSavepoint sp = tb_arena_save(arena); - - // build IFG (and degree table) - build_ifg(ctx, arena, &ra); - - // clone before doing all the fancy node removals - uint64_t* ifg_copy = tb_arena_alloc(arena, ra.ifg_len * ra.ifg_stride * sizeof(uint64_t)); - int* deg_copy = tb_arena_alloc(arena, ra.ifg_len * sizeof(int)); - memcpy(ifg_copy, ra.ifg, ra.ifg_len * ra.ifg_stride * sizeof(uint64_t)); - memcpy(deg_copy, ra.degree, ra.ifg_len * sizeof(int)); - - // compute spill weights - float* costs = tb_arena_alloc(arena, ra.ifg_len * sizeof(float)); - FOREACH_N(i, 1, ra.ifg_len) { - float w = deg_copy[i]; - costs[i] = deg_copy[i]; - } - - // simplify/select stack - cap = ra.ifg_len; - stk = tb_arena_alloc(arena, ra.ifg_len * sizeof(LiveInterval*)); - - NL_ChunkedArr spills = nl_chunked_arr_alloc(tmp_arena); - - // simplify => split cycle - simplify_split: { - cnt = 0; - - // simplify phase: - // push all nodes with a degree < k - FOREACH_N(i, 0, ra.ifg_len) if (ctx->id2interval[i]) { - RegMask mask = ctx->id2interval[i]->mask; - int class = mask.class; - int k_limit = class != REG_CLASS_STK ? tb_popcount64(mask.mask) : INT_MAX; - - if (ra.degree[i] < k_limit) { - assert(cnt < cap); - stk[cnt++] = ctx->id2interval[i]; - ifg_remove_edges(&ra, i); - } - } - - // split phase: - if (!ifg_empty(&ra)) { - int best_spill = -1; - float best_cost = FLT_MAX; - - // pick next best spill - FOREACH_N(i, 0, ra.ifg_len) if (ctx->id2interval[i]) { - RegMask mask = ctx->id2interval[i]->mask; - int class = mask.class; - int k_limit = class != REG_CLASS_STK ? tb_popcount64(mask.mask) : INT_MAX; - - if (ra.degree[i] >= k_limit && costs[i] < best_cost) { - best_cost = costs[i]; - best_spill = i; - } - } - - printf("spill v%d\n", best_spill); - - nl_chunked_arr_put(&spills, ctx->id2interval[best_spill]); - ifg_remove_edges(&ra, best_spill); - goto simplify_split; - } - } - - ra.ifg = ifg_copy; - ra.degree = deg_copy; - - // insert spill code - for (NL_ArrChunk* restrict chk = spills.first; chk; chk = chk->next) { - FOREACH_N(i, 0, chk->count) { - LiveInterval* spill = chk->elems[i]; - - spill->class = REG_CLASS_STK; - spill->assigned = ra.spills++; - } - } - - // finally... we're done - if (spills.first->count == 0) { - break; - } - - tb_arena_restore(arena, sp); - } - - // select phase - while (cnt) { - LiveInterval* interval = stk[--cnt]; - int i = interval->id; - - uint64_t mask = interval->mask.mask; - FOREACH_N(j, 0, ra.ifg_stride) { - uint64_t bits = ra.ifg[i*ra.ifg_stride + j]; - if (bits == 0) continue; - - FOREACH_N(k, 0, 64) if ((bits >> k) & 1) { - assert(ctx->id2interval[j*64 + k]->mask.class == interval->mask.class); - uint64_t assign_mask = 1ull << ctx->id2interval[j*64 + k]->assigned; - mask &= ~assign_mask; - } - } - - assert(mask != 0 && "couldn't select color :("); - interval->class = interval->mask.class; - interval->assigned = tb_ffs64(mask) - 1; - printf("v%d = R%d\n", i, interval->assigned); - } - } - - ctx->stack_usage += (ra.spills - ctx->num_regs[0]) * 8; -} diff --git a/vendor/tb/src/codegen.h b/vendor/tb/src/codegen.h deleted file mode 100644 index e855c06b..00000000 --- a/vendor/tb/src/codegen.h +++ /dev/null @@ -1,309 +0,0 @@ -// This is the general codegen framework, it combines together the various components in -// the target dependent part, if you see any of these TODOs and wanna contribute, those -// are good places to go. -// -// This can be broken down into a few steps: -// -// GCM => Init => Per BB: { Local sched, Tiling } => Dataflow => Regalloc => Emit -// -// Here's where you can find more info on these: -// -// * GCM: handled by the tb_pass_schedule (TODO ways to skew global scheduling). -// -// * Init: This is where you can process ABI details and fill in certain important details -// for later like ctx.sched which is the scheduler, and ctx.regalloc which is the register -// allocator. -// -// * Local sched: handled by ctx.sched, it's just a function you can replace with your -// own based on whatever you wanna achieve, for now i've only got the topo sort greedy -// sched (TODO implement a latency-based list scheduler) -// -// * Tiling: handled by isel_node, by default we walk the BB backwards building one tile -// per node, we can call fold_node to say we expended a use of a value, this way if we -// we've folded all uses of a value we don't materialize it alone. isel_node has to fill -// in the register constraints, it'll return the output mask and construct input masks. -// -// * Regalloc: handled by ctx.regalloc (TODO implement more register allocators) -// -// * Emit: handled by emit_tile, writes out the bytes now that we've completed regalloc. -// -#pragma once - -#include "opt/passes.h" -#include "emitter.h" - -enum { - // all we can fit into 3bits, but also... 8 classes is a lot. - // - // * x86 has 3 currently: GPR, Vector, and Stack. - MAX_REG_CLASSES = 8, - - // every platform has a stack regclass, the mask is actually an offset - // on the stack (useful for parameter passing). - REG_CLASS_STK = 0, -}; - -// represents a set of registers, usually for register constraints. -// we can also say that a value might fit into the stack with may_spill. -typedef struct { - uint64_t class : 3; - uint64_t may_spill : 1; - uint64_t mask : 60; -} RegMask; - -#define REGMASK(c, m) (RegMask){ REG_CLASS_ ## c, 0, m } -#define REGMASK2(c, m) (RegMask){ REG_CLASS_ ## c, 1, m } -#define REGEMPTY (RegMask){ 0 } - -typedef struct { - int start, end; -} LiveRange; - -typedef struct { - int may_spill : 1; - int pos : 31; -} UsePos; - -typedef enum { - // single node tile - TILE_NORMAL, - // SoN doesn't have a jump op, this serves that purpose - TILE_GOTO, - // performs a move operation between two live intervals - TILE_SPILL_MOVE, - // debug line - TILE_LOCATION, -} TileTag; - -typedef struct Tile Tile; -typedef struct LiveInterval LiveInterval; - -typedef struct { - LiveInterval* src; - RegMask mask; -} TileInput; - -struct LiveInterval { - int id; // used by live sets - int active_range; - - RegMask mask; - TB_DataType dt; - - Tile* tile; - - int8_t class; - int8_t reg; - int8_t assigned; - - LiveInterval* hint; - LiveInterval* split_kid; - - int use_cap, use_count; - UsePos* uses; - - int range_cap, range_count; - LiveRange* ranges; -}; - -// represents a pattern of instructions -struct Tile { - struct Tile* prev; - struct Tile* next; - - TileTag tag : 8; - uint32_t flags : 24; - - int time; - - union { - // tag = TILE_GOTO, this is the successor - TB_Node* succ; - - // tag = TILE_SPILL_MOVE - TB_DataType spill_dt; - - // tag = TILE_NORMAL - void* aux; - - // tag = TILE_LOCATION - TB_NodeLocation* loc; - }; - - int out_count; - int in_count; - TileInput* ins; - - TB_Node* n; - LiveInterval* outs[]; -}; - -typedef struct { - int id; - - TB_Node* n; - TB_Node* end_n; - - Tile* start; - Tile* end; - - // dataflow - Set gen, kill; - Set live_in, live_out; -} MachineBB; - -typedef struct { - TB_SymbolPatch* patch; - TB_Location* loc; - TB_Location* end; - Comment* comment; -} Disasm; - -// Static-sized hash map -typedef struct { - TB_Node* k; - MachineBB* v; -} NodeToBB; - -typedef struct { - // number of times we asked for it - // to materialize, only certain nodes - // can be rematerialized several times - // like constants. - int mat_count; - int use_count; - Tile* tile; -} ValueDesc; - -typedef struct Ctx Ctx; -typedef void (*TB_RegAlloc)(Ctx* restrict ctx, TB_Arena* arena); -typedef bool (*TB_2Addr)(TB_Node* n); - -// how many live intervals do we need to allocate to actually store -// this register, usually 0 or 1 but occassionally (especially on 32bit -// platforms when doing 64bit arithmetic) we'll get 2. -typedef int (*TB_RegCount)(Ctx* restrict ctx, TB_Node* n); - -typedef struct { - uint32_t* pos; - uint32_t target; -} JumpTablePatch; - -typedef struct { - int class, reg; - int stk; -} CalleeSpill; - -struct Ctx { - TB_Passes* p; - TB_CGEmitter emit; - - TB_Module* module; - TB_Function* f; - TB_FeatureSet features; - - // user-provided details - TB_Scheduler sched; - TB_RegAlloc regalloc; - TB_RegCount regcnt; - TB_2Addr _2addr; - - // target-dependent index - int abi_index; - int fallthrough; - - uint8_t prologue_length; - uint8_t epilogue_length; - uint8_t nop_pads; - - // TB_Node* -> MachineBB* - struct { - size_t exp; - NodeToBB* entries; - } node_to_bb; - - // Basic blocks - int bb_count; - MachineBB* machine_bbs; - - // Values - ValueDesc* values; // [n.gvn] - LiveInterval** id2interval; // [tile.id] - LiveInterval* fixed[MAX_REG_CLASSES]; - - // Regalloc - int initial_spills; - int stack_slot_size; - int interval_count; - int stack_header; - int stack_usage; - int call_usage; - int num_fixed; - int num_classes; - int num_regs[MAX_REG_CLASSES]; - uint64_t callee_saved[MAX_REG_CLASSES]; - - // where scratch registers can go, a mask is used to avoid - // allocating special regiters. - RegMask normie_mask[MAX_REG_CLASSES]; - - DynArray(CalleeSpill) callee_spills; - NL_Map(TB_Node*, int) stack_slots; - DynArray(TB_StackSlot) debug_stack_slots; - DynArray(JumpTablePatch) jump_table_patches; - - // Line info - MachineBB* current_emit_bb; - int current_emit_bb_pos; - - DynArray(TB_Location) locations; -}; - -void tb__lsra(Ctx* restrict ctx, TB_Arena* arena); -void tb__chaitin(Ctx* restrict ctx, TB_Arena* arena); - -void tb__print_regmask(RegMask mask); - -static int fixed_reg_mask(RegMask mask) { - if (mask.class == REG_CLASS_STK) { - return mask.mask; - } else { - return tb_popcount64(mask.mask) == 1 ? 63 - tb_clz64(mask.mask) : -1; - } -} - -static uint32_t node_to_bb_hash(void* ptr) { return (((uintptr_t) ptr) * 11400714819323198485ull) >> 32ull; } -static MachineBB* node_to_bb(Ctx* restrict ctx, TB_Node* n) { - uint32_t h = node_to_bb_hash(n); - - size_t mask = (1 << ctx->node_to_bb.exp) - 1; - size_t first = h & mask, i = first; - do { - if (ctx->node_to_bb.entries[i].k == n) { - return ctx->node_to_bb.entries[i].v; - } - - i = (i + 1) & mask; - } while (i != first); - - abort(); -} - -static void node_to_bb_put(Ctx* restrict ctx, TB_Node* n, MachineBB* bb) { - uint32_t h = node_to_bb_hash(n); - - size_t mask = (1 << ctx->node_to_bb.exp) - 1; - size_t first = h & mask, i = first; - do { - if (ctx->node_to_bb.entries[i].k == NULL) { - ctx->node_to_bb.entries[i].k = n; - ctx->node_to_bb.entries[i].v = bb; - return; - } - - i = (i + 1) & mask; - } while (i != first); - - abort(); -} - diff --git a/vendor/tb/src/codegen_impl.h b/vendor/tb/src/codegen_impl.h deleted file mode 100644 index 92cf931f..00000000 --- a/vendor/tb/src/codegen_impl.h +++ /dev/null @@ -1,726 +0,0 @@ -// See codegen.h for more details, this is the implementation file for it, each target -// will include this to define their own copy of the codegen. -// -// Your job is to implement: -// isel_node, init_ctx, emit_tile, disassemble. -// -#include "codegen.h" - -// Implemented by the target, go read the mips_target.c docs on this. -static void isel_node(Ctx* restrict ctx, Tile* dst, TB_Node* n); -static int reg_count(Ctx* restrict ctx, TB_Node* n); -static void init_ctx(Ctx* restrict ctx, TB_ABI abi); -static void emit_tile(Ctx* restrict ctx, TB_CGEmitter* e, Tile* t); -static void pre_emit(Ctx* restrict ctx, TB_CGEmitter* e, TB_Node* n); -static void post_emit(Ctx* restrict ctx, TB_CGEmitter* e); -static void on_basic_block(Ctx* restrict ctx, TB_CGEmitter* e, int bb); -static void disassemble(TB_CGEmitter* e, Disasm* restrict d, int bb, size_t pos, size_t end); - -static ValueDesc* val_at(Ctx* restrict ctx, TB_Node* n) { - if (ctx->values[n->gvn].use_count < 0) { - int count = 0; - FOR_USERS(u, n) count++; - ctx->values[n->gvn].use_count = count; - } - - return &ctx->values[n->gvn]; -} - -static Tile* get_tile(Ctx* restrict ctx, TB_Node* n) { - ValueDesc* val = val_at(ctx, n); - if (val->tile == NULL) { - int nregs = ctx->regcnt(ctx, n); - Tile* tile = tb_arena_alloc(tmp_arena, sizeof(Tile) + nregs*sizeof(LiveInterval*)); - *tile = (Tile){ .tag = TILE_NORMAL, .n = n, .out_count = nregs }; - - FOREACH_N(i, 0, nregs) { - tile->outs[i] = tb_arena_alloc(tmp_arena, sizeof(LiveInterval)); - *tile->outs[i] = (LiveInterval){ - .tile = tile, - .id = ctx->interval_count++, - .reg = -1, - .assigned = -1, - .range_cap = 4, .range_count = 1, - .ranges = tb_platform_heap_alloc(8 * sizeof(LiveRange)) - }; - tile->outs[i]->ranges[0] = (LiveRange){ INT_MAX, INT_MAX }; - } - val->tile = tile; - } - - return val->tile; -} - -static LiveInterval* get_interval(Ctx* restrict ctx, TB_Node* n, int i) { - Tile* t = get_tile(ctx, n); - assert(i < t->out_count); - return t->outs[i]; -} - -// you're expected to set the masks in the returned array -static TileInput* tile_set_ins(Ctx* restrict ctx, Tile* t, TB_Node* n, int start, int end) { - t->ins = tb_arena_alloc(tmp_arena, (end - start) * sizeof(TileInput)); - t->in_count = end - start; - FOREACH_N(i, start, end) { - t->ins[i - start].src = get_interval(ctx, n->inputs[i], 0); - } - return t->ins; -} - -// fills all inputs with the same mask -static TileInput* tile_broadcast_ins(Ctx* restrict ctx, Tile* t, TB_Node* n, int start, int end, RegMask rm) { - t->ins = tb_arena_alloc(tmp_arena, (end - start) * sizeof(TileInput)); - t->in_count = end - start; - FOREACH_N(i, start, end) { - t->ins[i - start].src = get_tile(ctx, n->inputs[i])->outs[0]; - t->ins[i - start].mask = rm; - } - return t->ins; -} - -static int try_init_stack_slot(Ctx* restrict ctx, TB_Node* n) { - if (n->type == TB_LOCAL) { - TB_NodeLocal* local = TB_NODE_GET_EXTRA(n); - ptrdiff_t search = nl_map_get(ctx->stack_slots, n); - if (search >= 0) { - return ctx->stack_slots[search].v; - } else { - ctx->stack_usage = align_up(ctx->stack_usage + local->size, local->align); - nl_map_put(ctx->stack_slots, n, ctx->stack_usage); - return ctx->stack_usage; - } - } else { - return 0; - } -} - -static int get_stack_slot(Ctx* restrict ctx, TB_Node* n) { - int pos = nl_map_get_checked(ctx->stack_slots, n); - return ctx->stack_usage - pos; -} - -static LiveInterval* canonical_interval(Ctx* restrict ctx, LiveInterval* interval, RegMask mask) { - int reg = fixed_reg_mask(mask); - if (reg >= 0) { - return &ctx->fixed[mask.class][reg]; - } else { - return interval; - } -} - -void tb__print_regmask(RegMask mask) { - if (mask.class == REG_CLASS_STK) { - if (mask.mask == 0) { - printf("[any spill]"); - } else { - printf("[SP + %"PRId64"]", mask.mask*8); - } - } else { - int i = 0; - bool comma = false; - uint64_t bits = mask.mask; - - printf("["); - while (bits) { - // skip zeros - int skip = __builtin_ffs(bits) - 1; - i += skip, bits >>= skip; - - if (!comma) { - comma = true; - } else { - printf(", "); - } - - // find sequence of ones - int len = __builtin_ffs(~bits) - 1; - printf("R%d", i); - if (len > 1) { - printf(" .. R%d", i+len-1); - } - - // skip ones - bits >>= len, i += len; - } - - if (mask.may_spill) { - printf(" | SPILL"); - } - printf("]"); - } -} - -static bool null_2addr(TB_Node* n) { return false; } -static void compile_function(TB_Passes* restrict p, TB_FunctionOutput* restrict func_out, const TB_FeatureSet* features, TB_Arena* code_arena, bool emit_asm) { - verify_tmp_arena(p); - - TB_Arena* arena = tmp_arena; - TB_ArenaSavepoint sp = tb_arena_save(arena); - - TB_Function* restrict f = p->f; - TB_OPTDEBUG(CODEGEN)(tb_pass_print(p)); - - Ctx ctx = { - .module = f->super.module, - .f = f, - .p = p, - .num_classes = REG_CLASS_COUNT, - .regcnt = reg_count, - .emit = { - .output = func_out, - .arena = arena, - .has_comments = false, - } - }; - - // allocate entire top of the code arena (we'll trim it later if possible) - ctx.emit.capacity = code_arena->high_point - code_arena->watermark; - ctx.emit.data = tb_arena_alloc(code_arena, ctx.emit.capacity); - - if (features == NULL) { - ctx.features = (TB_FeatureSet){ 0 }; - } else { - ctx.features = *features; - } - - Worklist* restrict ws = &p->worklist; - - // legalize step takes out any of our 16bit and 8bit math ops - tb_pass_prep(p); - tb_pass_legalize(p, f->super.module->target_arch); - - worklist_clear(ws); - - TB_CFG cfg; - CUIK_TIMED_BLOCK("global sched") { - // We need to generate a CFG - cfg = tb_compute_rpo(f, p); - // And perform global scheduling - tb_pass_schedule(p, cfg, false); - } - - nl_map_create(ctx.stack_slots, 4); - init_ctx(&ctx, f->super.module->target_abi); - - if (ctx._2addr == NULL) { - ctx._2addr = null_2addr; - } - - ctx.values = tb_arena_alloc(arena, f->node_count * sizeof(Tile*)); - memset(ctx.values, 0, f->node_count * sizeof(Tile*)); - - ctx.values = tb_arena_alloc(arena, f->node_count * sizeof(ValueDesc)); - FOREACH_N(i, 0, f->node_count) { - ctx.values[i].use_count = -1; - ctx.values[i].mat_count = 0; - ctx.values[i].tile = NULL; - } - - // allocate more stuff now that we've run stats on the IR - ctx.emit.label_count = cfg.block_count; - ctx.emit.labels = tb_arena_alloc(arena, cfg.block_count * sizeof(uint32_t)); - memset(ctx.emit.labels, 0, cfg.block_count * sizeof(uint32_t)); - - int bb_count = 0; - MachineBB* restrict machine_bbs = tb_arena_alloc(arena, cfg.block_count * sizeof(MachineBB)); - TB_Node** bbs = ws->items; - - size_t cap = ((cfg.block_count * 4) / 3); - ctx.node_to_bb.exp = 64 - __builtin_clzll((cap < 4 ? 4 : cap) - 1); - ctx.node_to_bb.entries = tb_arena_alloc(arena, (1u << ctx.node_to_bb.exp) * sizeof(NodeToBB)); - memset(ctx.node_to_bb.entries, 0, (1u << ctx.node_to_bb.exp) * sizeof(NodeToBB)); - - CUIK_TIMED_BLOCK("isel") { - assert(dyn_array_length(ws->items) == cfg.block_count); - - // define all PHIs early and sort BB order - int stop_bb = -1; - FOREACH_N(i, 0, cfg.block_count) { - TB_Node* end = nl_map_get_checked(cfg.node_to_block, bbs[i]).end; - if (end->type == TB_RETURN) { - stop_bb = i; - } else { - machine_bbs[bb_count++] = (MachineBB){ i }; - } - } - - // enter END block at the... end - if (stop_bb >= 0) { - machine_bbs[bb_count++] = (MachineBB){ stop_bb }; - } - - // build blocks in reverse - DynArray(PhiVal) phi_vals = NULL; - FOREACH_REVERSE_N(i, 0, bb_count) { - int bbid = machine_bbs[i].id; - TB_Node* bb_start = bbs[bbid]; - TB_BasicBlock* bb = p->scheduled[bb_start->gvn]; - - node_to_bb_put(&ctx, bb_start, &machine_bbs[i]); - size_t base = dyn_array_length(ws->items); - - // phase 1: logical schedule - CUIK_TIMED_BLOCK("phase 1") { - dyn_array_clear(phi_vals); - ctx.sched(p, &cfg, ws, &phi_vals, bb, bb->end); - } - - // phase 2: reverse walk to generate tiles (greedily) - CUIK_TIMED_BLOCK("phase 2") { - TB_OPTDEBUG(CODEGEN)(printf("BB %d\n", bbid)); - - // force materialization of phi ins - FOREACH_N(i, 0, dyn_array_length(phi_vals)) { - PhiVal* v = &phi_vals[i]; - get_tile(&ctx, v->n); - } - - Tile* top = NULL; - Tile* bot = NULL; - TB_NodeLocation* last_loc = NULL; - FOREACH_REVERSE_N(i, cfg.block_count, dyn_array_length(ws->items)) { - TB_Node* n = ws->items[i]; - if (n->type == TB_PHI) { - continue; - } else if (n->type != TB_MULPAIR && (n->dt.type == TB_TUPLE || n->dt.type == TB_CONTROL || n->dt.type == TB_MEMORY)) { - // these are always run because they're cool effect nodes - TB_NodeLocation* v; - if (v = nl_table_get(&f->locations, n), v) { - if (last_loc) { - TB_OPTDEBUG(CODEGEN)(printf(" LOCATION\n")); - - Tile* loc = TB_ARENA_ALLOC(arena, Tile); - *loc = (Tile){ .tag = TILE_LOCATION, .loc = last_loc }; - loc->next = top; - - // attach to list - if (top) top->prev = loc; - if (!bot) bot = loc; - top = loc; - } - last_loc = v; - } - } else { - if (ctx.values[n->gvn].tile == NULL) { - TB_OPTDEBUG(CODEGEN)(printf(" FOLDED "), print_node_sexpr(n, 0), printf("\n")); - continue; - } - } - - TB_OPTDEBUG(CODEGEN)(printf(" TILE "), print_node_sexpr(n, 0), printf("\n")); - - Tile* tile = get_tile(&ctx, n); - tile->next = top; - - // attach to list - if (top) top->prev = tile; - if (!bot) bot = tile; - top = tile; - - FOREACH_N(j, 0, tile->out_count) { - tile->outs[0]->dt = TB_TYPE_VOID; - } - - isel_node(&ctx, tile, n); - if (tile->out_count > 0) { - FOREACH_N(j, 0, tile->out_count) { - LiveInterval* out = tile->outs[j]; - assert(out->dt.raw && "we should've defined this by now"); - TB_OPTDEBUG(CODEGEN)(printf(" v%d ", out->id), tb__print_regmask(out->mask), printf("\n")); - } - } else { - TB_OPTDEBUG(CODEGEN)(printf(" no def\n")); - } - - FOREACH_N(j, 0, tile->in_count) { - TB_OPTDEBUG(CODEGEN)(printf(" IN[%zu] = ", j), tb__print_regmask(tile->ins[j].mask), printf("\n")); - } - } - - if (last_loc) { - TB_OPTDEBUG(CODEGEN)(printf(" LOCATION\n")); - - Tile* loc = TB_ARENA_ALLOC(arena, Tile); - *loc = (Tile){ .tag = TILE_LOCATION, .loc = last_loc }; - loc->next = top; - - // attach to list - if (top) top->prev = loc; - if (!bot) bot = loc; - top = loc; - } - - // if the endpoint is a not a terminator, we've hit some implicit GOTO edge - TB_Node* end = bb->end; - if (!cfg_is_terminator(end)) { - TB_OPTDEBUG(CODEGEN)(printf(" TERMINATOR %u: ", end->gvn), print_node_sexpr(end, 0), printf("\n")); - - // writeback phis - FOREACH_N(i, 0, dyn_array_length(phi_vals)) { - PhiVal* v = &phi_vals[i]; - - Tile* phi_tile = get_tile(&ctx, v->phi); - isel_node(&ctx, phi_tile, v->phi); - - // PHIs are weird because they have multiple tiles with the same destination. - // post phi elimination we don't have "SSA" really. - // construct live interval - FOREACH_N(i, 0, phi_tile->out_count) { - LiveInterval* phi_it = phi_tile->outs[i]; - phi_it->tile = phi_tile; - phi_it->dt = v->phi->dt; - - LiveInterval* src = get_interval(&ctx, v->n, i); - src->dt = v->phi->dt; - - TB_OPTDEBUG(CODEGEN)(printf(" PHI %u: ", v->phi->gvn), print_node_sexpr(v->phi, 0), printf("\n")); - TB_OPTDEBUG(CODEGEN)(printf(" v%d ", phi_it->id), tb__print_regmask(phi_it->mask), printf("\n")); - - Tile* move = tb_arena_alloc(arena, sizeof(Tile) + sizeof(LiveInterval*)); - *move = (Tile){ .prev = bot, .tag = TILE_SPILL_MOVE }; - move->spill_dt = v->phi->dt; - move->n = v->phi; - move->ins = tb_arena_alloc(tmp_arena, sizeof(Tile*)); - move->in_count = 1; - move->ins[0].src = src; - move->ins[0].mask = phi_it->mask; - move->out_count = 1; - move->outs[0] = phi_it; - bot->next = move; - bot = move; - } - } - - Tile* tile = TB_ARENA_ALLOC(arena, Tile); - TB_Node* succ_n = cfg_next_control(end); - *tile = (Tile){ .prev = bot, .tag = TILE_GOTO, .succ = succ_n }; - bot->next = tile; - bot = tile; - } - dyn_array_set_length(ws->items, base); - - machine_bbs[i].start = top; - machine_bbs[i].end = bot; - machine_bbs[i].n = bb_start; - machine_bbs[i].end_n = end; - } - } - dyn_array_destroy(phi_vals); - log_debug("%s: tmp_arena=%.1f KiB (postISel)", f->super.name, tb_arena_current_size(arena) / 1024.0f); - } - - CUIK_TIMED_BLOCK("create physical intervals") { - FOREACH_N(i, 0, ctx.num_classes) { - LiveInterval* intervals = tb_arena_alloc(arena, ctx.num_regs[i] * sizeof(LiveInterval)); - FOREACH_N(j, 0, ctx.num_regs[i]) { - intervals[j] = (LiveInterval){ - .id = ctx.interval_count++, - .assigned = -1, .hint = NULL, .reg = j, - .mask = { i, 0, 1u << j }, - .class = i, - .range_cap = 4, - .range_count = 1, - }; - intervals[j].ranges = tb_arena_alloc(arena, 4 * sizeof(LiveRange)); - intervals[j].ranges[0] = (LiveRange){ INT_MAX, INT_MAX }; - } - ctx.fixed[i] = intervals; - ctx.num_fixed += ctx.num_regs[i]; - } - } - - CUIK_TIMED_BLOCK("liveness") { - int interval_count = ctx.interval_count; - ctx.id2interval = tb_arena_alloc(arena, interval_count * sizeof(Tile*)); - - // local liveness (along with placing tiles on a timeline) - FOREACH_N(i, 0, bb_count) { - MachineBB* mbb = &machine_bbs[i]; - mbb->live_in = set_create_in_arena(arena, interval_count); - mbb->live_out = set_create_in_arena(arena, interval_count); - } - - // we don't need to keep the GEN and KILL sets, this doesn't save us - // much memory but it would potentially mean not using new cachelines - // in a few of the later stages. - TB_ArenaSavepoint sp = tb_arena_save(arena); - CUIK_TIMED_BLOCK("local") { - FOREACH_N(i, 0, bb_count) { - MachineBB* mbb = &machine_bbs[i]; - int bbid = mbb->id; - - mbb->gen = set_create_in_arena(arena, interval_count); - mbb->kill = set_create_in_arena(arena, interval_count); - - Set* gen = &mbb->gen; - Set* kill = &mbb->kill; - for (Tile* t = mbb->start; t; t = t->next) { - FOREACH_N(j, 0, t->in_count) { - LiveInterval* in_def = t->ins[j].src; - if (in_def && !set_get(kill, in_def->id)) { - set_put(gen, in_def->id); - } - } - - FOREACH_N(j, 0, t->out_count) { - LiveInterval* interval = t->outs[j]; - set_put(kill, interval->id); - ctx.id2interval[interval->id] = interval; - } - } - } - } - - // generate global live sets - CUIK_TIMED_BLOCK("global") { - size_t base = dyn_array_length(ws->items); - - // all BB go into the worklist - FOREACH_REVERSE_N(i, 0, bb_count) { - // in(bb) = use(bb) - set_copy(&machine_bbs[i].live_in, &machine_bbs[i].gen); - - TB_Node* n = bbs[machine_bbs[i].id]; - dyn_array_put(ws->items, n); - } - - Set visited = set_create_in_arena(arena, bb_count); - while (dyn_array_length(ws->items) > base) CUIK_TIMED_BLOCK("iter") - { - TB_Node* bb = dyn_array_pop(ws->items); - MachineBB* mbb = node_to_bb(&ctx, bb); - set_remove(&visited, mbb - machine_bbs); - - Set* live_out = &mbb->live_out; - set_clear(live_out); - - // walk all successors - TB_Node* end = mbb->end_n; - if (end->type == TB_BRANCH) { - FOR_USERS(u, end) { - if (u->n->type == TB_PROJ) { - // union with successor's lives - TB_Node* succ = cfg_next_bb_after_cproj(u->n); - set_union(live_out, &node_to_bb(&ctx, succ)->live_in); - } - } - } else if (!cfg_is_endpoint(end)) { - // union with successor's lives - TB_Node* succ = cfg_next_control(end); - if (succ) set_union(live_out, &node_to_bb(&ctx, succ)->live_in); - } - - Set* restrict live_in = &mbb->live_in; - Set* restrict kill = &mbb->kill; - Set* restrict gen = &mbb->gen; - - // live_in = (live_out - live_kill) U live_gen - bool changes = false; - FOREACH_N(i, 0, (interval_count + 63) / 64) { - uint64_t new_in = (live_out->data[i] & ~kill->data[i]) | gen->data[i]; - - changes |= (live_in->data[i] != new_in); - live_in->data[i] = new_in; - } - - // if we have changes, mark the predeccesors - if (changes && !(bb->type == TB_PROJ && bb->inputs[0]->type == TB_ROOT)) { - FOREACH_N(i, 0, bb->input_count) { - TB_Node* pred = cfg_get_pred(&cfg, bb, i); - if (pred->input_count > 0) { - MachineBB* pred_mbb = node_to_bb(&ctx, pred); - if (!set_get(&visited, pred_mbb - machine_bbs)) { - set_put(&visited, pred_mbb - machine_bbs); - dyn_array_put(ws->items, pred); - } - } - } - } - } - dyn_array_set_length(ws->items, base); - } - - #if TB_OPTDEBUG_DATAFLOW - // log live ins and outs - FOREACH_N(i, 0, bb_count) { - MachineBB* mbb = &machine_bbs[i]; - - printf("BB%d:\n live-ins:", mbb->id); - FOREACH_N(j, 0, interval_count) if (set_get(&mbb->live_in, j)) { - printf(" v%zu", j); - } - printf("\n live-outs:"); - FOREACH_N(j, 0, interval_count) if (set_get(&mbb->live_out, j)) { - printf(" v%zu", j); - } - printf("\n gen:"); - FOREACH_N(j, 0, interval_count) if (set_get(&mbb->gen, j)) { - printf(" v%zu", j); - } - printf("\n kill:"); - FOREACH_N(j, 0, interval_count) if (set_get(&mbb->kill, j)) { - printf(" v%zu", j); - } - printf("\n"); - } - #endif - - log_debug("%s: tmp_arena=%.1f KiB (dataflow)", f->super.name, tb_arena_current_size(arena) / 1024.0f); - tb_arena_restore(arena, sp); - } - - CUIK_TIMED_BLOCK("regalloc") { - log_debug("%s: tmp_arena=%.1f KiB (preRA)", f->super.name, tb_arena_current_size(arena) / 1024.0f); - - ctx.bb_count = bb_count; - ctx.machine_bbs = machine_bbs; - ctx.regalloc(&ctx, arena); - - log_debug("%s: tmp_arena=%.1f KiB (postRA)", f->super.name, tb_arena_current_size(arena) / 1024.0f); - } - - CUIK_TIMED_BLOCK("emit") { - TB_CGEmitter* e = &ctx.emit; - pre_emit(&ctx, e, f->root_node); - - FOREACH_N(i, 0, bb_count) { - int bbid = machine_bbs[i].id; - Tile* t = machine_bbs[i].start; - - if (i + 1 < bb_count) { - ctx.fallthrough = machine_bbs[i + 1].id; - } else { - ctx.fallthrough = INT_MAX; - } - ctx.current_emit_bb = &machine_bbs[i]; - ctx.current_emit_bb_pos = GET_CODE_POS(e); - - // line info is BB-local - TB_NodeLocation* last_loc = NULL; - - // mark label - on_basic_block(&ctx, e, bbid); - while (t) { - if (t->tag == TILE_LOCATION) { - TB_Location l = { - .file = t->loc->file, - .line = t->loc->line, - .column = t->loc->column, - .pos = GET_CODE_POS(e) - }; - - size_t top = dyn_array_length(ctx.locations); - if (top == 0 || (ctx.locations[top - 1].pos != l.pos && !is_same_location(&l, &ctx.locations[top - 1]))) { - dyn_array_put(ctx.locations, l); - } - } else { - emit_tile(&ctx, e, t); - } - t = t->next; - } - } - - post_emit(&ctx, e); - - // Fill jump table entries - CUIK_TIMED_BLOCK("jump tables") { - dyn_array_for(i, ctx.jump_table_patches) { - uint32_t target = ctx.emit.labels[ctx.jump_table_patches[i].target]; - assert((target & 0x80000000) && "target label wasn't resolved... what?"); - *ctx.jump_table_patches[i].pos = target & ~0x80000000; - } - } - } - - if (ctx.locations) { - ctx.locations[0].pos = 0; - } - - // trim code arena (it fits in a single chunk so just arena free the top) - code_arena->watermark = (char*) &ctx.emit.data[ctx.emit.count]; - tb_arena_realign(code_arena); - - // TODO(NeGate): move the assembly output to code arena - if (emit_asm) CUIK_TIMED_BLOCK("dissassembly") { - dyn_array_for(i, ctx.debug_stack_slots) { - TB_StackSlot* s = &ctx.debug_stack_slots[i]; - EMITA(&ctx.emit, "// %s = [rsp + %d]\n", s->name, s->storage.offset); - } - EMITA(&ctx.emit, "%s:\n", f->super.name); - - Disasm d = { - func_out->first_patch, - ctx.locations, - &ctx.locations[dyn_array_length(ctx.locations)], - ctx.emit.comment_head, - }; - - if (ctx.prologue_length) { - disassemble(&ctx.emit, &d, -1, 0, ctx.prologue_length); - } - - FOREACH_N(i, 0, bb_count) { - int bbid = machine_bbs[i].id; - TB_Node* bb = bbs[bbid]; - - uint32_t start = ctx.emit.labels[bbid] & ~0x80000000; - uint32_t end = ctx.emit.count; - if (i + 1 < bb_count) { - end = ctx.emit.labels[machine_bbs[i + 1].id] & ~0x80000000; - } - - disassemble(&ctx.emit, &d, bbid, start, end); - } - } - - // cleanup memory - nl_map_free(ctx.stack_slots); - tb_free_cfg(&cfg); - - log_debug("%s: code_arena=%.1f KiB", f->super.name, tb_arena_current_size(code_arena) / 1024.0f); - tb_arena_restore(arena, sp); - p->scheduled = NULL; - - // we're done, clean up - func_out->asm_out = ctx.emit.head_asm; - func_out->code = ctx.emit.data; - func_out->code_size = ctx.emit.count; - func_out->locations = ctx.locations; - func_out->stack_slots = ctx.debug_stack_slots; - func_out->stack_usage = ctx.stack_usage; - func_out->prologue_length = ctx.prologue_length; - func_out->epilogue_length = ctx.epilogue_length; - func_out->nop_pads = ctx.nop_pads; -} - -static void get_data_type_size(TB_DataType dt, size_t* out_size, size_t* out_align) { - switch (dt.type) { - case TB_INT: { - // above 64bits we really dont care that much about natural alignment - bool is_big_int = dt.data > 64; - - // round up bits to a byte - int bits = is_big_int ? ((dt.data + 7) / 8) : tb_next_pow2(dt.data - 1); - - *out_size = ((bits+7) / 8); - *out_align = is_big_int ? 8 : ((dt.data + 7) / 8); - break; - } - case TB_FLOAT: { - int s = 0; - if (dt.data == TB_FLT_32) s = 4; - else if (dt.data == TB_FLT_64) s = 8; - else tb_unreachable(); - - *out_size = s; - *out_align = s; - break; - } - case TB_PTR: { - *out_size = 8; - *out_align = 8; - break; - } - default: tb_unreachable(); - } -} diff --git a/vendor/tb/src/debug/cv.c b/vendor/tb/src/debug/cv.c deleted file mode 100644 index c1bf6f28..00000000 --- a/vendor/tb/src/debug/cv.c +++ /dev/null @@ -1,553 +0,0 @@ -#include "../tb_internal.h" -#include "cv.h" - -#include "cv_type_builder.c" - -#include - -#if defined(_WIN32) && !defined(_POSIX_C_SOURCE) -#define fileno _fileno -#define fstat _fstat -#define stat _stat -#define strdup _strdup -#endif - -// constant sized "hash map" which is used to -// deduplicate types in the codeview -#define MAX_TYPE_ENTRY_LOOKUP_SIZE 1024 - -static void md5sum_file(uint8_t out_bytes[16], const char* filepath) { - FILE* file = fopen(filepath, "rb"); - if (!file) { - printf("Could not read file: %s\n", filepath); - abort(); - } - - int descriptor = fileno(file); - - struct stat file_stats; - if (fstat(descriptor, &file_stats) == -1) { - fclose(file); - abort(); - } - - size_t len = file_stats.st_size; - unsigned char* data = tb_platform_heap_alloc(len + 1); - - fseek(file, 0, SEEK_SET); - fread(data, 1, len, file); - - tb__md5sum(out_bytes, data, len); - - fclose(file); - tb_platform_heap_free(data); -} - -static uint16_t get_codeview_type(TB_DataType dt) { - switch (dt.type) { - case TB_INT: { - if (dt.data <= 0) return 0x0003; // T_VOID - if (dt.data <= 1) return 0x0030; // T_BOOL08 - if (dt.data <= 8) return 0x0020; // T_UCHAR - if (dt.data <= 16) return 0x0073; // T_UINT2 - if (dt.data <= 32) return 0x0075; // T_UINT4 - if (dt.data <= 64) return 0x0023; // T_UQUAD - return 0x0023; // T_64PUCHAR - } - case TB_FLOAT: { - if (dt.data == TB_FLT_32) return 0x0040; // T_REAL32 - if (dt.data == TB_FLT_64) return 0x0041; // T_REAL64 - - assert(0 && "Unknown float type"); - } - case TB_PTR: { - return 0x0023; // T_64PUCHAR - } - default: assert(0 && "TODO: missing type in CodeView output"); - } - - return 0x0003; // T_VOID -} - -static uint16_t convert_to_codeview_type(CV_Builder* builder, TB_DebugType* type) { - if (type->type_id != 0) { - return type->type_id; - } - - switch (type->tag) { - case TB_DEBUG_TYPE_VOID: return (type->type_id = T_VOID); - case TB_DEBUG_TYPE_BOOL: return (type->type_id = T_BOOL08); // T_BOOL08 - - case TB_DEBUG_TYPE_INT: - case TB_DEBUG_TYPE_UINT: { - bool is_signed = (type->tag == TB_DEBUG_TYPE_INT); - - if (type->int_bits <= 8) return is_signed ? T_CHAR : T_UCHAR; - if (type->int_bits <= 16) return is_signed ? T_INT2 : T_UINT2; - if (type->int_bits <= 32) return is_signed ? T_INT4 : T_UINT4; - if (type->int_bits <= 64) return is_signed ? T_INT8 : T_UINT8; - assert(0 && "Unsupported int type"); - } - - case TB_DEBUG_TYPE_FLOAT: { - switch (type->float_fmt) { - case TB_FLT_32: return (type->type_id = T_REAL32); - case TB_FLT_64: return (type->type_id = T_REAL64); - default: assert(0 && "Unknown float type"); - } - } - - case TB_DEBUG_TYPE_ARRAY: - return (type->type_id = tb_codeview_builder_add_array(builder, convert_to_codeview_type(builder, type->array.base), debug_type_size(TB_ABI_WIN64, type->array.base) * type->array.count)); - - case TB_DEBUG_TYPE_POINTER: - return (type->type_id = tb_codeview_builder_add_pointer(builder, convert_to_codeview_type(builder, type->ptr_to))); - - case TB_DEBUG_TYPE_FUNCTION: - return (type->type_id = tb_codeview_builder_add_pointer(builder, 0x0003)); - - case TB_DEBUG_TYPE_ALIAS: - return (type->type_id = tb_codeview_builder_add_alias(builder, convert_to_codeview_type(builder, type->alias.type), type->alias.name)); - - case TB_DEBUG_TYPE_STRUCT: - case TB_DEBUG_TYPE_UNION: { - if (type->type_id_fwd) { - return type->type_id_fwd; - } - - // generate forward declaration - // TODO(NeGate): we might wanna avoid generating a forward declaration here if we never use it - CV_RecordType rec_type = type->tag == TB_DEBUG_TYPE_STRUCT ? LF_STRUCTURE : LF_UNION; - type->type_id_fwd = tb_codeview_builder_add_incomplete_record(builder, rec_type, type->record.tag); - - if (type->record.count == 0) { - // it's incomplete so it doesn't matter - return type->type_id_fwd; - } - - TB_ArenaSavepoint sp = tb_arena_save(tmp_arena); - CV_Field* list = tb_arena_alloc(tmp_arena, type->record.count * sizeof(CV_Field)); - FOREACH_N(i, 0, type->record.count) { - const TB_DebugType* f = type->record.members[i]; - assert(f->tag == TB_DEBUG_TYPE_FIELD); - - list[i].type = convert_to_codeview_type(builder, f->field.type); - list[i].name = f->field.name; - list[i].offset = f->field.offset; - } - - CV_TypeIndex field_list = tb_codeview_builder_add_field_list(builder, type->record.count, list); - tb_arena_restore(tmp_arena, sp); - - return (type->type_id = tb_codeview_builder_add_record(builder, rec_type, type->record.count, field_list, type->record.size, type->record.tag)); - } - - default: - assert(0 && "TODO: missing type in CodeView output"); - return 0x0003; - } -} - -static TB_Slice gimme_cstr_as_slice(TB_Arena* arena, const char* str) { - TB_Slice s = { .length = strlen(str) }; - s.data = memcpy(tb_arena_alloc(arena, s.length), str, s.length); - return s; -} - -static void add_reloc(TB_ObjectSection* section, const TB_ObjectReloc* reloc, size_t cap) { - assert(section->relocation_count < cap); - section->relocations[section->relocation_count++] = *reloc; -} - -static bool codeview_supported_target(TB_Module* m) { - return true; -} - -static int codeview_number_of_debug_sections(TB_Module* m) { - return 2; -} - -// Based on this, it's the only nice CodeView source out there: -// https://github.com/netwide-assembler/nasm/blob/master/output/codeview.c -static TB_SectionGroup codeview_generate_debug_info(TB_Module* m, TB_Arena* arena) { - TB_ObjectSection* sections = tb_arena_alloc(arena, 2 * sizeof(TB_ObjectSection)); - sections[0] = (TB_ObjectSection){ gimme_cstr_as_slice(arena, ".debug$S") }; - sections[1] = (TB_ObjectSection){ gimme_cstr_as_slice(arena, ".debug$T") }; - - size_t global_count = m->symbol_count[TB_SYMBOL_GLOBAL]; - - // debug$S does quite a few relocations :P, namely saying that - // certain things point to specific areas of .text section - size_t reloc_cap = (2 * global_count) + (4 * m->compiled_function_count); - sections[0].relocations = tb_arena_alloc(arena, reloc_cap * sizeof(TB_ObjectReloc)); - - TB_ArenaSavepoint sp = tb_arena_save(arena); - - // Write type table - size_t file_count = nl_map__get_header(m->files)->count; - uint32_t* file_table_offset = tb_arena_alloc(arena, file_count * sizeof(uint32_t)); - - TB_Emitter debugs_out = { 0 }; - - CV_TypeEntry* lookup_table = tb_arena_alloc(arena, 1024 * sizeof(CV_TypeEntry)); - memset(lookup_table, 0, 1024 * sizeof(CV_TypeEntry)); - CV_Builder builder = tb_codeview_builder_create(1024, lookup_table); - - // Write symbol info table - { - static const char creator_str[] = "Bitch"; - uint32_t creator_length = 2 + 4 + 2 + (3 * 2) + (3 * 2) + sizeof(creator_str) + 2; - - static const char dummy_path[] = "fallback.o"; - tb_out4b(&debugs_out, 0x00000004); - - // File nametable - CUIK_TIMED_BLOCK("write file nametable") { - tb_out4b(&debugs_out, 0x000000F3); - - size_t field_length_patch = debugs_out.count; - tb_out4b(&debugs_out, 0); - tb_out1b(&debugs_out, 0); - - // skip the NULL file entry - size_t pos = 1, counter = 0; - file_table_offset[0] = 0; - nl_map_for_str(i, m->files) { - TB_SourceFile* f = m->files[i].v; - f->id = counter++; - - tb_out_reserve(&debugs_out, f->len + 1); - tb_outs_UNSAFE(&debugs_out, f->len + 1, (const uint8_t*) f->path); - - file_table_offset[f->id] = pos; - pos += f->len + 1; - } - - tb_patch4b(&debugs_out, field_length_patch, (debugs_out.count - field_length_patch) - 4); - align_up_emitter(&debugs_out, 4); - } - - // Source file table - // we practically transmute the file_table_offset from meaning file string - // table entries into source file entries. - CUIK_TIMED_BLOCK("write file hashes") { - tb_out4b(&debugs_out, 0x000000F4); - - size_t field_length_patch = debugs_out.count; - tb_out4b(&debugs_out, 0); - - size_t pos = 0; - nl_map_for_str(i, m->files) { - TB_SourceFile* f = m->files[i].v; - - uint8_t sum[16]; - md5sum_file(sum, (const char*) f->path); - - tb_out4b(&debugs_out, file_table_offset[f->id]); - tb_out2b(&debugs_out, 0x0110); - tb_outs(&debugs_out, MD5_HASHBYTES, sum); - tb_out2b(&debugs_out, 0); - - file_table_offset[f->id] = pos; - pos += 4 + 2 + MD5_HASHBYTES + 2; - } - - tb_patch4b(&debugs_out, field_length_patch, (debugs_out.count - field_length_patch) - 4); - align_up_emitter(&debugs_out, 4); - } - - // Line info table - CUIK_TIMED_BLOCK("write line info") { - dyn_array_for(i, m->sections) { - DynArray(TB_FunctionOutput*) funcs = m->sections[i].funcs; - dyn_array_for(j, funcs) { - TB_FunctionOutput* out_f = funcs[j]; - - // Layout crap - DynArray(TB_Location) lines = out_f->locations; - - tb_out4b(&debugs_out, 0x000000F2); - size_t field_length_patch = debugs_out.count; - tb_out4b(&debugs_out, 0); - - // Source mapping header - size_t func_id = out_f->parent->super.symbol_id; - { - size_t patch_pos = debugs_out.count; - add_reloc(§ions[0], &(TB_ObjectReloc){ TB_OBJECT_RELOC_SECREL, func_id, patch_pos }, reloc_cap); - add_reloc(§ions[0], &(TB_ObjectReloc){ TB_OBJECT_RELOC_SECTION, func_id, patch_pos + 4 }, reloc_cap); - } - - tb_out4b(&debugs_out, 0); // SECREL | .text - tb_out4b(&debugs_out, 0); // SECTION | .text - tb_out4b(&debugs_out, out_f->code_size - out_f->nop_pads); - - // when we make new file line regions - // we backpatch the line count for the - // region we just finished - uint32_t backpatch = 0; - int last_line = 0; - TB_SourceFile* last_file = 0; - uint32_t current_line_count = 0; - - dyn_array_for(line_id, lines) { - TB_Location loc = lines[line_id]; - - if (last_file != loc.file) { - if (backpatch) { - tb_patch4b(&debugs_out, backpatch, current_line_count); - tb_patch4b(&debugs_out, backpatch + 4, 12 + (current_line_count * 8)); - } - last_file = loc.file; - - // File entry - tb_out4b(&debugs_out, file_table_offset[loc.file->id]); - backpatch = debugs_out.count; - tb_out4b(&debugs_out, 0); - tb_out4b(&debugs_out, 0); - - // printf(" FILE %s\n", loc.file->path); - current_line_count = 0; - last_line = 0; - } - - if (last_line != loc.line) { - last_line = loc.line; - // printf(" * LINE %d : %x\n", loc.line, loc.pos); - - tb_out4b(&debugs_out, loc.pos); - tb_out4b(&debugs_out, loc.line); - current_line_count++; - } - } - - // finalize the patch work - if (backpatch) { - tb_patch4b(&debugs_out, backpatch, current_line_count); - tb_patch4b(&debugs_out, backpatch + 4, 12 + (current_line_count * 8)); - } - - tb_patch4b(&debugs_out, field_length_patch, (debugs_out.count - field_length_patch) - 4); - // printf("\n"); - } - } - - align_up_emitter(&debugs_out, 4); - } - - // Symbol table - tb_out4b(&debugs_out, 0x000000F1); - - size_t field_length_patch = debugs_out.count; - tb_out4b(&debugs_out, 0); - - // Symbol info object - { - uint32_t obj_length = 2 + 4 + sizeof(dummy_path); - tb_out2b(&debugs_out, obj_length); - tb_out2b(&debugs_out, 0x1101); - tb_out4b(&debugs_out, 0); - - tb_out_reserve(&debugs_out, sizeof(dummy_path)); - tb_outs_UNSAFE(&debugs_out, sizeof(dummy_path), (const uint8_t*) dummy_path); - } - - // Symbol info properties - { - tb_out2b(&debugs_out, creator_length); - tb_out2b(&debugs_out, 0x1116); - tb_out4b(&debugs_out, 0); - - tb_out2b(&debugs_out, 0x00D0); // machine - tb_out2b(&debugs_out, 0); // verFEMajor - tb_out2b(&debugs_out, 0); // verFEMinor - tb_out2b(&debugs_out, 0); // verFEBuild - - tb_out2b(&debugs_out, TB_VERSION_MAJOR); // verMajor - tb_out2b(&debugs_out, TB_VERSION_MINOR); // verMinor - tb_out2b(&debugs_out, TB_VERSION_PATCH); // verBuild - - tb_out_reserve(&debugs_out, sizeof(creator_str)); - tb_outs_UNSAFE(&debugs_out, sizeof(creator_str), (const uint8_t*)creator_str); - - tb_out2b(&debugs_out, 0); - } - - CUIK_TIMED_BLOCK("cv: globals") { - dyn_array_for(i, m->sections) { - DynArray(TB_Global*) globals = m->sections[i].globals; - dyn_array_for(j, globals) { - TB_Global* g = globals[j]; - if (g->super.name == NULL) continue; - - size_t name_len = strlen(g->super.name) + 1; - CV_TypeIndex type = g->dbg_type ? convert_to_codeview_type(&builder, g->dbg_type) : T_VOID; - - // printf("%-20s : %d\n", name, type); - size_t baseline = debugs_out.count; - tb_out2b(&debugs_out, 0); - tb_out2b(&debugs_out, S_GDATA32); - tb_out4b(&debugs_out, type); // type index - { - size_t id = g->super.symbol_id; - size_t patch_pos = debugs_out.count; - add_reloc(§ions[0], &(TB_ObjectReloc){ TB_OBJECT_RELOC_SECREL, id, patch_pos }, reloc_cap); - add_reloc(§ions[0], &(TB_ObjectReloc){ TB_OBJECT_RELOC_SECTION, id, patch_pos + 4 }, reloc_cap); - } - tb_out4b(&debugs_out, 0); // offset - tb_out2b(&debugs_out, 0); // section - - tb_out_reserve(&debugs_out, name_len); - tb_outs_UNSAFE(&debugs_out, name_len, (const uint8_t*) g->super.name); - - // patch field length - tb_patch2b(&debugs_out, baseline, (debugs_out.count - baseline) - 2); - } - } - } - - // Symbols - CUIK_TIMED_BLOCK("cv: funcs") { - dyn_array_for(i, m->sections) { - DynArray(TB_FunctionOutput*) funcs = m->sections[i].funcs; - dyn_array_for(j, funcs) { - TB_FunctionOutput* out_f = funcs[j]; - TB_Function* f = out_f->parent; - - const char* name = f->super.name; - size_t name_len = strlen(f->super.name) + 1; - - size_t baseline = debugs_out.count; - tb_out2b(&debugs_out, 0); - tb_out2b(&debugs_out, S_GPROC32_ID); - - tb_out4b(&debugs_out, 0); // pointer to the parent - tb_out4b(&debugs_out, 0); // pointer to this blocks end (left as zero?) - tb_out4b(&debugs_out, 0); // pointer to the next symbol (left as zero?) - - CV_TypeIndex function_type; - { - const TB_FunctionPrototype* proto = f->prototype; - TB_ArenaSavepoint sp = tb_arena_save(arena); - - // Create argument list - CV_TypeIndex* params = tb_arena_alloc(arena, proto->param_count * sizeof(CV_TypeIndex)); - FOREACH_N(i, 0, proto->param_count) { - TB_DebugType* t = proto->params[i].debug_type; - params[i] = t ? convert_to_codeview_type(&builder, t) : T_VOID; - } - - CV_TypeIndex arg_list = tb_codeview_builder_add_arg_list(&builder, proto->param_count, params, proto->has_varargs); - tb_arena_restore(arena, sp); - - // Create return type... if it's multiple returns use a struct - CV_TypeIndex return_type = T_VOID; - if (proto->return_count == 1) { - const TB_PrototypeParam* ret = &TB_PROTOTYPE_RETURNS(proto)[0]; - return_type = ret->debug_type ? convert_to_codeview_type(&builder, ret->debug_type) : get_codeview_type(ret->dt); - } - - // Create the procedure type - CV_TypeIndex proc = tb_codeview_builder_add_procedure(&builder, return_type, arg_list, proto->param_count); - - // Create the function ID type... which is somehow different from the procedure... - // it's basically referring to the procedure but it has a name - function_type = tb_codeview_builder_add_function_id(&builder, proc, name); - } - - tb_out4b(&debugs_out, out_f->code_size - out_f->nop_pads); // procedure length - tb_out4b(&debugs_out, 0); // debug start offset - tb_out4b(&debugs_out, 0); // debug end offset - tb_out4b(&debugs_out, function_type); // type index - - // we save this location because there's two relocations - // we'll put there: - // type target size - // SECREL .text 4 bytes - // SECTION .text 2 bytes - { - size_t func_id = f->super.symbol_id; - size_t patch_pos = debugs_out.count; - add_reloc(§ions[0], &(TB_ObjectReloc){ TB_OBJECT_RELOC_SECREL, func_id, patch_pos }, reloc_cap); - add_reloc(§ions[0], &(TB_ObjectReloc){ TB_OBJECT_RELOC_SECTION, func_id, patch_pos + 4 }, reloc_cap); - } - tb_out4b(&debugs_out, 0); // offset - tb_out2b(&debugs_out, 0); // segment - - // the 1 means we have a frame pointer present - tb_out1b(&debugs_out, 0); // flags - - tb_out_reserve(&debugs_out, name_len); - tb_outs_UNSAFE(&debugs_out, name_len, (const uint8_t*)name); - - // patch field length - tb_patch2b(&debugs_out, baseline, (debugs_out.count - baseline) - 2); - - { - // frameproc - size_t frameproc_baseline = debugs_out.count; - - tb_out2b(&debugs_out, 0); - tb_out2b(&debugs_out, S_FRAMEPROC); - - size_t stack_usage = out_f->stack_usage == 8 ? 0 : out_f->stack_usage; - - tb_out4b(&debugs_out, stack_usage); // count of bytes of total frame of procedure - tb_out4b(&debugs_out, 0); // count of bytes of padding in the frame - tb_out4b(&debugs_out, 0); // offset (relative to frame poniter) to where padding starts - tb_out4b(&debugs_out, 0); // count of bytes of callee save registers - tb_out4b(&debugs_out, 0); // offset of exception handler - tb_out2b(&debugs_out, 0); // section id of exception handler - tb_out4b(&debugs_out, 0x00014000); // flags - - tb_patch2b(&debugs_out, frameproc_baseline, (debugs_out.count - frameproc_baseline) - 2); - - dyn_array_for(j, out_f->stack_slots) { - int stack_pos = out_f->stack_slots[j].storage.offset; - TB_DebugType* type = out_f->stack_slots[j].type; - - const char* var_name = out_f->stack_slots[j].name; - assert(var_name); - - size_t var_name_len = strlen(var_name); - uint32_t type_index = convert_to_codeview_type(&builder, type); - - // define S_REGREL32 - CV_RegRel32 l = { - .reclen = sizeof(CV_RegRel32) + (var_name_len + 1) - 2, - .rectyp = S_REGREL32, - .off = stack_pos, - .typind = type_index, - // AMD64_RBP is 334, AMD64_RSP is 335 - .reg = 335, - }; - tb_outs(&debugs_out, sizeof(CV_RegRel32), &l); - tb_outs(&debugs_out, var_name_len + 1, (const uint8_t*) var_name); - } - } - - // end the block - tb_out2b(&debugs_out, 2); - tb_out2b(&debugs_out, S_PROC_ID_END); - } - } - - tb_patch4b(&debugs_out, field_length_patch, (debugs_out.count - field_length_patch) - 4); - align_up_emitter(&debugs_out, 4); - } - } - tb_codeview_builder_done(&builder); - tb_arena_restore(arena, sp); - - sections[0].raw_data = (TB_Slice){ debugs_out.data, debugs_out.count }; - sections[1].raw_data = (TB_Slice){ builder.type_section.data, builder.type_section.count }; - - return (TB_SectionGroup) { 2, sections }; -} - -IDebugFormat tb__codeview_debug_format = { - "CodeView", - codeview_supported_target, - codeview_number_of_debug_sections, - codeview_generate_debug_info -}; diff --git a/vendor/tb/src/debug/cv.h b/vendor/tb/src/debug/cv.h deleted file mode 100644 index e6d8c491..00000000 --- a/vendor/tb/src/debug/cv.h +++ /dev/null @@ -1,53 +0,0 @@ -#pragma once -#include "../objects/coff.h" - -typedef uint32_t CV_TypeIndex; - -typedef struct { - // Type table - CV_TypeIndex type_entry_count; - TB_Emitter type_section; - - // Type table cache - bool needs_to_free_lookup; - size_t lookup_table_size; // must be a power of two - CV_TypeEntry* lookup_table; - - // Symbol table - // TODO -} CV_Builder; - -typedef struct { - CV_TypeIndex type; - const char* name; - TB_CharUnits offset; -} CV_Field; - -// map to their Codeview record types because it's simpler -typedef enum { - CV_CLASS = 0x1504, CV_STRUCT = 0x1505, CV_UNION = 0x1506, -} CV_RecordType; - -// if cache is NULL and cache_size is not 0, then it'll allocate and zeroed it for you -// which will also be freed on tb_codeview_builder_done -// -// cache_size must be a power of two -// if cache is non-NULL it has cache_size elements and zeroed -CV_Builder tb_codeview_builder_create(size_t cache_size, CV_TypeEntry* cache); - -// doesn't free the section memory -void tb_codeview_builder_done(CV_Builder* builder); - -// Type builder API -CV_TypeIndex tb_codeview_builder_add_array(CV_Builder* builder, CV_TypeIndex base, size_t count); -CV_TypeIndex tb_codeview_builder_add_pointer(CV_Builder* builder, CV_TypeIndex ptr_to); -CV_TypeIndex tb_codeview_builder_add_arg_list(CV_Builder* builder, size_t count, const CV_TypeIndex* args, bool has_varargs); -CV_TypeIndex tb_codeview_builder_add_procedure(CV_Builder* builder, CV_TypeIndex return_type, CV_TypeIndex arg_list, size_t param_count); - -// function ID is just a procedure with name, it's used by the symbol table -CV_TypeIndex tb_codeview_builder_add_function_id(CV_Builder* builder, CV_TypeIndex proc_type, const char* name); - -// When refering to records from other types you must use this forward declaration -CV_TypeIndex tb_codeview_builder_add_incomplete_record(CV_Builder* builder, CV_RecordType rec_type, const char* name); -CV_TypeIndex tb_codeview_builder_add_field_list(CV_Builder* builder, size_t count, const CV_Field* fields); -CV_TypeIndex tb_codeview_builder_add_record(CV_Builder* builder, CV_RecordType rec_type, size_t field_count, CV_TypeIndex field_list, TB_CharUnits size, const char* name); diff --git a/vendor/tb/src/debug/cv_type_builder.c b/vendor/tb/src/debug/cv_type_builder.c deleted file mode 100644 index 7430ce9f..00000000 --- a/vendor/tb/src/debug/cv_type_builder.c +++ /dev/null @@ -1,330 +0,0 @@ -#include "cv.h" - -static uint32_t hash_buffer(uint32_t hash, size_t n, const void* s) { - const uint8_t* p = s; - while (n--) { - hash = (hash ^ *p++) * 16777619; - } - return hash; -} - -static void align_up_type_record(TB_Emitter* e) { - // type records need to be 4 byte aligned with a decreasing number of LF_PAD0+i ending in 0 - size_t align = e->count % 4; - if (align == 0) return; - - size_t pad = 4 - align; - while (pad > 0) { - tb_out1b(e, LF_PAD0 + pad); - pad -= 1; - } -} - -static void align_up_emitter(TB_Emitter* e, size_t u) { - size_t pad = align_up(e->count, u) - e->count; - while (pad--) tb_out1b(e, 0x00); -} - -static CV_TypeIndex find_or_make_cv_type(CV_Builder* builder, size_t length, const void* k) { - #if 0 - // This "dead" code is for educational purposes since i know that there aren't many Codeview resources. - // the path that gets correctly run will merely deduplicate the results i simulate here as a size-optimization - CV_TypeIndex type_index = builder->type_entry_count++; - tb_outs(sect, length, k); - return type_index; - #else - // Hash it - const uint8_t* key = k; - uint32_t hash = hash_buffer(0, length, key); - - // Search (if there's a collision replace the old one) - assert(tb_is_power_of_two(builder->lookup_table_size)); - size_t index = hash & (builder->lookup_table_size - 1); - CV_TypeEntry lookup = builder->lookup_table[index]; - - // printf("Lookup %zu (%x hash, has match? %s)\n", index, hash, lookup.key ? "yea" : "naw"); - if (lookup.key) { - // verify it even matches - size_t lookup_size = tb_get2b(&builder->type_section, lookup.key) + 2; - - if (length == lookup_size && memcmp(key, &builder->type_section.data[lookup.key], length) == 0) { - //printf("Saved %zu bytes (%d)\n", length, lookup.value); - return lookup.value; - } - } - - CV_TypeIndex type_index = builder->type_entry_count++; - // printf("Used %zu bytes (%d)\n", length, type_index); - - builder->lookup_table[index].key = builder->type_section.count; - builder->lookup_table[index].value = type_index; - - // NOTE: MSF/PDB requires the type entries to be properly aligned to 4bytes... we - // might wanna add an option to the builder for that. - tb_outs(&builder->type_section, length, key); - return type_index; - #endif -} - -CV_Builder tb_codeview_builder_create(size_t cache_size, CV_TypeEntry* cache) { - assert(tb_is_power_of_two(cache_size)); - - // start the type table - TB_Emitter e = { 0 }; - tb_out4b(&e, 0x00000004); - - if (cache == NULL) { - cache = tb_platform_heap_alloc(cache_size * sizeof(CV_TypeEntry)); - } - - return (CV_Builder){ - .type_section = e, - - .needs_to_free_lookup = (cache == NULL), - .lookup_table_size = cache_size, - .lookup_table = cache, - - // user defined types start at 0x1000 - .type_entry_count = 0x1000, - }; -} - -void tb_codeview_builder_done(CV_Builder* builder) { - if (builder->needs_to_free_lookup) { - tb_platform_heap_free(builder); - } - - // align_up_type_record(&builder->type_section); -} - -CV_TypeIndex tb_codeview_builder_add_array(CV_Builder* builder, CV_TypeIndex base, size_t byte_count) { - #pragma pack(push,1) - typedef struct { - uint16_t len; - uint16_t leaf; // LF_ARRAY - uint32_t elemtype; // type index of element type - uint32_t idxtype; // type index of indexing type - - // uint8_t data[]; // variable length data specifying size in bytes and name - // - // This is technically what needs to be written but within this builder we hardcode - // a count and a null name - struct { - uint16_t type; - uint32_t value; - } count; - - char name[1]; // keep as zero - } Array; - #pragma pack(pop) - - assert(byte_count == (uint32_t) byte_count); - Array arr = { - .len = sizeof(Array) - 2, - .leaf = LF_ARRAY, - .elemtype = base, - .idxtype = T_INT8, - .count = { LF_LONG, byte_count }, - }; - - return find_or_make_cv_type(builder, sizeof(arr), &arr); -} - -CV_TypeIndex tb_codeview_builder_add_pointer(CV_Builder* builder, CV_TypeIndex ptr_to) { - CV_LFPointer ptr = { - .len = sizeof(CV_LFPointer) - 2, - .leaf = LF_POINTER, - .utype = ptr_to, - .attr = { - .ptrtype = 0x0c, // CV_PTR_64 - .ptrmode = 0, // CV_PTR_MODE_PTR - .size = 8, - } - }; - - return find_or_make_cv_type(builder, sizeof(ptr), &ptr); -} - -CV_TypeIndex tb_codeview_builder_add_arg_list(CV_Builder* builder, size_t count, const CV_TypeIndex* args, bool has_varargs) { - enum { SMALL_ARR_CAP = 8 + (16 * 8) }; - uint8_t tmp[SMALL_ARR_CAP]; - - // this is what the arglist is like - #pragma pack(push,1) - typedef struct { - uint16_t len; - uint16_t type; // LF_ARGLIST - uint32_t arg_count; - CV_TypeIndex args[]; - } Arglist; - #pragma pack(pop) - - // use allocated space if we couldn't fit - Arglist* data = (Arglist*) tmp; - size_t length = sizeof(Arglist) + ((count + has_varargs) * sizeof(CV_TypeIndex)); - if (length >= SMALL_ARR_CAP) data = tb_platform_heap_alloc(length); - - data->len = length - 2; - data->type = LF_ARGLIST; - data->arg_count = count; - memcpy(data->args, args, count * sizeof(CV_TypeIndex)); - - // varargs add a dummy type at the end of the list - if (has_varargs) { - data->args[count] = 0; - } - - CV_TypeIndex type = find_or_make_cv_type(builder, length, data); - if (length >= SMALL_ARR_CAP) tb_platform_heap_free(data); - - return type; -} - -CV_TypeIndex tb_codeview_builder_add_procedure(CV_Builder* builder, CV_TypeIndex return_type, CV_TypeIndex arg_list, size_t param_count) { - #pragma pack(push,1) - struct Procedure { - uint16_t len; - uint16_t leaf; // LF_PROCEDURE - uint32_t rvtype; // type index of return value - uint8_t calltype; // calling convention (CV_call_t) - uint8_t funcattr; // attributes - uint16_t parmcount; // number of parameters - uint32_t arglist; // type index of argument list - } proc = { - .len = sizeof(struct Procedure) - 2, - .leaf = LF_PROCEDURE, - .rvtype = return_type, - .parmcount = param_count, - .arglist = arg_list, - }; - #pragma pack(pop) - - return find_or_make_cv_type(builder, sizeof(proc), &proc); -} - -CV_TypeIndex tb_codeview_builder_add_function_id(CV_Builder* builder, CV_TypeIndex proc_type, const char* name) { - size_t name_size = strlen(name) + 1; - - #pragma pack(push,1) - // TODO(NeGate): support scopes - struct Function { - uint16_t len; // doesn't include itself, so sizeof(T)-2 - uint16_t leaf; // LF_FUNC_ID - uint32_t scopeId; // parent scope of the ID, 0 if global - uint32_t type; // function type - uint8_t name[]; // null-terminated string - } func = { - .len = sizeof(struct Function) + name_size - 2, - .leaf = LF_FUNC_ID, - .scopeId = 0, - .type = proc_type, - }; - #pragma pack(pop) - - CV_TypeIndex id = builder->type_entry_count++; - tb_outs(&builder->type_section, sizeof(func), &func); - tb_outs(&builder->type_section, name_size, name); - return id; -} - -CV_TypeIndex tb_codeview_builder_add_alias(CV_Builder* builder, CV_RecordType base, const char* name) { - CV_TypeIndex id = builder->type_entry_count++; - size_t name_size = strlen(name) + 1; - - CV_LFAlias a = { - .len = sizeof(CV_LFAlias) + name_size - 2, - .leaf = LF_ALIAS, - .utype = base, - }; - tb_outs(&builder->type_section, sizeof(a), &a); - tb_outs(&builder->type_section, name_size, name); - return id; -} - -CV_TypeIndex tb_codeview_builder_add_incomplete_record(CV_Builder* builder, CV_RecordType rec_type, const char* name) { - CV_TypeIndex id = builder->type_entry_count++; - size_t name_size = strlen(name) + 1; - - CV_LFStruct s = { - .len = sizeof(CV_LFStruct) + 3 + name_size - 2, - .leaf = rec_type, - - // it's a forward declaration - .property.fwdref = 1, - }; - tb_outs(&builder->type_section, sizeof(s), &s); - - // write 0 as the size - tb_out2b(&builder->type_section, LF_CHAR); - tb_out1b(&builder->type_section, 0); - - tb_outs(&builder->type_section, name_size, name); - return id; -} - -CV_TypeIndex tb_codeview_builder_add_field_list(CV_Builder* builder, size_t count, const CV_Field* fields) { - // write field list - CV_TypeIndex id = builder->type_entry_count++; - - size_t patch_pos = builder->type_section.count; - tb_out2b(&builder->type_section, 0); // length (we'll patch it later) - tb_out2b(&builder->type_section, LF_FIELDLIST); // type - - FOREACH_N(i, 0, count) { - size_t name_size = strlen(fields[i].name) + 1; - - // write member heading - CV_LFMember m = { - .leaf = LF_MEMBER, - .index = fields[i].type, - }; - tb_outs(&builder->type_section, sizeof(CV_LFMember), &m); - - // write offset - if (fields[i].offset == (int8_t) fields[i].offset) { - // write it as 3 bytes instead of 6 since it's a common case - tb_out2b(&builder->type_section, LF_CHAR); - tb_out1b(&builder->type_section, fields[i].offset); - } else { - tb_out2b(&builder->type_section, LF_LONG); - tb_out4b(&builder->type_section, fields[i].offset); - } - - // write out C string - tb_outs(&builder->type_section, name_size, fields[i].name); - align_up_type_record(&builder->type_section); - } - tb_patch2b(&builder->type_section, patch_pos, (builder->type_section.count - patch_pos) - 2); - return id; -} - -CV_TypeIndex tb_codeview_builder_add_record(CV_Builder* builder, CV_RecordType rec_type, size_t field_count, CV_TypeIndex field_list, TB_CharUnits size, const char* name) { - // write struct type - CV_TypeIndex id = builder->type_entry_count++; - size_t name_size = strlen(name) + 1; - - size_t patch_pos = builder->type_section.count; - CV_LFStruct s = { - .leaf = rec_type, - .count = field_count, - .field = field_list, - }; - tb_outs(&builder->type_section, sizeof(s), &s); - - if (size == (int8_t) size) { - // write it as 3 bytes instead of 6 since it's a common case - tb_out2b(&builder->type_section, LF_CHAR); - tb_out1b(&builder->type_section, size); - } else { - // write size (simple LF_LONG) - tb_out2b(&builder->type_section, LF_LONG); - tb_out4b(&builder->type_section, size); - } - - // write name - tb_outs(&builder->type_section, name_size, name); - - // backpatch length - tb_patch2b(&builder->type_section, patch_pos, (builder->type_section.count - patch_pos) - 2); - return id; -} diff --git a/vendor/tb/src/debug/sdg.c b/vendor/tb/src/debug/sdg.c deleted file mode 100644 index 959eeca1..00000000 --- a/vendor/tb/src/debug/sdg.c +++ /dev/null @@ -1,221 +0,0 @@ -#include "../tb_internal.h" -#include - -typedef struct { - TB_Emitter e; - size_t count; -} SDG_Types; - -static SDG_TypeIndex sdg_get_type(SDG_Types* types, TB_DebugType* type) { - if (type->type_id != 0) { - return type->type_id; - } - - switch (type->tag) { - case TB_DEBUG_TYPE_VOID: return (type->type_id = SDG_PRIM_VOID); - case TB_DEBUG_TYPE_BOOL: return (type->type_id = SDG_PRIM_BOOL8); - - case TB_DEBUG_TYPE_INT: - case TB_DEBUG_TYPE_UINT: { - bool is_signed = (type->tag == TB_DEBUG_TYPE_INT); - - if (type->int_bits <= 8) return is_signed ? SDG_PRIM_INT8 : SDG_PRIM_UINT8; - if (type->int_bits <= 16) return is_signed ? SDG_PRIM_INT16 : SDG_PRIM_UINT16; - if (type->int_bits <= 32) return is_signed ? SDG_PRIM_INT32 : SDG_PRIM_UINT32; - if (type->int_bits <= 64) return is_signed ? SDG_PRIM_INT64 : SDG_PRIM_UINT64; - assert(0 && "Unsupported int type"); - } - - case TB_DEBUG_TYPE_FLOAT: { - switch (type->float_fmt) { - case TB_FLT_32: return (type->type_id = T_REAL32); - case TB_FLT_64: return (type->type_id = T_REAL64); - default: assert(0 && "Unknown float type"); - } - } - - case TB_DEBUG_TYPE_FUNCTION: { - tb_todo(); - return (type->type_id = 0); - } - - case TB_DEBUG_TYPE_POINTER: { - SDG_TypeIndex ty = 0x100 + types->count++; - - SDG_Type t = { - .tag = SDG_TYPE_PTR, - .base = sdg_get_type(types, type->ptr_to) - }; - tb_outs(&types->e, sizeof(t), &t); - - return (type->type_id = ty); - } - - default: tb_todo(); - } -} - -static SDG_TypeIndex sdg_get_type_from_dt(TB_DataType dt) { - // assert(dt.width == 0 && "TODO: implement vector types in CodeView output"); - switch (dt.type) { - case TB_INT: { - if (dt.data <= 0) return SDG_PRIM_VOID; - if (dt.data <= 1) return SDG_PRIM_BOOL8; - if (dt.data <= 8) return SDG_PRIM_UINT8; - if (dt.data <= 16) return SDG_PRIM_UINT16; - if (dt.data <= 32) return SDG_PRIM_UINT32; - if (dt.data <= 64) return SDG_PRIM_UINT64; - return SDG_PRIM_VOID; - } - case TB_FLOAT: { - if (dt.data == TB_FLT_32) return SDG_PRIM_FLOAT; - if (dt.data == TB_FLT_64) return SDG_PRIM_DOUBLE; - - assert(0 && "Unknown float type"); - } - case TB_PTR: { - return SDG_PRIM_POINTER | SDG_PRIM_VOID; - } - default: return tb_assert(0, "todo: missing type in CodeView output"); - } -} - -static bool sdg_supported_target(TB_Module* m) { return true; } -static int sdg_number_of_debug_sections(TB_Module* m) { return 2; } - -static size_t write_normie_sym(TB_Emitter* e, TB_ObjectSection* section, int tag, const char* name, SDG_TypeIndex ty, size_t sym_id, size_t code_size) { - size_t len = strlen(name); - SDG_NormalSymbol sym = { - { .tag = tag, .content_ptr = sizeof(SDG_NormalSymbol) + len + 1 }, - .type = ty, .size = code_size - }; - - size_t sym_next_patch = tb_outs(e, sizeof(sym), &sym); - tb_outs(e, len + 1, name); - - // fill RVA - section->relocations[section->relocation_count++] = (TB_ObjectReloc){ - TB_OBJECT_RELOC_ADDR32NB, sym_id, - sym_next_patch + offsetof(SDG_NormalSymbol, rva) - }; - return sym_next_patch; -} - -// there's quite a few places that mark the next field for symbols -#define MARK_NEXT(patch_pos) (((SDG_Symbol*) tb_out_get(&symtab, patch_pos))->next = symtab.count) -#define MARK_KIDS(patch_pos, c) (((SDG_Symbol*) tb_out_get(&symtab, patch_pos))->kid_count = c) -static TB_SectionGroup sdg_generate_debug_info(TB_Module* m, TB_Arena* arena) { - TB_ObjectSection* sections = tb_platform_heap_alloc(1 * sizeof(TB_ObjectSection)); - sections[0] = (TB_ObjectSection){ gimme_cstr_as_slice(arena, ".sdg$S") }; - sections[1] = (TB_ObjectSection){ gimme_cstr_as_slice(arena, ".sdg$T") }; - - size_t reloc_cap = m->symbol_count[TB_SYMBOL_GLOBAL] + m->compiled_function_count; - sections[0].relocations = tb_platform_heap_alloc(reloc_cap * sizeof(TB_ObjectReloc)); - - TB_Emitter symtab = { 0 }; - SDG_Types types = { 0 }; - - TB_ArenaSavepoint sp = tb_arena_save(arena); - - // we only store one module so we never fill the next - SDG_Module mod = { { SDG_SYMBOL_MODULE, sizeof(SDG_Module)+sizeof("fallback.o") } }; - size_t type_table_patch = tb_outs(&symtab, sizeof(mod), &mod); - - tb_outs(&symtab, sizeof("fallback.o"), "fallback.o"); - - // emit file table into symbol table. - // skip the NULL file entry - size_t file_count = nl_map__get_header(m->files)->count; - size_t next_patch = 0; - - size_t mod_kids = 0; - nl_map_for_str(i, m->files) { - size_t len = m->files[i].v->len; - const uint8_t* data = m->files[i].v->path; - - SDG_File file = { { SDG_SYMBOL_FILE } }; - next_patch = tb_outs(&symtab, sizeof(file), &file); - - tb_outs(&symtab, len, data); - tb_out1b(&symtab, 0); - - MARK_NEXT(next_patch); - mod_kids++; - } - - // functions - dyn_array_for(i, m->sections) { - DynArray(TB_FunctionOutput*) funcs = m->sections[i].funcs; - dyn_array_for(j, funcs) { - TB_FunctionOutput* out_f = funcs[j]; - const char* name = out_f->parent->super.name; - size_t func_id = out_f->parent->super.symbol_id; - - SDG_TypeIndex ty = 0; - { - const TB_FunctionPrototype* proto = out_f->parent->prototype; - ty = 0x100 + types.count++; - - SDG_Type t = { SDG_TYPE_FUNC, proto->param_count }; - if (proto->return_count == 1) { - const TB_PrototypeParam* ret = &proto->params[proto->param_count]; - if (ret->debug_type) { - t.base = sdg_get_type(&types, ret->debug_type); - } else { - t.base = sdg_get_type_from_dt(ret->dt); - } - } else { - t.base = SDG_PRIM_VOID; - } - tb_outs(&types.e, sizeof(t), &t); - - FOREACH_N(i, 0, proto->param_count) { - TB_DebugType* param_ty = proto->params[i].debug_type; - SDG_TypeIndex param = param_ty ? sdg_get_type(&types, param_ty) : sdg_get_type_from_dt(proto->params[i].dt); - tb_outs(&types.e, sizeof(param), ¶m); - } - } - - next_patch = write_normie_sym(&symtab, §ions[0], SDG_SYMBOL_PROC, name, ty, func_id, out_f->code_size); - MARK_NEXT(next_patch); - mod_kids++; - } - - DynArray(TB_Global*) globals = m->sections[i].globals; - dyn_array_for(j, globals) { - TB_Global* g = globals[j]; - if (g->super.name[0] == 0) { - continue; - } - - SDG_TypeIndex ty = g->dbg_type ? sdg_get_type(&types, g->dbg_type) : SDG_PRIM_VOID; - - next_patch = write_normie_sym(&symtab, §ions[0], SDG_SYMBOL_GLOBAL, g->super.name, ty, g->super.symbol_id, g->size); - MARK_NEXT(next_patch); - mod_kids++; - } - - align_up_emitter(&symtab, 4); - } - - // clear next field - MARK_KIDS(0, mod_kids); - ((SDG_Symbol*) &symtab.data[next_patch])->next = 0; - - // write type table patch (it'll just follow immediately after the symbols) - ((SDG_Module*) &symtab.data[0])->type_table = symtab.count; - ((SDG_Module*) &symtab.data[0])->type_count = types.count; - - tb_arena_restore(arena, sp); - - sections[0].raw_data = (TB_Slice){ symtab.data, symtab.count }; - sections[1].raw_data = (TB_Slice){ types.e.data, types.e.count }; - return (TB_SectionGroup) { 2, sections }; -} - -IDebugFormat tb__sdg_debug_format = { - "SDG", - sdg_supported_target, - sdg_number_of_debug_sections, - sdg_generate_debug_info -}; diff --git a/vendor/tb/src/debug_builder.c b/vendor/tb/src/debug_builder.c deleted file mode 100644 index a5eef6c5..00000000 --- a/vendor/tb/src/debug_builder.c +++ /dev/null @@ -1,127 +0,0 @@ -#include "tb_internal.h" - -#define NEW(...) memcpy(make_type(m), &(TB_DebugType){ __VA_ARGS__ }, sizeof(TB_DebugType)) - -static TB_DebugType* make_type(TB_Module* m) { - return tb_arena_alloc(get_permanent_arena(m), sizeof(TB_DebugType)); -} - -TB_API TB_DebugType* tb_debug_get_void(TB_Module* m) { - static TB_DebugType type = { TB_DEBUG_TYPE_VOID }; - return &type; -} - -TB_API TB_DebugType* tb_debug_get_bool(TB_Module* m) { - static TB_DebugType type = { TB_DEBUG_TYPE_BOOL }; - return &type; -} - -TB_API TB_DebugType* tb_debug_get_integer(TB_Module* m, bool is_signed, int bits) { - static TB_DebugType types[] = { - { .tag = TB_DEBUG_TYPE_UINT, .int_bits = 1 }, - { .tag = TB_DEBUG_TYPE_UINT, .int_bits = 8 }, - { .tag = TB_DEBUG_TYPE_UINT, .int_bits = 16 }, - { .tag = TB_DEBUG_TYPE_UINT, .int_bits = 32 }, - { .tag = TB_DEBUG_TYPE_UINT, .int_bits = 64 }, - { .tag = TB_DEBUG_TYPE_UINT, .int_bits = 128 }, - - { .tag = TB_DEBUG_TYPE_INT, .int_bits = 1 }, - { .tag = TB_DEBUG_TYPE_INT, .int_bits = 8 }, - { .tag = TB_DEBUG_TYPE_INT, .int_bits = 16 }, - { .tag = TB_DEBUG_TYPE_INT, .int_bits = 32 }, - { .tag = TB_DEBUG_TYPE_INT, .int_bits = 64 }, - { .tag = TB_DEBUG_TYPE_INT, .int_bits = 128 }, - }; - - int b = (is_signed ? 6 : 0); - if (bits <= 1) return &types[b + 0]; - if (bits <= 8) return &types[b + 1]; - if (bits <= 16) return &types[b + 2]; - if (bits <= 32) return &types[b + 3]; - if (bits <= 64) return &types[b + 4]; - if (bits <= 128) return &types[b + 5]; - tb_todo(); -} - -TB_API TB_DebugType* tb_debug_get_float(TB_Module* m, TB_FloatFormat fmt) { - static TB_DebugType types[] = { - [TB_FLT_32] = { TB_DEBUG_TYPE_FLOAT, .float_fmt = TB_FLT_32 }, - [TB_FLT_64] = { TB_DEBUG_TYPE_FLOAT, .float_fmt = TB_FLT_64 }, - }; - - return &types[fmt]; -} - -TB_API TB_DebugType* tb_debug_create_ptr(TB_Module* m, TB_DebugType* base) { - assert(base != NULL); - return NEW(TB_DEBUG_TYPE_POINTER, .ptr_to = base); -} - -TB_API TB_DebugType* tb_debug_create_array(TB_Module* m, TB_DebugType* base, size_t count) { - return NEW(TB_DEBUG_TYPE_ARRAY, .array = { base, count }); -} - -TB_API TB_DebugType* tb_debug_create_struct(TB_Module* m, ptrdiff_t len, const char* tag) { - if (len < 0) len = tag ? strlen(tag) : 0; - - TB_DebugType* t = NEW(TB_DEBUG_TYPE_STRUCT); - t->record.tag = tb__arena_strdup(m, len, tag); - return t; -} - -TB_API TB_DebugType* tb_debug_create_union(TB_Module* m, ptrdiff_t len, const char* tag) { - if (len < 0) len = tag ? strlen(tag) : 0; - - TB_DebugType* t = NEW(TB_DEBUG_TYPE_UNION); - t->record.len = len; - t->record.tag = tb__arena_strdup(m, len, tag); - return t; -} - -TB_API TB_DebugType* tb_debug_create_alias(TB_Module* m, TB_DebugType* base, ptrdiff_t len, const char* name) { - if (len < 0) len = name ? strlen(name) : 0; - return NEW(TB_DEBUG_TYPE_ALIAS, .alias = { len, tb__arena_strdup(m, len, name), base }); -} - -TB_API TB_DebugType* tb_debug_create_field(TB_Module* m, TB_DebugType* type, ptrdiff_t len, const char* name, TB_CharUnits offset) { - assert(name); - if (len < 0) len = name ? strlen(name) : 0; - return NEW(TB_DEBUG_TYPE_FIELD, .field = { len, tb__arena_strdup(m, len, name), offset, type }); -} - -TB_API TB_DebugType** tb_debug_record_begin(TB_Module* m, TB_DebugType* type, size_t count) { - type->record.count = count; - return (type->record.members = tb_arena_alloc(get_permanent_arena(m), count * sizeof(TB_DebugType*))); -} - -TB_API void tb_debug_record_end(TB_DebugType* type, TB_CharUnits size, TB_CharUnits align) { - type->record.size = size; - type->record.align = align; -} - -TB_API TB_DebugType* tb_debug_create_func(TB_Module* m, TB_CallingConv cc, size_t param_count, size_t return_count, bool has_varargs) { - TB_DebugType* t = NEW(TB_DEBUG_TYPE_FUNCTION); - t->func.cc = cc; - t->func.has_varargs = has_varargs; - t->func.param_count = param_count; - t->func.params = TB_ARENA_ARR_ALLOC(get_permanent_arena(m), param_count, TB_DebugType*); - t->func.return_count = return_count; - t->func.returns = TB_ARENA_ARR_ALLOC(get_permanent_arena(m), return_count, TB_DebugType*); - return t; -} - -TB_API size_t tb_debug_func_return_count(TB_DebugType* type) { - return type->func.return_count; -} - -TB_API size_t tb_debug_func_param_count(TB_DebugType* type) { - return type->func.param_count; -} - -TB_API TB_DebugType** tb_debug_func_params(TB_DebugType* type) { - return type->func.params; -} - -TB_API TB_DebugType** tb_debug_func_returns(TB_DebugType* type) { - return type->func.returns; -} diff --git a/vendor/tb/src/disasm.c b/vendor/tb/src/disasm.c deleted file mode 100644 index 192c16fb..00000000 --- a/vendor/tb/src/disasm.c +++ /dev/null @@ -1,21 +0,0 @@ -#include - -ptrdiff_t tb_print_disassembly_inst(TB_Arch arch, size_t length, const void* ptr) { - switch (arch) { - #ifdef TB_HAS_X64 - case TB_ARCH_X86_64: { - TB_X86_Inst inst; - if (!tb_x86_disasm(&inst, length, ptr)) { - return -1; - } - - tb_x86_print_inst(stdout, &inst); - return inst.length; - } - #endif - - default: - tb_todo(); - } -} - diff --git a/vendor/tb/src/emitter.h b/vendor/tb/src/emitter.h deleted file mode 100644 index 4f4fe323..00000000 --- a/vendor/tb/src/emitter.h +++ /dev/null @@ -1,168 +0,0 @@ -// cgemit is short for Codegen Emitter -// -// for better runtime performance this is included into the specific -// files it's used in -#pragma once -#include "tb_internal.h" - -// We really only need the position where to patch -// it since it's all internal and the target is implicit. -typedef uint32_t ReturnPatch; - -typedef struct LabelPatch { - int pos; - int target_lbl; -} LabelPatch; - -typedef struct Comment { - struct Comment* next; - uint32_t pos; - uint32_t line_len; - char line[]; -} Comment; - -typedef struct { - // technically NULLable, just can't use patches if NULL - TB_FunctionOutput* output; - TB_Arena* arena; - - TB_Assembly *head_asm, *tail_asm; - uint64_t total_asm; - - // this is mapped to a giant buffer and is technically - // allow to use the entire rest of said buffer - size_t count, capacity; - uint8_t* data; - - size_t label_count; - uint32_t* labels; - - bool has_comments; - Comment* comment_head; - Comment* comment_tail; -} TB_CGEmitter; - -// Helper macros -#define EMITA(e, fmt, ...) tb_asm_print(e, fmt, ## __VA_ARGS__) -#define EMIT1(e, b) (*((uint8_t*) tb_cgemit_reserve(e, 1)) = (b), (e)->count += 1) -#define EMIT2(e, b) do { uint16_t _b = (b); memcpy(tb_cgemit_reserve(e, 2), &_b, 2); (e)->count += 2; } while (0) -#define EMIT4(e, b) do { uint32_t _b = (b); memcpy(tb_cgemit_reserve(e, 4), &_b, 4); (e)->count += 4; } while (0) -#define EMIT8(e, b) do { uint64_t _b = (b); memcpy(tb_cgemit_reserve(e, 8), &_b, 8); (e)->count += 8; } while (0) -#define PATCH2(e, p, b) do { uint16_t _b = (b); memcpy(&(e)->data[p], &_b, 2); } while (0) -#define PATCH4(e, p, b) do { uint32_t _b = (b); memcpy(&(e)->data[p], &_b, 4); } while (0) -#define GET_CODE_POS(e) ((e)->count) -#define RELOC4(e, p, b) tb_reloc4(e, p, b) - -static void tb_reloc4(TB_CGEmitter* restrict e, uint32_t p, uint32_t b) { - void* ptr = &e->data[p]; - - // i love UBsan... - uint32_t tmp; - memcpy(&tmp, ptr, 4); - tmp += b; - memcpy(ptr, &tmp, 4); -} - -static int tb_emit_get_label(TB_CGEmitter* restrict e, uint32_t pos) { - FOREACH_N(i, 0, e->label_count) { - assert(e->labels[i] & 0x80000000); - if ((e->labels[i] & ~0x80000000) == pos) { - return i; - } - } - - return 0; -} - -static void tb_emit_comment(TB_CGEmitter* restrict e, TB_Arena* arena, const char* fmt, ...) { - Comment* comment = tb_arena_alloc(arena, sizeof(Comment) + 100); - comment->next = NULL; - comment->pos = e->count; - - va_list ap; - va_start(ap, fmt); - comment->line_len = vsnprintf(comment->line, 100, fmt, ap); - va_end(ap); - - if (e->comment_tail) { - e->comment_tail->next = comment; - e->comment_tail = comment; - } else { - e->comment_head = e->comment_tail = comment; - } -} - -static void tb_asm_print(TB_CGEmitter* restrict e, const char* fmt, ...) { - // make sure we have enough bytes for the operation - TB_Assembly* new_head = e->tail_asm; - if (new_head == NULL || new_head->length + 100 >= TB_ASSEMBLY_CHUNK_CAP) { - new_head = tb_platform_valloc(TB_ASSEMBLY_CHUNK_CAP); - // new_head->next = NULL; - // new_head->length = 0; - - if (e->tail_asm == NULL) { - e->tail_asm = e->head_asm = new_head; - } else { - e->tail_asm->next = new_head; - e->tail_asm = new_head; - } - } - - va_list ap; - va_start(ap, fmt); - int len = vsnprintf(&new_head->data[new_head->length], 100, fmt, ap); - va_end(ap); - - new_head->length += len; - e->total_asm += len; -} - -static void tb_emit_rel32(TB_CGEmitter* restrict e, uint32_t* head, uint32_t pos) { - uint32_t curr = *head; - if (curr & 0x80000000) { - // the label target is resolved, we need to do the relocation now - uint32_t target = curr & 0x7FFFFFFF; - PATCH4(e, pos, target - (pos + 4)); - } else { - PATCH4(e, pos, curr); - *head = pos; - } -} - -static void tb_resolve_rel32(TB_CGEmitter* restrict e, uint32_t* head, uint32_t target) { - // walk previous relocations - uint32_t curr = *head; - while (curr != 0 && (curr & 0x80000000) == 0) { - uint32_t next; - memcpy(&next, &e->data[curr], 4); - PATCH4(e, curr, target - (curr + 4)); - curr = next; - } - - // store the target and mark it as resolved - *head = 0x80000000 | target; -} - -static void* tb_cgemit_reserve(TB_CGEmitter* restrict e, size_t count) { - if (e->count + count >= e->capacity) { - // we don't really want massive code buffers... functions shouldn't really be that big - size_t chunk_size = tb_arena_chunk_size(e->arena); - if (e->capacity >= chunk_size - sizeof(TB_Arena)) { - tb_panic("could not allocate code buffer (too big lmao)\n"); - } - - // reallocate arena - size_t old_cap = e->capacity; - void* old = e->data; - - e->capacity = chunk_size - sizeof(TB_Arena); - e->data = tb_arena_alloc(e->arena, e->capacity); - memcpy(e->data, old, old_cap); - } - - return &e->data[e->count]; -} - -static void tb_cgemit_commit(TB_CGEmitter* restrict e, size_t bytes) { - e->count += bytes; -} diff --git a/vendor/tb/src/exporter.c b/vendor/tb/src/exporter.c deleted file mode 100644 index 6fde96c2..00000000 --- a/vendor/tb/src/exporter.c +++ /dev/null @@ -1,246 +0,0 @@ -#include "tb_internal.h" - -TB_ExportBuffer tb_coff_write_output(TB_Module* restrict m, TB_Arena* dst_arena, const IDebugFormat* dbg); -TB_ExportBuffer tb_macho_write_output(TB_Module* restrict m, TB_Arena* dst_arena, const IDebugFormat* dbg); -TB_ExportBuffer tb_elf64obj_write_output(TB_Module* restrict m, TB_Arena* dst_arena, const IDebugFormat* dbg); -TB_ExportBuffer tb_wasm_write_output(TB_Module* restrict m, TB_Arena* dst_arena, const IDebugFormat* dbg); - -static const IDebugFormat* find_debug_format(TB_DebugFormat debug_fmt) { - // Place all debug formats here - extern IDebugFormat tb__codeview_debug_format; - extern IDebugFormat tb__sdg_debug_format; - - switch (debug_fmt) { - case TB_DEBUGFMT_SDG: return &tb__sdg_debug_format; - case TB_DEBUGFMT_CODEVIEW: return &tb__codeview_debug_format; - default: return NULL; - } -} - -TB_API TB_ExportBuffer tb_module_object_export(TB_Module* m, TB_Arena* dst_arena, TB_DebugFormat debug_fmt){ - typedef TB_ExportBuffer ExporterFn(TB_Module* restrict m, TB_Arena* dst_arena, const IDebugFormat* dbg); - - // map target systems to exporters (maybe we wanna decouple this later) - static ExporterFn* const fn[TB_SYSTEM_MAX] = { - [TB_SYSTEM_WINDOWS] = tb_coff_write_output, - [TB_SYSTEM_MACOS] = tb_macho_write_output, - [TB_SYSTEM_LINUX] = tb_elf64obj_write_output, - [TB_SYSTEM_WASM] = tb_wasm_write_output, - }; - - assert(fn[m->target_system] != NULL && "TODO"); - TB_ExportBuffer e; - CUIK_TIMED_BLOCK("export") { - e = fn[m->target_system](m, dst_arena, find_debug_format(debug_fmt)); - } - return e; -} - -TB_API bool tb_export_buffer_to_file(TB_ExportBuffer buffer, const char* path) { - if (buffer.total == 0) { - fprintf(stderr, "\x1b[31merror\x1b[0m: could not export '%s' (no contents)\n", path); - return false; - } - - FILE* file = fopen(path, "wb"); - if (file == NULL) { - fprintf(stderr, "\x1b[31merror\x1b[0m: could not open file for writing! %s\n", path); - return false; - } - - for (TB_ExportChunk* c = buffer.head; c != NULL; c = c->next) { - if (c->size > 0 && fwrite(c->data, c->size, 1, file) != 1) { - fprintf(stderr, "\x1b[31merror\x1b[0m: could not write to file! %s (not enough storage?)\n", path); - return false; - } - } - - fclose(file); - return true; -} - -static int compare_symbols(const void* a, const void* b) { - const TB_Symbol* sym_a = *(const TB_Symbol**) a; - const TB_Symbol* sym_b = *(const TB_Symbol**) b; - return (sym_a->ordinal > sym_b->ordinal) - (sym_a->ordinal < sym_b->ordinal); -} - -static int compare_functions(const void* a, const void* b) { - const TB_FunctionOutput* sym_a = *(const TB_FunctionOutput**) a; - const TB_FunctionOutput* sym_b = *(const TB_FunctionOutput**) b; - return (sym_a->ordinal > sym_b->ordinal) - (sym_a->ordinal < sym_b->ordinal); -} - -static void layout_section(TB_ModuleSection* restrict section) { - CUIK_TIMED_BLOCK_ARGS("layout section", section->name) { - size_t offset = 0; - - dyn_array_for(i, section->globals) { - TB_Global* g = section->globals[i]; - - offset = align_up(offset, g->align); - g->pos = offset; - offset += g->size; - } - section->total_size = offset; - } -} - -ExportList tb_module_layout_sections(TB_Module* m) { - TB_Arena* arena = tb_thread_info(m)->tmp_arena; - - size_t external_count = 0; - TB_External** externals = tb_arena_alloc(arena, m->symbol_count[TB_SYMBOL_EXTERNAL] * sizeof(TB_External*)); - - // unpack function data into the streams we actually care for. - // avoids needing to walk so many sparse data structures later on. - TB_ThreadInfo* info = atomic_load_explicit(&m->first_info_in_module, memory_order_relaxed); - while (info != NULL) { - TB_ThreadInfo* next = info->next_in_module; - - // unpack symbols - TB_Symbol** syms = (TB_Symbol**) info->symbols.data; - if (syms) FOREACH_N(i, 0, 1ull << info->symbols.exp) { - TB_Symbol* s = syms[i]; - if (s == NULL || s == NL_HASHSET_TOMB) continue; - - switch (atomic_load_explicit(&s->tag, memory_order_relaxed)) { - case TB_SYMBOL_FUNCTION: { - TB_Function* f = (TB_Function*) s; - TB_ModuleSection* sec = &m->sections[f->section]; - - // we only care for compiled functions - TB_FunctionOutput* out_f = f->output; - if (out_f != NULL) { - out_f->ordinal = f->super.ordinal; - dyn_array_put(sec->funcs, out_f); - } - break; - } - case TB_SYMBOL_GLOBAL: { - TB_Global* g = (TB_Global*) s; - TB_ModuleSection* sec = &m->sections[g->parent]; - dyn_array_put(sec->globals, g); - break; - } - case TB_SYMBOL_EXTERNAL: { - // resolved externals are just globals or functions, we don't add them here - TB_External* e = (TB_External*) s; - if (atomic_load_explicit(&e->resolved, memory_order_relaxed) == NULL) { - externals[external_count++] = e; - } - break; - } - default: break; - } - } - - info = next; - } - - dyn_array_for(i, m->sections) { - TB_ModuleSection* sec = &m->sections[i]; - - CUIK_TIMED_BLOCK_ARGS("sort", sec->name) { - qsort(sec->funcs, dyn_array_length(sec->funcs), sizeof(TB_FunctionOutput*), compare_functions); - qsort(sec->globals, dyn_array_length(sec->globals), sizeof(TB_Symbol*), compare_symbols); - } - - // layout - CUIK_TIMED_BLOCK_ARGS("layout", sec->name) { - // place functions first - size_t offset = 0; - dyn_array_for(i, sec->funcs) { - sec->funcs[i]->code_pos = offset; - offset += sec->funcs[i]->code_size; - } - - // then globals - dyn_array_for(i, sec->globals) { - TB_Global* g = sec->globals[i]; - - offset = align_up(offset, g->align); - g->pos = offset; - offset += g->size; - } - - sec->total_size = offset; - } - } - - return (ExportList){ external_count, externals }; -} - -size_t tb_helper_write_section(TB_Module* m, size_t write_pos, TB_ModuleSection* section, uint8_t* output, uint32_t pos) { - assert(write_pos == pos); - uint8_t* data = &output[pos]; - - // place functions - dyn_array_for(i, section->funcs) { - TB_FunctionOutput* out_f = section->funcs[i]; - - if (out_f != NULL) { - memcpy(data + out_f->code_pos, out_f->code, out_f->code_size); - } - } - - // place globals - dyn_array_for(i, section->globals) { - TB_Global* restrict g = section->globals[i]; - - memset(&data[g->pos], 0, g->size); - FOREACH_N(k, 0, g->obj_count) { - if (g->objects[k].type == TB_INIT_OBJ_REGION) { - assert(g->objects[k].offset + g->objects[k].region.size <= g->size); - memcpy(&data[g->pos + g->objects[k].offset], g->objects[k].region.ptr, g->objects[k].region.size); - } - } - } - - return write_pos + section->total_size; -} - -size_t tb__layout_relocations(TB_Module* m, DynArray(TB_ModuleSection) sections, const ICodeGen* restrict code_gen, size_t output_size, size_t reloc_size) { - // calculate relocation layout - dyn_array_for(i, sections) { - TB_ModuleSection* sec = §ions[i]; - size_t reloc_count = 0; - - dyn_array_for(i, sec->funcs) { - reloc_count += code_gen->emit_call_patches(m, sec->funcs[i]); - } - - dyn_array_for(j, sec->globals) { - TB_Global* restrict g = sec->globals[j]; - FOREACH_N(k, 0, g->obj_count) { - reloc_count += (g->objects[k].type == TB_INIT_OBJ_RELOC); - } - } - - sec->reloc_count = reloc_count; - sec->reloc_pos = output_size; - output_size += reloc_count * reloc_size; - } - - return output_size; -} - -TB_ExportChunk* tb_export_make_chunk(TB_Arena* arena, size_t size) { - TB_ExportChunk* c = tb_arena_alloc(arena, sizeof(TB_ExportChunk) + size); - c->next = NULL; - c->pos = 0; - c->size = size; - return c; -} - -void tb_export_append_chunk(TB_ExportBuffer* buffer, TB_ExportChunk* c) { - if (buffer->head == NULL) { - buffer->head = buffer->tail = c; - } else { - buffer->tail->next = c; - buffer->tail = c; - } - - c->pos = buffer->total; - buffer->total += c->size; -} diff --git a/vendor/tb/src/generic_cg.h b/vendor/tb/src/generic_cg.h deleted file mode 100644 index cc5b4b19..00000000 --- a/vendor/tb/src/generic_cg.h +++ /dev/null @@ -1,1109 +0,0 @@ -#include "opt/passes.h" -#include "emitter.h" -#include -#include - -static thread_local bool reg_alloc_log; - -enum { - CG_VAL_UNRESOLVED = 0, - CG_VAL_FLAGS = 1, - CG_VAL_REGISTER = 2, -}; - -enum { - INST_LABEL = 1024, - INST_LINE, - - // inline machine code - INST_INLINE, - - // marks the terminator - INST_TERMINATOR, - INST_EPILOGUE, - - // this is where parameters come from - INST_ENTRY, - - // XORPS xmm0, xmm0 - // or XOR eax, eax - INST_ZERO, -}; - -typedef struct Inst Inst; - -// the first set of indices are reserved for physical registers, the -// rest are allocated as virtual registers. -typedef int RegIndex; -_Static_assert(sizeof(TB_PhysicalReg) == sizeof(RegIndex), "these should be the same"); - -typedef struct MachineBB { - Inst* first; - - // what's the terminator, it helps us walk successors - TB_Node* end_node; - - int start, end; - int terminator; - - // local live sets - Set gen, kill; - // global - Set live_in, live_out; -} MachineBB; - -typedef struct { - int uses; - RegIndex vreg; -} ValueDesc; - -typedef struct LiveInterval LiveInterval; -typedef NL_Map(TB_Node*, MachineBB) MachineBBs; - -typedef struct { - uint32_t* pos; - uint32_t target; -} JumpTablePatch; - -typedef struct { - TB_CGEmitter emit; - - TB_Module* module; - TB_Function* f; - TB_ABI target_abi; - - int caller_usage; - int fallthrough; - - TB_Passes* p; - - int bb_count; - int* bb_order; - - // Scheduling - TB_CFG cfg; - Worklist worklist; // reusing from TB_Passes. - ValueDesc* values; // the indices match the GVN. - TB_Scheduler sched; - - size_t our_phis; - DynArray(PhiVal) phi_vals; - DynArray(JumpTablePatch) jump_table_patches; - - // Regalloc - DynArray(LiveInterval) intervals; - - // machine output sequences - Inst *first, *head; - MachineBBs machine_bbs; - - // Line info - DynArray(TB_Location) locations; - - // Stack - uint32_t stack_usage; - NL_Map(TB_Node*, int) stack_slots; - DynArray(TB_StackSlot) debug_stack_slots; - - uint64_t regs_to_save; -} Ctx; - -typedef struct { - TB_SymbolPatch* patch; - TB_Location* loc; - TB_Location* end; -} Disasm; - -static bool fits_into_int8(uint64_t x) { - int8_t y = x & 0xFF; - return (int64_t)y == x; -} - -static bool fits_into_int32(uint64_t x) { - uint32_t hi = x >> 32ull; - return hi == 0 || hi == 0xFFFFFFFF; -} - -static void init_regalloc(Ctx* restrict ctx); - -static TB_X86_DataType legalize(TB_DataType dt); -static bool is_terminator(int type); -static bool wont_spill_around(int type); -static int classify_reg_class(TB_DataType dt); -static void isel(Ctx* restrict ctx, TB_Node* n, int dst); -static void disassemble(TB_CGEmitter* e, Disasm* restrict d, int bb, size_t pos, size_t end); -static bool should_rematerialize(TB_Node* n); - -static void emit_code(Ctx* restrict ctx, TB_FunctionOutput* restrict func_out); -static void mark_callee_saved_constraints(Ctx* restrict ctx, uint64_t callee_saved[CG_REGISTER_CLASSES]); - -static void add_debug_local(Ctx* restrict ctx, TB_Node* n, int pos) { - ptrdiff_t search = nl_map_get(ctx->f->attribs, n); - if (search < 0) { - return; - } - - // could be costly if you had more than like 50 attributes per stack slot... which you - // wouldn't do right? - DynArray(TB_Attrib) attribs = ctx->f->attribs[search].v; - dyn_array_for(i, attribs) { - TB_Attrib* a = &attribs[i]; - if (a->tag == TB_ATTRIB_VARIABLE) { - TB_StackSlot s = { - .position = pos, - .storage_type = a->var.storage, - .name = a->var.name, - }; - dyn_array_put(ctx->debug_stack_slots, s); - break; - } - } -} - -static const char* reg_name(int rg, int num) { - return (rg == REG_CLASS_XMM ? XMM_NAMES : GPR_NAMES)[num]; -} - -//////////////////////////////// -// Instructions -//////////////////////////////// -typedef enum { - INST_LOCK = 1, - INST_REP = 2, - INST_REPNE = 4, - - // operands - INST_MEM = 16, - INST_GLOBAL = 32, // operand to TB_Symbol* - INST_NODE = 64, // operand to TB_Node* - INST_IMM = 128, // operand in imm - INST_ABS = 256, // operand in abs - - // memory op - INST_INDEXED = 1024, - INST_SPILL = 2048, - - // epilogue - INST_RET = 4096, -} InstFlags; - -struct Inst { - Inst* next; - - // prefixes - InstType type; - InstFlags flags; - - TB_X86_DataType dt; - int time; - - short mem_slot; - uint8_t save_count; - - union { - TB_Symbol* s; - TB_Node* n; - TB_Attrib* a; - int l; - }; - - union { - int32_t imm; - uint64_t abs; - }; - - int32_t disp; - - uint8_t scale; - uint8_t out_count, in_count, tmp_count; - - // operands all go after the instruction in memory. - // - // RegIndex outs[out_count]; - // RegIndex ins[in_count]; - // RegIndex tmps[tmp_count]; - // RegIndex saves[save_count]; - // - RegIndex operands[]; -}; - -// generic instructions -static Inst* inst_jmp(int target); - -static Inst* inst_label(TB_Node* n) { - Inst* i = TB_ARENA_ALLOC(tmp_arena, Inst); - *i = (Inst){ .type = INST_LABEL, .flags = INST_NODE, .n = n }; - return i; -} - -static Inst* inst_line(TB_Node* n) { - Inst* i = TB_ARENA_ALLOC(tmp_arena, Inst); - *i = (Inst){ .type = INST_LINE, .flags = INST_NODE, .n = n }; - return i; -} - -#define SUBMIT(i) append_inst(ctx, i) -static void append_inst(Ctx* restrict ctx, Inst* inst) { - ctx->head->next = inst; - ctx->head = inst; -} - -static Inst* alloc_inst(int type, TB_DataType dt, int outs, int ins, int tmps) { - int total = outs + ins + tmps; - Inst* i = tb_arena_alloc(tmp_arena, sizeof(Inst) + (total * sizeof(RegIndex))); - *i = (Inst){ .type = type, .dt = legalize(dt), .out_count = outs, ins, tmps }; - return i; -} - -static Inst* inst_move(TB_DataType dt, RegIndex dst, RegIndex src) { - assert(dst >= 0); - int machine_dt = legalize(dt); - - Inst* i = tb_arena_alloc(tmp_arena, sizeof(Inst) + (2 * sizeof(RegIndex))); - *i = (Inst){ .type = machine_dt >= TB_X86_TYPE_SSE_SS ? FP_MOV : MOV, .dt = machine_dt, .out_count = 1, 1 }; - i->operands[0] = dst; - i->operands[1] = src; - return i; -} - -static Inst* inst_op_global(int type, TB_DataType dt, RegIndex dst, TB_Symbol* s) { - Inst* i = alloc_inst(type, dt, 1, 1, 0); - i->flags = INST_GLOBAL; - i->mem_slot = 1; - i->operands[0] = dst; - i->operands[1] = RSP; - i->s = s; - i->disp = 0; - return i; -} - -static Inst* inst_op_global_disp(int type, TB_DataType dt, RegIndex dst, TB_Symbol* s, int32_t disp) { - Inst* i = alloc_inst(type, dt, 1, 1, 0); - i->flags = INST_GLOBAL; - i->mem_slot = 1; - i->operands[0] = dst; - i->operands[1] = RSP; - i->s = s; - i->disp = disp; - return i; -} - -static Inst* inst_op_abs(int type, TB_DataType dt, RegIndex dst, uint64_t imm) { - Inst* i = alloc_inst(type, dt, 1, 0, 0); - i->flags = INST_ABS; - i->operands[0] = dst; - i->abs = imm; - return i; -} - -static Inst* inst_op_rm(int type, TB_DataType dt, RegIndex dst, RegIndex base, RegIndex index, Scale scale, int32_t disp) { - Inst* i = alloc_inst(type, dt, 1, index >= 0 ? 2 : 1, 0); - i->flags = INST_MEM | (index >= 0 ? INST_INDEXED : 0); - i->mem_slot = 1; - i->operands[0] = dst; - i->operands[1] = base; - if (index >= 0) { - i->operands[2] = index; - } - i->disp = disp; - i->scale = scale; - return i; -} - -static Inst* inst_op_rrm(int type, TB_DataType dt, RegIndex dst, RegIndex src, RegIndex base, RegIndex index, Scale scale, int32_t disp) { - Inst* i = alloc_inst(type, dt, 1, index >= 0 ? 3 : 2, 0); - i->flags = INST_MEM | (index >= 0 ? INST_INDEXED : 0); - i->mem_slot = 2; - i->operands[0] = dst; - i->operands[1] = src; - i->operands[2] = base; - if (index >= 0) { - i->operands[3] = index; - } - i->disp = disp; - i->scale = scale; - return i; -} - -static Inst* inst_op_mr(int type, TB_DataType dt, RegIndex base, RegIndex index, Scale scale, int32_t disp, RegIndex src) { - Inst* i = alloc_inst(type, dt, 0, index >= 0 ? 3 : 2, 0); - i->flags = INST_MEM | (index >= 0 ? INST_INDEXED : 0); - i->mem_slot = 0; - if (index >= 0) { - i->operands[0] = base; - i->operands[1] = index; - i->operands[2] = src; - } else { - i->operands[0] = base; - i->operands[1] = src; - } - i->disp = disp; - i->scale = scale; - return i; -} - -static Inst* inst_op_rri(int type, TB_DataType dt, RegIndex dst, RegIndex src, int32_t imm) { - Inst* i = alloc_inst(type, dt, 1, 1, 0); - i->flags = INST_IMM; - i->operands[0] = dst; - i->operands[1] = src; - i->imm = imm; - return i; -} - -static Inst* inst_op_rrr(int type, TB_DataType dt, RegIndex dst, RegIndex lhs, RegIndex rhs) { - Inst* i = alloc_inst(type, dt, 1, 2, 0); - i->operands[0] = dst; - i->operands[1] = lhs; - i->operands[2] = rhs; - return i; -} - -static Inst* inst_op_rri_tmp(int type, TB_DataType dt, RegIndex dst, RegIndex src, int32_t imm, RegIndex tmp) { - Inst* i = alloc_inst(type, dt, 1, 1, 1); - i->flags = INST_IMM; - i->operands[0] = dst; - i->operands[1] = src; - i->operands[2] = tmp; - i->imm = imm; - return i; -} - -static Inst* inst_op_rrr_tmp(int type, TB_DataType dt, RegIndex dst, RegIndex lhs, RegIndex rhs, RegIndex tmp) { - Inst* i = alloc_inst(type, dt, 1, 2, 1); - i->operands[0] = dst; - i->operands[1] = lhs; - i->operands[2] = rhs; - i->operands[3] = tmp; - return i; -} - -static Inst* inst_op_imm(int type, TB_DataType dt, RegIndex dst, int32_t imm) { - Inst* i = alloc_inst(type, dt, 1, 0, 0); - i->flags = INST_IMM; - i->operands[0] = dst; - i->imm = imm; - return i; -} - -static Inst* inst_op_ri(int type, TB_DataType dt, RegIndex src, int32_t imm) { - Inst* i = alloc_inst(type, dt, 0, 1, 0); - i->flags = INST_IMM; - i->operands[0] = src; - i->imm = imm; - return i; -} - -static Inst* inst_op_r(int type, TB_DataType dt, RegIndex dst) { - Inst* i = alloc_inst(type, dt, 1, 0, 0); - i->operands[0] = dst; - return i; -} - -static Inst* inst_op_rr(int type, TB_DataType dt, RegIndex dst, RegIndex src) { - Inst* i = alloc_inst(type, dt, 1, 1, 0); - i->operands[0] = dst; - i->operands[1] = src; - return i; -} - -static Inst* inst_op_rr_no_dst(int type, TB_DataType dt, RegIndex lhs, RegIndex rhs) { - Inst* i = alloc_inst(type, dt, 0, 2, 0); - i->operands[0] = lhs; - i->operands[1] = rhs; - return i; -} - -static Inst* inst_op_zero(TB_DataType dt, RegIndex dst) { - Inst* i = alloc_inst(INST_ZERO, dt, 1, 0, 0); - i->operands[0] = dst; - return i; -} - -//////////////////////////////// -// Register allocation -//////////////////////////////// -#include "reg_alloc.h" - -#define DEF(n, dt) alloc_vreg(ctx, n, dt) -static int alloc_vreg(Ctx* restrict ctx, TB_Node* n, TB_DataType dt) { - int i = dyn_array_length(ctx->intervals); - dyn_array_put(ctx->intervals, (LiveInterval){ - .reg_class = classify_reg_class(dt), - .n = n, .reg = -1, .hint = -1, .assigned = -1, - .dt = legalize(dt), .split_kid = -1 - }); - - LiveInterval* it = &ctx->intervals[i]; - it->ranges = tb_platform_heap_alloc(4 * sizeof(LiveRange)); - it->range_count = 1; - it->range_cap = 4; - it->ranges[0] = (LiveRange){ INT_MAX, INT_MAX }; - return i; -} - -static void hint_reg(Ctx* restrict ctx, int i, int j) { - if (ctx->intervals[i].hint < 0) { - ctx->intervals[i].hint = j; - } -} - -//////////////////////////////// -// Data flow analysis -//////////////////////////////// -static DynArray(int) liveness(Ctx* restrict ctx, TB_Function* f) { - size_t interval_count = dyn_array_length(ctx->intervals); - TB_Arena* arena = tmp_arena; - - // find BB boundaries in sequences - MachineBBs seq_bb = NULL; - nl_map_create(seq_bb, ctx->bb_count); - - TB_Node** bbs = ctx->worklist.items; - int* bb_order = ctx->bb_order; - FOREACH_N(i, 0, ctx->bb_count) { - TB_Node* n = bbs[bb_order[i]]; - TB_BasicBlock* bb = &nl_map_get_checked(ctx->cfg.node_to_block, n); - - MachineBB mbb = { - .end_node = bb->end, - .gen = set_create_in_arena(arena, interval_count), - .kill = set_create_in_arena(arena, interval_count), - .live_in = set_create_in_arena(arena, interval_count), - .live_out = set_create_in_arena(arena, interval_count) - }; - - nl_map_put(seq_bb, n, mbb); - } - - // generate local live sets - int timeline = 4; - DynArray(int) epilogues = NULL; - - // CUIK_TIMED_BLOCK("local liveness") - { - if (ctx->first) { - Inst* restrict inst = ctx->first; - assert(inst->type == INST_LABEL); - - // initial label - TB_Node* bb = ctx->worklist.items[0]; - MachineBB* mbb = &nl_map_get_checked(seq_bb, bb); - mbb->first = inst; - mbb->start = 2; - inst->time = 2; - inst = inst->next; - - for (; inst; inst = inst->next) { - if (inst->type == INST_LABEL) { - nl_map_get_checked(seq_bb, bb).end = timeline; - timeline += 4; - - tb_assert(inst->flags & INST_NODE, "label instruction has no TB_Node* for the region"); - bb = inst->n; - mbb = &nl_map_get_checked(seq_bb, bb); - mbb->first = inst->next; - mbb->start = timeline; - } else if (is_terminator(inst->type) && mbb->terminator == 0) { - mbb->terminator = timeline; - } else if (inst->type == INST_EPILOGUE) { - dyn_array_put(epilogues, timeline); - } - - Set* restrict gen = &mbb->gen; - Set* restrict kill = &mbb->kill; - - inst->time = timeline; - timeline += 2; - - RegIndex* ins = inst->operands + inst->out_count; - FOREACH_N(i, 0, inst->in_count) { - if (!set_get(kill, ins[i])) { - set_put(gen, ins[i]); - } - } - - RegIndex* outs = inst->operands; - FOREACH_N(i, 0, inst->out_count) { - set_put(kill, outs[i]); - } - } - - mbb->end = timeline; - } - } - - // generate global live sets - size_t base = dyn_array_length(ctx->worklist.items); - - // all nodes go into the worklist - FOREACH_REVERSE_N(i, 0, ctx->bb_count) { - TB_Node* n = bbs[bb_order[i]]; - dyn_array_put(ctx->worklist.items, n); - - // in(bb) = use(bb) - MachineBB* mbb = &nl_map_get_checked(seq_bb, n); - set_copy(&mbb->live_in, &mbb->gen); - } - - TB_ArenaSavepoint sp = tb_arena_save(arena); - Set visited = set_create_in_arena(arena, ctx->f->node_count); - while (dyn_array_length(ctx->worklist.items) > base) CUIK_TIMED_BLOCK("global iter") - { - TB_Node* bb = dyn_array_pop(ctx->worklist.items); - MachineBB* mbb = &nl_map_get_checked(seq_bb, bb); - - set_remove(&visited, bb->gvn); - - Set* restrict live_out = &mbb->live_out; - set_clear(live_out); - - // walk all successors - TB_Node* end = mbb->end_node; - if (end->type == TB_BRANCH) { - for (User* u = end->users; u; u = u->next) { - if (u->n->type == TB_PROJ) { - // union with successor's lives - TB_Node* succ = cfg_next_bb_after_cproj(u->n); - set_union(live_out, &nl_map_get_checked(seq_bb, succ).live_in); - } - } - } else { - // union with successor's lives - TB_Node* succ = cfg_next_control(end); - if (succ) set_union(live_out, &nl_map_get_checked(seq_bb, succ).live_in); - } - - Set* restrict live_in = &mbb->live_in; - Set* restrict kill = &mbb->kill; - Set* restrict gen = &mbb->gen; - - // live_in = (live_out - live_kill) U live_gen - bool changes = false; - FOREACH_N(i, 0, (interval_count + 63) / 64) { - uint64_t new_in = (live_out->data[i] & ~kill->data[i]) | gen->data[i]; - - changes |= (live_in->data[i] != new_in); - live_in->data[i] = new_in; - } - - // if we have changes, mark the predeccesors - if (changes && !(bb->type == TB_PROJ && bb->inputs[0]->type == TB_START)) { - FOREACH_N(i, 0, bb->input_count) { - TB_Node* pred = get_pred_cfg(&ctx->cfg, bb, i); - if (pred->input_count > 0 && !set_get(&visited, pred->gvn)) { - set_put(&visited, pred->gvn); - dyn_array_put(ctx->worklist.items, pred); - } - } - } - } - tb_arena_restore(arena, sp); - - /*if (reg_alloc_log) { - FOREACH_N(i, 0, ctx->bb_count) { - TB_Node* n = bbs[bb_order[i]]; - MachineBB* mbb = &nl_map_get_checked(seq_bb, n); - - bool in = set_get(&mbb->live_in, 33); - bool out = set_get(&mbb->live_out, 33); - printf(".bb%d [%d - %d]: %s %s\n", bb_order[i], mbb->start, mbb->end, in?"in":"", out?"out":""); - } - }*/ - - ctx->machine_bbs = seq_bb; - return epilogues; -} - -static ValueDesc* lookup_val(Ctx* restrict ctx, TB_Node* n) { - return worklist_test(&ctx->worklist, n) ? &ctx->values[n->gvn] : NULL; -} - -static void put_val(Ctx* restrict ctx, TB_Node* n, int src) { - ValueDesc* val = lookup_val(ctx, n); - if (val) val->vreg = src; -} - -// generated lazily to avoid allocating one for a node which is -// always folded. -static RegIndex input_reg(Ctx* restrict ctx, TB_Node* n) { - ValueDesc* val = lookup_val(ctx, n); - if (val == NULL) { - DO_IF(TB_OPTDEBUG_CODEGEN)(log_debug("%s: materialize on the spot for node %zu", ctx->f->super.name, n->gvn)); - int tmp = DEF(n, n->dt); - isel(ctx, n, tmp); - return tmp; - } - - val->uses -= 1; - - if (val->vreg >= 0) { - return val->vreg; - } else if (should_rematerialize(n)) { - DO_IF(TB_OPTDEBUG_CODEGEN)(log_debug("%s: materialize on the spot for node %zu", ctx->f->super.name, n->gvn)); - int tmp = DEF(n, n->dt); - isel(ctx, n, tmp); - return tmp; - } else { - int i = DEF(n, n->dt); - return (val->vreg = i); - } -} - -static void use(Ctx* restrict ctx, TB_Node* n) { - ValueDesc* v = lookup_val(ctx, n); - if (v != NULL) { v->uses -= 1; } -} - -static void fake_unuse(Ctx* restrict ctx, TB_Node* n) { - ValueDesc* v = lookup_val(ctx, n); - assert(v != NULL); - v->uses += 1; -} - -static bool on_last_use(Ctx* restrict ctx, TB_Node* n) { - ValueDesc* v = lookup_val(ctx, n); - return v ? v->uses == 1 : false; -} - -static bool has_users(Ctx* restrict ctx, TB_Node* n) { - if (n != NULL) { - ValueDesc* v = lookup_val(ctx, n); - return v ? v->vreg >= 0 || v->uses > 0 : false; - } - return false; -} - -static void isel_region(Ctx* restrict ctx, TB_Node* bb_start, TB_Node* end, size_t rpo_index) { - assert(dyn_array_length(ctx->worklist.items) == ctx->cfg.block_count); - TB_Scheduled scheduled = ctx->p->scheduled; - TB_BasicBlock* bb = nl_map_get_checked(scheduled, bb_start); - - // phase 1: logical schedule - DynArray(PhiVal) phi_vals = ctx->phi_vals; - CUIK_TIMED_BLOCK("phase 1") { - ctx->sched(ctx->p, &ctx->cfg, &ctx->worklist, &phi_vals, bb, end); - - // schedule params - if (rpo_index == 0) { - for (User* use = ctx->f->start_node->users; use; use = use->next) { - TB_Node* use_n = use->n; - if (use_n->type == TB_PROJ && !worklist_test_n_set(&ctx->worklist, use_n)) { - dyn_array_put(ctx->worklist.items, use_n); - } - } - } - } - - // phase 2: define all the nodes in this BB - CUIK_TIMED_BLOCK("phase 2") { - FOREACH_N(i, ctx->cfg.block_count, dyn_array_length(ctx->worklist.items)) { - TB_Node* n = ctx->worklist.items[i]; - - // track non-dead users - size_t use_count = 0; - for (User* use = n->users; use; use = use->next) { - if (nl_map_get(scheduled, use->n) >= 0) use_count++; - } - - // we don't have to worry about resizing here which is really nice - ctx->values[n->gvn].uses = use_count; - ctx->values[n->gvn].vreg = -1; - } - } - - // phase 3: within the BB, the phi nodes should view itself as the previous value - // not the new one we're producing. - size_t our_phis = ctx->our_phis = dyn_array_length(phi_vals); - CUIK_TIMED_BLOCK("phase 3") { - FOREACH_N(i, 0, our_phis) { - PhiVal* v = &phi_vals[i]; - - // mark the proper output, especially before we make the BB-local ones - v->dst = input_reg(ctx, v->phi); - } - - if (bb_start->type == TB_REGION) { - for (User* use = find_users(ctx->p, bb_start); use; use = use->next) { - if (use->n->type == TB_PHI && use->n->dt.type != TB_MEMORY) { - ValueDesc* val = &ctx->values[use->n->gvn]; - - // copy PHI into temporary - PhiVal p = { .phi = use->n, .dst = input_reg(ctx, use->n) }; - dyn_array_put(phi_vals, p); - - TB_DataType dt = p.phi->dt; - int tmp = DEF(NULL, dt); - SUBMIT(inst_move(dt, tmp, p.dst)); - - // assign temporary as the PHI until the end of the BB - val->vreg = tmp; - } - } - } - - if (rpo_index == 0) { - isel(ctx, ctx->f->start_node, -1); - } - } - - // phase 4: walk all nodes (we're allowed to fold nodes into those which appear later) - // - // isel is emitting start->end but we're iterating in reverse order so we need - // to reverse the instruction stream as we go, it's a linked list so it's not - // hard. - int bbid = nl_map_get_checked(ctx->cfg.node_to_block, bb_start).id; - TB_OPTDEBUG(CODEGEN)(printf("BB %d\n", bbid)); - - CUIK_TIMED_BLOCK("phase 4") { - Inst* head = ctx->head; - Inst* last = NULL; - TB_Node* prev_loc = NULL; - FOREACH_REVERSE_N(i, ctx->cfg.block_count, dyn_array_length(ctx->worklist.items)) { - TB_Node* n = ctx->worklist.items[i]; - if (n->type == TB_START) { - continue; - } - - ValueDesc* val = lookup_val(ctx, n); - - if (n != end && val->vreg < 0 && should_rematerialize(n)) { - DO_IF(TB_OPTDEBUG_CODEGEN)( - printf(" DISCARD %u: ", n->gvn), - print_node_sexpr(n, 0), - printf("\n") - ); - continue; - } - - // attach to dummy list - Inst dummy; - dummy.next = NULL; - ctx->head = &dummy; - - if (n->type == TB_SAFEPOINT_NOP) { - TB_OPTDEBUG(CODEGEN)( - printf(" LINE %u: ", n->gvn), - print_node_sexpr(n, 0), - printf("\n") - ); - - SUBMIT(inst_line(n)); - } else if (n->type != TB_MULPAIR && (n->dt.type == TB_TUPLE || n->dt.type == TB_CONTROL || n->dt.type == TB_MEMORY)) { - TB_OPTDEBUG(CODEGEN)( - printf(" EFFECT %u: ", n->gvn), - print_node_sexpr(n, 0), - printf("\n") - ); - - if (n->type == TB_BRANCH) { - assert(ctx->our_phis == 0 && "branches don't get phi edges, they should've been split"); - } - - isel(ctx, n, val->vreg); - } else if (val->uses > 0 || val->vreg >= 0) { - if (val->vreg < 0) { - val->vreg = DEF(n, n->dt); - } - - TB_OPTDEBUG(CODEGEN)( - printf(" DATA %u: ", n->gvn), - print_node_sexpr(n, 0), - printf(" (write v%u)\n", val->vreg) - ); - - isel(ctx, n, val->vreg); - } else { - TB_OPTDEBUG(CODEGEN)( - printf(" DEAD %u: ", n->gvn), - print_node_sexpr(n, 0), - printf("\n") - ); - } - - Inst* seq_start = dummy.next; - Inst* seq_end = ctx->head; - assert(seq_end->next == NULL); - - if (seq_start != NULL) { - if (last == NULL) { - last = seq_end; - head->next = dummy.next; - } else { - Inst* old_next = head->next; - head->next = seq_start; - seq_end->next = old_next; - } - } - } - - // restore the PHI value to normal - FOREACH_N(i, our_phis, dyn_array_length(phi_vals)) { - PhiVal* v = &phi_vals[i]; - lookup_val(ctx, v->phi)->vreg = v->dst; - } - - ctx->head = last ? last : head; - - if (!cfg_is_terminator(end)) { - TB_OPTDEBUG(CODEGEN)( - printf(" TERMINATOR %u: ", end->gvn), - print_node_sexpr(end, 0), - printf("\n") - ); - - if (end->type == TB_SAFEPOINT_NOP) { - SUBMIT(inst_line(end)); - } - - // implicit goto - TB_Node* succ_n = cfg_next_control(end); - - SUBMIT(alloc_inst(INST_TERMINATOR, TB_TYPE_VOID, 0, 0, 0)); - - // writeback phis - FOREACH_N(i, 0, ctx->our_phis) { - PhiVal* v = &phi_vals[i]; - - TB_DataType dt = v->phi->dt; - int src = input_reg(ctx, v->n); - - TB_OPTDEBUG(CODEGEN)( - printf(" PHI %u: ", v->phi->gvn), - print_node_sexpr(v->phi, 0), - printf(" (write v%u)\n", v->dst) - ); - - hint_reg(ctx, v->dst, src); - SUBMIT(inst_move(dt, v->dst, src)); - } - - int succ = nl_map_get_checked(ctx->cfg.node_to_block, succ_n).id; - if (ctx->fallthrough != succ) { - SUBMIT(inst_jmp(succ)); - } - } - - dyn_array_clear(phi_vals); - ctx->phi_vals = phi_vals; - } - - dyn_array_set_length(ctx->worklist.items, ctx->cfg.block_count); -} - -// Codegen through here is done in phases -static void compile_function(TB_Passes* restrict p, TB_FunctionOutput* restrict func_out, const TB_FeatureSet* features, uint8_t* out, size_t out_capacity, bool emit_asm) { - verify_tmp_arena(p); - - TB_Function* restrict f = p->f; - DO_IF(TB_OPTDEBUG_PEEP)(log_debug("%s: starting codegen with %d nodes", f->super.name, f->node_count)); - - #if 0 - if (!strcmp(f->super.name, "SDL_main")) { - reg_alloc_log = true; - tb_pass_print(p); - } else { - reg_alloc_log = false; - } - #endif - - Ctx ctx = { - .module = f->super.module, - .f = f, - .p = p, - .target_abi = f->super.module->target_abi, - .emit = { - .f = f, - .output = func_out, - .data = out, - .capacity = out_capacity, - } - }; - - CUIK_TIMED_BLOCK("init regalloc") { - init_regalloc(&ctx); - } - - worklist_clear(&p->worklist); - ctx.values = tb_arena_alloc(tmp_arena, f->node_count * sizeof(ValueDesc)); - - // We need to generate a CFG - ctx.cfg = tb_compute_rpo(f, p); - - // And perform global scheduling - tb_pass_schedule(p, ctx.cfg); - ctx.sched = greedy_scheduler; - ctx.worklist = p->worklist; - - // allocate more stuff now that we've run stats on the IR - ctx.emit.label_count = ctx.cfg.block_count; - ctx.emit.labels = tb_arena_alloc(tmp_arena, ctx.cfg.block_count * sizeof(uint32_t)); - - nl_map_create(ctx.stack_slots, 8); - dyn_array_create(ctx.debug_stack_slots, 8); - - worklist_clear_visited(&ctx.worklist); - - // Instruction selection: - // we just decide which instructions to emit, which operands are - // fixed and which need allocation. For now regalloc is handled - // immediately but in theory it could be delayed until all selection - // is done. - ctx.bb_count = 0; - int* bb_order = ctx.bb_order = tb_arena_alloc(tmp_arena, ctx.cfg.block_count * sizeof(int)); - - CUIK_TIMED_BLOCK("isel") { - assert(dyn_array_length(ctx.worklist.items) == ctx.cfg.block_count); - - // define all PHIs early and sort BB order - int stop_bb = -1; - FOREACH_N(i, 0, ctx.cfg.block_count) { - TB_Node* bb = ctx.worklist.items[i]; - - for (User* use = find_users(p, bb); use; use = use->next) { - TB_Node* n = use->n; - if (n->type == TB_PHI && n->dt.type != TB_MEMORY) { - worklist_test_n_set(&ctx.worklist, n); - ctx.values[n->gvn].uses = INT_MAX; - ctx.values[n->gvn].vreg = -1; - } - } - - TB_Node* end = nl_map_get_checked(ctx.cfg.node_to_block, bb).end; - if (end->type == TB_END) { - stop_bb = i; - } else { - bb_order[ctx.bb_count++] = i; - } - } - - // enter END block at the... end - if (stop_bb >= 0) { - bb_order[ctx.bb_count++] = stop_bb; - } - - TB_Node** bbs = ctx.worklist.items; - FOREACH_N(i, 0, ctx.bb_count) { - TB_Node* bb = bbs[bb_order[i]]; - - ctx.emit.labels[bb_order[i]] = 0; - if (i + 1 < ctx.bb_count) { - ctx.fallthrough = bb_order[i + 1]; - /*if (i + 2 < ctx.bb_count && cfg_basically_empty_only_mem_phis(bbs[ctx.fallthrough])) { - ctx.fallthrough = bb_order[i + 2]; - }*/ - } else { - ctx.fallthrough = INT_MAX; - } - - Inst* label = inst_label(bb); - if (ctx.first == NULL) { - ctx.first = ctx.head = label; - } else { - append_inst(&ctx, label); - } - - TB_Node* end = nl_map_get_checked(ctx.cfg.node_to_block, bb).end; - isel_region(&ctx, bb, end, i); - } - } - p->worklist = ctx.worklist; - - { - DynArray(int) end; - CUIK_TIMED_BLOCK("data flow") { - end = liveness(&ctx, f); - } - - // we can in theory have other regalloc solutions and eventually will put - // graph coloring here. - ctx.stack_usage = linear_scan(&ctx, f, ctx.stack_usage, end); - - // Arch-specific: convert instruction buffer into actual instructions - CUIK_TIMED_BLOCK("emit code") { - emit_code(&ctx, func_out); - } - - // Fill jump table entries - CUIK_TIMED_BLOCK("jump tables") { - dyn_array_for(i, ctx.jump_table_patches) { - uint32_t target = ctx.emit.labels[ctx.jump_table_patches[i].target]; - assert((target & 0x80000000) && "target label wasn't resolved... what?"); - *ctx.jump_table_patches[i].pos = target & ~0x80000000; - } - } - } - - if (emit_asm) CUIK_TIMED_BLOCK("dissassembly") { - EMITA(&ctx.emit, "%s:\n", f->super.name); - - Disasm d = { func_out->first_patch, ctx.locations, &ctx.locations[dyn_array_length(ctx.locations)] }; - - // dump prologue - disassemble(&ctx.emit, &d, -1, 0, func_out->prologue_length); - - TB_Node** bbs = ctx.worklist.items; - FOREACH_N(i, 0, ctx.bb_count) { - TB_Node* bb = bbs[bb_order[i]]; - - uint32_t start = ctx.emit.labels[bb_order[i]] & ~0x80000000; - uint32_t end = ctx.emit.count; - if (i + 1 < ctx.bb_count) { - end = ctx.emit.labels[bb_order[i + 1]] & ~0x80000000; - } - - disassemble(&ctx.emit, &d, bb_order[i], start, end); - } - } - - tb_free_cfg(&ctx.cfg); - nl_map_free(ctx.machine_bbs); - dyn_array_destroy(ctx.jump_table_patches); - dyn_array_destroy(ctx.intervals); - dyn_array_destroy(ctx.phi_vals); - - if (dyn_array_length(ctx.locations)) { - ctx.locations[0].pos = 0; - } - - // we're done, clean up - func_out->asm_out = ctx.emit.head_asm; - func_out->code = ctx.emit.data; - func_out->code_size = ctx.emit.count; - func_out->stack_usage = ctx.stack_usage; - func_out->locations = ctx.locations; - func_out->stack_slots = ctx.debug_stack_slots; - nl_map_free(ctx.stack_slots); -} - -static void get_data_type_size(TB_DataType dt, size_t* out_size, size_t* out_align) { - switch (dt.type) { - case TB_INT: { - // above 64bits we really dont care that much about natural alignment - bool is_big_int = dt.data > 64; - - // round up bits to a byte - int bits = is_big_int ? ((dt.data + 7) / 8) : tb_next_pow2(dt.data - 1); - - *out_size = ((bits+7) / 8); - *out_align = is_big_int ? 8 : ((dt.data + 7) / 8); - break; - } - case TB_FLOAT: { - int s = 0; - if (dt.data == TB_FLT_32) s = 4; - else if (dt.data == TB_FLT_64) s = 8; - else tb_unreachable(); - - *out_size = s; - *out_align = s; - break; - } - case TB_PTR: { - *out_size = 8; - *out_align = 8; - break; - } - default: tb_unreachable(); - } -} diff --git a/vendor/tb/src/hash.c b/vendor/tb/src/hash.c deleted file mode 100644 index 0cbd804b..00000000 --- a/vendor/tb/src/hash.c +++ /dev/null @@ -1,112 +0,0 @@ -#include "tb_internal.h" - -// leftrotate function definition -#define LEFTROTATE(x, c) (((x) << (c)) | ((x) >> (32 - (c)))) - -void tb__md5sum(uint8_t* out_bytes, uint8_t* initial_msg, size_t initial_len) { - // Message (to prepare) - uint8_t* msg = NULL; - - // Note: All variables are unsigned 32 bit and wrap modulo 2^32 when - // calculating - - // r specifies the per-round shift amounts - uint32_t r[] = { - 7, 12, 17, 22, 7, 12, 17, 22, 7, 12, 17, 22, 7, 12, 17, - 22, 5, 9, 14, 20, 5, 9, 14, 20, 5, 9, 14, 20, 5, 9, 14, - 20, 4, 11, 16, 23, 4, 11, 16, 23, 4, 11, 16, 23, 4, 11, 16, - 23, 6, 10, 15, 21, 6, 10, 15, 21, 6, 10, 15, 21, 6, 10, 15, - 21 - }; - - // Use binary integer part of the sines of integers (in radians) as - // constants// Initialize variables: - uint32_t k[] = { - 0xd76aa478, 0xe8c7b756, 0x242070db, 0xc1bdceee, 0xf57c0faf, 0x4787c62a, 0xa8304613, - 0xfd469501, 0x698098d8, 0x8b44f7af, 0xffff5bb1, 0x895cd7be, 0x6b901122, 0xfd987193, - 0xa679438e, 0x49b40821, 0xf61e2562, 0xc040b340, 0x265e5a51, 0xe9b6c7aa, 0xd62f105d, - 0x02441453, 0xd8a1e681, 0xe7d3fbc8, 0x21e1cde6, 0xc33707d6, 0xf4d50d87, 0x455a14ed, - 0xa9e3e905, 0xfcefa3f8, 0x676f02d9, 0x8d2a4c8a, 0xfffa3942, 0x8771f681, 0x6d9d6122, - 0xfde5380c, 0xa4beea44, 0x4bdecfa9, 0xf6bb4b60, 0xbebfbc70, 0x289b7ec6, 0xeaa127fa, - 0xd4ef3085, 0x04881d05, 0xd9d4d039, 0xe6db99e5, 0x1fa27cf8, 0xc4ac5665, 0xf4292244, - 0x432aff97, 0xab9423a7, 0xfc93a039, 0x655b59c3, 0x8f0ccc92, 0xffeff47d, 0x85845dd1, - 0x6fa87e4f, 0xfe2ce6e0, 0xa3014314, 0x4e0811a1, 0xf7537e82, 0xbd3af235, 0x2ad7d2bb, - 0xeb86d391 - }; - - uint32_t h0 = 0x67452301, h1 = 0xefcdab89, h2 = 0x98badcfe, h3 = 0x10325476; - - // Pre-processing: adding a single 1 bit - // append "1" bit to message - /* Notice: the input bytes are considered as bits strings, - where the first bit is the most significant bit of the byte.[37] */ - - // Pre-processing: padding with zeros - // append "0" bit until message length in bit ≡ 448 (mod 512) - // append length mod (2 pow 64) to message - - int new_len = ((((initial_len + 8) / 64) + 1) * 64) - 8; - - msg = calloc(new_len + 64, 1); // also appends "0" bits - // (we alloc also 64 extra bytes...) - memcpy(msg, initial_msg, initial_len); - msg[initial_len] = 128; // write the "1" bit - - uint32_t bits_len = 8 * initial_len; // note, we append the len - memcpy(msg + new_len, &bits_len, 4); // in bits at the end of the buffer - - // Process the message in successive 512-bit chunks: - // for each 512-bit chunk of message: - for (int offset = 0; offset < new_len; offset += (512 / 8)) { - // break chunk into sixteen 32-bit words w[j], 0 ≤ j ≤ 15 - uint32_t* w = (uint32_t*)(msg + offset); - - // Initialize hash value for this chunk: - uint32_t a = h0; - uint32_t b = h1; - uint32_t c = h2; - uint32_t d = h3; - - // Main loop: - uint32_t i; - for (i = 0; i < 64; i++) { - uint32_t f, g; - if (i < 16) { - f = (b & c) | ((~b) & d); - g = i; - } else if (i < 32) { - f = (d & b) | ((~d) & c); - g = (5 * i + 1) % 16; - } else if (i < 48) { - f = b ^ c ^ d; - g = (3 * i + 5) % 16; - } else { - f = c ^ (b | (~d)); - g = (7 * i) % 16; - } - - uint32_t temp = d; - d = c; - c = b; - - uint32_t x = a + f + k[i] + w[g]; - b += (x << r[i]) | (x >> (32 - r[i])); - a = temp; - } - - // Add this chunk's hash to result so far: - h0 += a; - h1 += b; - h2 += c; - h3 += d; - } - - // TODO(NeGate): refactor this bullshit - memcpy(out_bytes, &h0, 4); - memcpy(out_bytes+4, &h1, 4); - memcpy(out_bytes+8, &h2, 4); - memcpy(out_bytes+12, &h3, 4); - - // cleanup - free(msg); -} diff --git a/vendor/tb/src/host.h b/vendor/tb/src/host.h deleted file mode 100644 index ab25f02c..00000000 --- a/vendor/tb/src/host.h +++ /dev/null @@ -1,19 +0,0 @@ -// architecture -#if defined(__x86_64__) || defined(_M_X64) || defined(_M_AMD64) -#define TB_HOST_X86_64 1 -#elif defined(i386) || defined(__i386) || defined(__i386__) -#define TB_HOST_X86 2 -#elif defined(__aarch64__) -#define TB_HOST_ARM64 3 -#elif defined(__arm__) -#define TB_HOST_ARM32 4 -#endif - -// system -#if defined(__APPLE__) && defined(__MACH__) -#define TB_HOST_OSX 1 -#elif defined(__gnu_linux__) || defined(__linux__) -#define TB_HOST_LINUX 2 -#elif defined(_WIN32) -#define TB_HOST_WINDOWS 3 -#endif diff --git a/vendor/tb/src/ir_printer.c b/vendor/tb/src/ir_printer.c deleted file mode 100644 index d0fba79f..00000000 --- a/vendor/tb/src/ir_printer.c +++ /dev/null @@ -1,373 +0,0 @@ -#include "tb_internal.h" -#include - -TB_API void tb_default_print_callback(void* user_data, const char* fmt, ...) { - va_list ap; - va_start(ap, fmt); - vfprintf((FILE*)user_data, fmt, ap); - va_end(ap); -} - -TB_API const char* tb_node_get_name(TB_Node* n) { - switch (n->type) { - case TB_NULL: return "BAD"; - case TB_UNREACHABLE: return "unreachable"; - - case TB_ROOT: return "root"; - case TB_RETURN: return "return"; - case TB_PROJ: return "proj"; - case TB_REGION: return "region"; - case TB_NATURAL_LOOP: return "loop"; - case TB_AFFINE_LOOP: return "loop.affine"; - case TB_CALLGRAPH: return "callgraph"; - - case TB_LOCAL: return "local"; - - case TB_VA_START: return "vastart"; - case TB_DEBUGBREAK: return "dbgbrk"; - - case TB_POISON: return "poison"; - case TB_DEAD: return "dead"; - case TB_INTEGER_CONST: return "int"; - case TB_FLOAT32_CONST: return "float32"; - case TB_FLOAT64_CONST: return "float64"; - - case TB_PHI: return "phi"; - case TB_SELECT: return "select"; - case TB_LOOKUP: return "lookup"; - - case TB_ARRAY_ACCESS: return "array"; - case TB_MEMBER_ACCESS: return "member"; - - case TB_CYCLE_COUNTER: return "cyclecnt"; - case TB_SAFEPOINT_POLL: return "safepoint.poll"; - - case TB_MEMSET: return "memset"; - case TB_MEMCPY: return "memcpy"; - case TB_SPLITMEM: return "split"; - case TB_MERGEMEM: return "merge"; - - case TB_ZERO_EXT: return "zxt"; - case TB_SIGN_EXT: return "sxt"; - case TB_FLOAT_EXT: return "fpxt"; - case TB_TRUNCATE: return "trunc"; - case TB_BITCAST: return "bitcast"; - case TB_UINT2FLOAT: return "uint2float"; - case TB_INT2FLOAT: return "int2float"; - case TB_FLOAT2UINT: return "float2uint"; - case TB_FLOAT2INT: return "float2int"; - case TB_SYMBOL: return "symbol"; - - case TB_CMP_NE: return "cmp.ne"; - case TB_CMP_EQ: return "cmp.eq"; - case TB_CMP_ULT: return "cmp.ult"; - case TB_CMP_ULE: return "cmp.ule"; - case TB_CMP_SLT: return "cmp.slt"; - case TB_CMP_SLE: return "cmp.sle"; - case TB_CMP_FLT: return "cmp.lt"; - case TB_CMP_FLE: return "cmp.le"; - - case TB_ATOMIC_LOAD: return "atomic.load"; - case TB_ATOMIC_XCHG: return "atomic.xchg"; - case TB_ATOMIC_ADD: return "atomic.add"; - case TB_ATOMIC_SUB: return "atomic.sub"; - case TB_ATOMIC_AND: return "atomic.and"; - case TB_ATOMIC_XOR: return "atomic.xor"; - case TB_ATOMIC_OR: return "atomic.or"; - case TB_ATOMIC_CAS: return "atomic.cas"; - - case TB_CLZ: return "clz"; - case TB_CTZ: return "ctz"; - case TB_NEG: return "neg"; - case TB_NOT: return "not"; - case TB_AND: return "and"; - case TB_OR: return "or"; - case TB_XOR: return "xor"; - case TB_ADD: return "add"; - case TB_SUB: return "sub"; - case TB_MUL: return "mul"; - case TB_UDIV: return "udiv"; - case TB_SDIV: return "sdiv"; - case TB_UMOD: return "umod"; - case TB_SMOD: return "smod"; - case TB_SHL: return "shl"; - case TB_SHR: return "shr"; - case TB_ROL: return "rol"; - case TB_ROR: return "ror"; - case TB_SAR: return "sar"; - case TB_ADC: return "adc"; - - case TB_FADD: return "fadd"; - case TB_FSUB: return "fsub"; - case TB_FMUL: return "fmul"; - case TB_FDIV: return "fdiv"; - case TB_FMAX: return "fmax"; - case TB_FMIN: return "fmin"; - - case TB_MULPAIR: return "mulpair"; - case TB_LOAD: return "load"; - case TB_STORE: return "store"; - case TB_READ: return "read"; - case TB_WRITE: return "write"; - - case TB_CALL: return "call"; - case TB_SYSCALL: return "syscall"; - case TB_BRANCH: return "branch"; - case TB_TAILCALL: return "tailcall"; - - default: tb_todo();return "(unknown)"; - } -} - -#define P(...) callback(user_data, __VA_ARGS__) -static void tb_print_type(TB_DataType dt, TB_PrintCallback callback, void* user_data) { - switch (dt.type) { - case TB_INT: { - if (dt.data == 0) P("void"); - else P("i%d", dt.data); - break; - } - case TB_PTR: { - if (dt.data == 0) P("ptr"); - else P("ptr%d", dt.data); - break; - } - case TB_FLOAT: { - if (dt.data == TB_FLT_32) P("f32"); - if (dt.data == TB_FLT_64) P("f64"); - break; - } - case TB_TUPLE: { - P("tuple"); - break; - } - case TB_MEMORY: { - P("memory"); - break; - } - case TB_CONTROL: { - P("control"); - break; - } - default: tb_todo(); - } -} - -static void print_proj(TB_PrintCallback callback, void* user_data, TB_Node* n, int index) { - switch (n->type) { - case TB_ROOT: { - if (index == 0) { - P("ctrl"); - } else if (index == 1) { - P("mem"); - } else if (index == 2) { - P("rpc"); - } else { - P("%c", 'a'+(index - 3)); - } - break; - } - - case TB_BRANCH: { - TB_NodeBranch* br = TB_NODE_GET_EXTRA(n); - if (br->succ_count == 2 && br->keys[0].key == 0) { - P("%s", index ? "false" : "true"); - } else if (index == 0) { - P("default"); - } else { - P("%"PRId64, br->keys[index - 1]); - } - break; - } - - case TB_TAILCALL: - if (index == 0) { - P("ctrl"); - } else if (index == 1) { - P("mem"); - } else if (index == 2) { - P("rpc"); - } else { - P("val"); - } - break; - - case TB_CALL: - if (index == 0) { - P("ctrl"); - } else if (index == 1) { - P("mem"); - } else { - P("val"); - } - break; - - case TB_SPLITMEM: - P("mem"); - break; - - default: tb_todo(); - } -} - -static void print_graph_node(TB_Function* f, TB_PrintCallback callback, void* user_data, size_t bb, TB_Node* restrict n) { - P(" r%u [label=\"{", n->gvn); - - bool ins = false; - FOREACH_N(i, 0, n->input_count) if (n->inputs[i]) { - if (!ins) P("{"); - else P("|"); - - P(" %zu", i, i); - ins = true; - } - - if (ins) { - P("}|"); - } - - if (n->dt.type == TB_TUPLE) { - TB_Node* projs[128] = { 0 }; - int limit = 0; - FOR_USERS(use, n) { - if (use->n->type == TB_PROJ) { - int index = TB_NODE_GET_EXTRA_T(use->n, TB_NodeProj)->index; - if (limit < index+1) limit = index+1; - - projs[index] = use->n; - } - } - - P("%s|{", tb_node_get_name(n)); - int outs = 0; - FOREACH_N(i, 0, limit) { - if (projs[i] != NULL) { - if (outs) P("|"); - - P("", i); - print_proj(callback, user_data, n, i); - outs++; - } - } - P("}}\"]"); - } else { - switch (n->type) { - case TB_REGION: P("region"); break; - - case TB_LOAD: { - P("ld."); - tb_print_type(n->dt, callback, user_data); - break; - } - case TB_STORE: { - P("st."); - tb_print_type(n->inputs[2]->dt, callback, user_data); - break; - } - - case TB_SYMBOL: { - TB_Symbol* sym = TB_NODE_GET_EXTRA_T(n, TB_NodeSymbol)->sym; - if (sym->name[0]) { - P("sym: %s", sym->name); - } else { - P("sym: %p", sym); - } - break; - } - - case TB_BITCAST: { - P("bitcast "); - tb_print_type(n->inputs[1]->dt, callback, user_data); - P(" to "); - tb_print_type(n->dt, callback, user_data); - break; - } - - case TB_INTEGER_CONST: { - TB_NodeInt* num = TB_NODE_GET_EXTRA(n); - if (num->value < 0xFFFF) { - P("cst: %"PRId64, num->value); - } else { - P("cst: %#0"PRIx64, num->value); - } - break; - } - - case TB_ARRAY_ACCESS: { - int64_t stride = TB_NODE_GET_EXTRA_T(n, TB_NodeArray)->stride; - P("*%td", stride); - break; - } - - case TB_MEMBER_ACCESS: { - int64_t offset = TB_NODE_GET_EXTRA_T(n, TB_NodeMember)->offset; - P("+%td", offset); - break; - } - - default: - P("%s", tb_node_get_name(n)); - break; - } - P("}\"]"); - } - - FOREACH_N(i, 0, n->input_count) if (n->inputs[i]) { - TB_Node* in = n->inputs[i]; - - const char* color = "black"; - if (in->dt.type == TB_CONTROL) { - color = "red"; - } else if (in->dt.type == TB_MEMORY) { - color = "blue"; - } - - if (in->type == TB_PROJ) { - int index = TB_NODE_GET_EXTRA_T(in, TB_NodeProj)->index; - - P("; r%u:p%d -> r%u:i%zu [color=%s]", in->inputs[0]->gvn, index, n->gvn, i, color); - } else { - P("; r%u -> r%u:i%zu [color=%s]", in->gvn, n->gvn, i, color); - } - } - P("\n"); -} - -TB_API void tb_pass_print_dot(TB_Passes* opt, TB_PrintCallback callback, void* user_data) { - TB_Function* f = opt->f; - Worklist old = opt->worklist; - Worklist tmp_ws = { 0 }; - worklist_alloc(&tmp_ws, f->node_count); - - P("digraph %s {\n node [ordering=in; shape=record];\n", f->super.name ? f->super.name : "unnamed"); - - opt->worklist = tmp_ws; - - Worklist* ws = &opt->worklist; - worklist_clear_visited(ws); - - worklist_test_n_set(ws, f->root_node); - dyn_array_put(ws->items, f->root_node); - - for (size_t i = 0; i < dyn_array_length(ws->items); i++) { - TB_Node* n = ws->items[i]; - - bool has_users = true; - FOR_USERS(u, n) { - TB_Node* out = u->n; - if (!worklist_test_n_set(ws, out)) { - dyn_array_put(ws->items, out); - } - has_users = true; - } - - if (has_users && n->type != TB_PROJ) { - print_graph_node(f, callback, user_data, i, n); - } - } - - worklist_free(ws); - opt->worklist = old; - - P("}\n"); -} diff --git a/vendor/tb/src/jit.c b/vendor/tb/src/jit.c deleted file mode 100644 index fc457c91..00000000 --- a/vendor/tb/src/jit.c +++ /dev/null @@ -1,584 +0,0 @@ -#include "tb_internal.h" -#include "host.h" -#include - -#ifdef _WIN32 -#define WIN32_LEAN_AND_MEAN -#include -#endif - -enum { - ALLOC_COOKIE = 0xBAADF00D, - ALLOC_GRANULARITY = 16, - STACK_SIZE = 2*1024*1024 -}; - -typedef struct { - uint8_t* pos; - - // when we insert a breakpoint, we're - // replacing some byte with an INT3 (at - // least on x86), we need to restore it - // before continuing execution. - uint8_t prev_byte; -} TB_Breakpoint; - -typedef struct { - uint32_t cookie; - uint32_t size; // if low bit is set, we're in use. - char data[]; -} FreeList; - -// addr -> symbol -typedef struct { - uint32_t k; - void* v; -} Tag; - -struct TB_JIT { - size_t capacity; - mtx_t lock; - NL_Strmap(void*) loaded_funcs; - - DynArray(TB_Breakpoint) breakpoints; - DynArray(Tag) tags; - - FreeList heap; -}; - -static const char* prot_names[] = { - "RO", "RW", "RX", "RXW", -}; - -void tb_jit_tag_object(TB_JIT* jit, void* ptr, void* tag) { - assert(tag); - uint32_t offset = (char*) ptr - (char*) jit; - - size_t i = 0, count = dyn_array_length(jit->tags); - for (; i < count; i++) { - if (offset < jit->tags[i].k) break; - } - - // we know where to insert - dyn_array_put_uninit(jit->tags, 1); - memmove(&jit->tags[i + 1], &jit->tags[i], (count - i) * sizeof(Tag)); - jit->tags[i] = (Tag){ offset, tag }; -} - -void* tb_jit_resolve_addr(TB_JIT* jit, void* ptr, uint32_t* out_offset) { - mtx_lock(&jit->lock); - ptrdiff_t offset = (char*) ptr - (char*) jit; - if (offset < 0 || offset >= jit->capacity) goto bad; - - size_t left = 0; - size_t right = dyn_array_length(jit->tags); - if (right == 0) goto bad; - - Tag* tags = jit->tags; - while (left < right) { - size_t middle = (left + right) / 2; - if (tags[middle].k > offset) { - right = middle; - } else { - left = middle + 1; - } - } - - size_t i = right - 1; - *out_offset = offset - tags[i].k; - return tags[i].v; - - bad: - mtx_unlock(&jit->lock); - return NULL; -} - -void* tb_jit_alloc_obj(TB_JIT* jit, size_t size, size_t align) { - mtx_lock(&jit->lock); - size = (size + ALLOC_GRANULARITY - 1) & ~(ALLOC_GRANULARITY - 1); - - FreeList* l = &jit->heap; - FreeList* end = (FreeList*) ((char*) jit + jit->capacity); - for (;;) { - assert(l->cookie == ALLOC_COOKIE); - - // free slot, let's see if it's big enough - if ((l->size & 1) == 0 && (l->size >> 1) >= size) { - // if the node is bigger than one alloc, we need to split it - size_t whole_size = l->size >> 1; - size_t split_size = sizeof(FreeList) + size; - if (whole_size > split_size) { - // split - l->size = (size << 1) | 1; - - // free leftovers - FreeList* rest = (FreeList*) &l->data[l->size >> 1]; - rest->cookie = ALLOC_COOKIE; - rest->size = (whole_size - split_size) << 1; - } else { - // take entire piece - l->size |= 1; - } - - mtx_unlock(&jit->lock); - return l->data; - } - - FreeList* next = (FreeList*) &l->data[l->size >> 1]; - if (next == end) { - break; - } - - // coalesce free regions when we see them here, it'll - // make later walks faster - while ((l->size & 1) == 0 && (next->size & 1) == 0) { - size_t new_size = (l->size >> 1) + sizeof(FreeList) + (next->size >> 1); - l->size = (new_size << 1); - - next = (FreeList*) &l->data[new_size]; - } - - l = next; - } - - mtx_unlock(&jit->lock); - return NULL; -} - -void tb_jit_free_obj(TB_JIT* jit, void* ptr) { - FreeList* obj = ((FreeList*) ptr) - 1; - assert(obj->cookie == ALLOC_COOKIE); - obj->size &= ~1; -} - -void tb_jit_dump_heap(TB_JIT* jit) { - mtx_lock(&jit->lock); - - FreeList* l = &jit->heap; - FreeList* next; - FreeList* end = (FreeList*) ((char*) jit + jit->capacity); - - printf("HEAP:\n"); - size_t tag = 0, tag_count = dyn_array_length(jit->tags); - char* base = (char*) jit; - for (;;) { - if (l->size & 1) { - printf("* ALLOC [%p %u", l->data, l->size >> 1); - - // found the next tag here - if (tag < tag_count && l->data == &base[jit->tags[tag].k]) { - printf(" TAG=%p", jit->tags[tag].v); - tag += 1; - } - - printf("]\n"); - } - - next = (FreeList*) &l->data[l->size >> 1]; - if (next == end) { - break; - } - l = next; - } - mtx_unlock(&jit->lock); -} - -static void* get_proc(TB_JIT* jit, const char* name) { - #ifdef _WIN32 - static HMODULE kernel32, user32, gdi32, opengl32, msvcrt; - if (user32 == NULL) { - kernel32 = LoadLibrary("kernel32.dll"); - user32 = LoadLibrary("user32.dll"); - gdi32 = LoadLibrary("gdi32.dll"); - opengl32 = LoadLibrary("opengl32.dll"); - msvcrt = LoadLibrary("msvcrt.dll"); - } - - // check cache first - ptrdiff_t search = nl_map_get_cstr(jit->loaded_funcs, name); - if (search >= 0) return jit->loaded_funcs[search].v; - - void* addr = GetProcAddress(NULL, name); - if (addr == NULL) addr = GetProcAddress(kernel32, name); - if (addr == NULL) addr = GetProcAddress(user32, name); - if (addr == NULL) addr = GetProcAddress(gdi32, name); - if (addr == NULL) addr = GetProcAddress(opengl32, name); - if (addr == NULL) addr = GetProcAddress(msvcrt, name); - - // printf("JIT: loaded %s (%p)\n", name, addr); - nl_map_put_cstr(jit->loaded_funcs, name, addr); - return addr; - #else - return NULL; - #endif -} - -static void* get_symbol_address(const TB_Symbol* s) { - if (s->tag == TB_SYMBOL_GLOBAL) { - return ((TB_Global*) s)->address; - } else if (s->tag == TB_SYMBOL_FUNCTION) { - return ((TB_Function*) s)->compiled_pos; - } else { - tb_todo(); - } -} - -void* tb_jit_place_function(TB_JIT* jit, TB_Function* f) { - TB_FunctionOutput* func_out = f->output; - if (f->compiled_pos != NULL) { - return f->compiled_pos; - } - - // copy machine code - char* dst = tb_jit_alloc_obj(jit, func_out->code_size, 16); - memcpy(dst, func_out->code, func_out->code_size); - f->compiled_pos = dst; - - log_debug("jit: apply function %s (%p)", f->super.name, dst); - - // apply relocations, any leftovers are mapped to thunks - for (TB_SymbolPatch* p = func_out->first_patch; p; p = p->next) { - size_t actual_pos = p->pos; - TB_SymbolTag tag = p->target->tag; - - int32_t* patch = (int32_t*) &dst[actual_pos]; - if (tag == TB_SYMBOL_FUNCTION) { - TB_Function* f = (TB_Function*) p->target; - void* addr = tb_jit_place_function(jit, f); - - int32_t rel32 = (intptr_t)addr - ((intptr_t)patch + 4); - *patch += rel32; - } else if (tag == TB_SYMBOL_EXTERNAL) { - TB_External* e = (TB_External*) p->target; - - void* addr = e->thunk ? e->thunk : p->target->address; - if (addr == NULL) { - addr = get_proc(jit, p->target->name); - if (addr == NULL) { - tb_panic("Could not find procedure: %s", p->target->name); - } - } - - ptrdiff_t rel = (intptr_t)addr - ((intptr_t)patch + 4); - int32_t rel32 = rel; - if (rel == rel32) { - memcpy(dst + actual_pos, &rel32, sizeof(int32_t)); - } else { - // generate thunk to make far call - char* thunk = tb_jit_alloc_obj(jit, 6 + sizeof(void*), 1); - thunk[0] = 0xFF; // jmp qword [rip] - thunk[1] = 0x25; - thunk[2] = 0x00; - thunk[3] = 0x00; - thunk[4] = 0x00; - thunk[5] = 0x00; - - // write final address into the thunk - memcpy(thunk + 6, &addr, sizeof(void*)); - e->thunk = thunk; - - int32_t rel32 = (intptr_t)thunk - ((intptr_t)patch + 4); - *patch += rel32; - } - } else if (tag == TB_SYMBOL_GLOBAL) { - TB_Global* g = (TB_Global*) p->target; - void* addr = tb_jit_place_global(jit, g); - - int32_t* patch = (int32_t*) &dst[actual_pos]; - int32_t rel32 = (intptr_t)addr - ((intptr_t)patch + 4); - *patch += rel32; - } else { - tb_todo(); - } - } - - return dst; -} - -void* tb_jit_place_global(TB_JIT* jit, TB_Global* g) { - if (g->address != NULL) { - return g->address; - } - - char* data = tb_jit_alloc_obj(jit, g->size, g->align); - g->address = data; - - log_debug("jit: apply global %s (%p)", g->super.name ? g->super.name : "", data); - - memset(data, 0, g->size); - FOREACH_N(k, 0, g->obj_count) { - if (g->objects[k].type == TB_INIT_OBJ_REGION) { - memcpy(&data[g->objects[k].offset], g->objects[k].region.ptr, g->objects[k].region.size); - } - } - - FOREACH_N(k, 0, g->obj_count) { - if (g->objects[k].type == TB_INIT_OBJ_RELOC) { - uintptr_t addr = (uintptr_t) get_symbol_address(g->objects[k].reloc); - - uintptr_t* dst = (uintptr_t*) &data[g->objects[k].offset]; - *dst += addr; - } - } - - return data; -} - -TB_JIT* tb_jit_begin(TB_Module* m, size_t jit_heap_capacity) { - if (jit_heap_capacity == 0) { - jit_heap_capacity = 2*1024*1024; - } - - TB_JIT* jit = tb_platform_valloc(jit_heap_capacity); - mtx_init(&jit->lock, mtx_plain); - jit->capacity = jit_heap_capacity; - jit->heap.cookie = ALLOC_COOKIE; - jit->heap.size = (jit_heap_capacity - sizeof(TB_JIT)) << 1; - - // a lil unsafe... im sorry momma - tb_platform_vprotect(jit, jit_heap_capacity, TB_PAGE_RXW); - return jit; -} - -void tb_jit_end(TB_JIT* jit) { - mtx_destroy(&jit->lock); - tb_platform_vfree(jit, jit->capacity); -} - -void* tb_jit_get_code_ptr(TB_Function* f) { - return f->compiled_pos; -} - -//////////////////////////////// -// Debugger -//////////////////////////////// -#if defined(CUIK__IS_X64) && defined(_WIN32) -struct TB_CPUContext { - // used for stack crawling - void* pc; - void* sp; - - void* old_sp; - void* except; - - TB_JIT* jit; - size_t ud_size; - - volatile bool interrupted; - volatile bool running; - - CONTEXT cont; - CONTEXT state; - - // safepoint page - _Alignas(4096) char poll_site[4096]; - - char user_data[]; -}; - -typedef struct { - // win64: rcx rdx r8 r9 - // sysv: rdi rsi rdx rcx r8 r9 - uint64_t gprs[6]; -} X64Params; - -// win64 trampoline ( sp pc params ctx -- rax ) -__declspec(allocate(".text")) __declspec(align(16)) -static const uint8_t tb_jit__trampoline[] = { - // save old SP, we'll come back for it later - 0x49, 0x89, 0x61, 0x10, // mov [r9 + 0x10], rsp - // use new SP - 0x48, 0x89, 0xCC, // mov rsp, rcx - // shuffle some params into win64 volatile regs - 0x48, 0x89, 0xD0, // mov rax, rdx - 0x4D, 0x89, 0xC2, // mov r10, r8 - // fill GPR params - 0x49, 0x8B, 0x4A, 0x00, // mov rcx, [r10 + 0x00] - 0x49, 0x8B, 0x52, 0x08, // mov rdx, [r10 + 0x08] - 0x4D, 0x8B, 0x42, 0x10, // mov r8, [r10 + 0x10] - 0x4D, 0x8B, 0x4A, 0x18, // mov r9, [r10 + 0x18] - 0xFF, 0xD0, // call rax - // restore stack & return normally - 0x48, 0x81, 0xE4, 0x00, 0x00, 0xE0, 0xFF, // and rsp, -0x200000 - 0x48, 0x8B, 0x64, 0x24, 0x10, // mov rsp, [rsp + 0x10] - 0xC3, // ret -}; - -static LONG except_handler(EXCEPTION_POINTERS* e) { - TB_CPUContext* cpu = (TB_CPUContext*) (e->ContextRecord->Rsp & -STACK_SIZE); - - // necessary for stack crawling later - cpu->pc = (void*) e->ContextRecord->Rip; - cpu->sp = (void*) e->ContextRecord->Rsp; - cpu->interrupted = true; - cpu->running = false; - - switch (e->ExceptionRecord->ExceptionCode) { - case EXCEPTION_GUARD_PAGE: { - if (e->ExceptionRecord->ExceptionInformation[1] == (uintptr_t) &cpu->poll_site[0]) { - return EXCEPTION_CONTINUE_SEARCH; - } - - // we hit a safepoint poll - void* pc = (void*) e->ContextRecord->Rip; - // TB_* sp = nl_table_get(cpu->safepoints, pc); - - // TODO(NeGate): copy values out of the regs/stk - break; - } - - case EXCEPTION_ACCESS_VIOLATION: { - // TODO(NeGate): NULLchk segv - // void* accessed = (void*) e->ExceptionRecord->ExceptionInformation[1]; - printf("PAUSE RIP=%p\n", cpu->pc); - break; - } - - case EXCEPTION_BREAKPOINT: - case EXCEPTION_SINGLE_STEP: - cpu->state = *e->ContextRecord; - break; - - default: - return EXCEPTION_CONTINUE_SEARCH; - } - - // jump out of the JIT - *e->ContextRecord = cpu->cont; - return EXCEPTION_CONTINUE_EXECUTION; -} - -TB_CPUContext* tb_jit_thread_create(TB_JIT* jit, size_t ud_size) { - TB_CPUContext* cpu = tb_jit_stack_create(); - cpu->ud_size = ud_size; - cpu->jit = jit; - cpu->except = AddVectoredExceptionHandler(1, except_handler); - memset(cpu->user_data, 0, ud_size); - return cpu; -} - -void* tb_jit_thread_pc(TB_CPUContext* cpu) { return cpu->pc; } -void* tb_jit_thread_sp(TB_CPUContext* cpu) { return cpu->sp; } - -size_t tb_jit_thread_pollsite(void) { return offsetof(TB_CPUContext, poll_site); } - -void* tb_jit_thread_get_userdata(TB_CPUContext* cpu) { - return cpu->user_data; -} - -void tb_jit_breakpoint(TB_JIT* jit, void* addr) { - TB_Breakpoint bp = { addr }; - dyn_array_for(i, jit->breakpoints) { - if (jit->breakpoints[i].pos == addr) { - return; - } - } - - dyn_array_put(jit->breakpoints, bp); -} - -bool tb_jit_thread_step(TB_CPUContext* cpu, uint64_t* ret, uintptr_t pc_start, uintptr_t pc_end) { - // step until we're out of the current PC region - DynArray(TB_Breakpoint) bp = cpu->jit->breakpoints; - uintptr_t rip = cpu->state.Rip; - while (rip >= pc_start && rip < pc_end) { - // printf("step %#"PRIxPTR" [%#"PRIxPTR" %#"PRIxPTR"]\n", rip, pc_start, pc_end); - - // install breakpoints - dyn_array_for(i, bp) { - uint8_t* code = bp[i].pos; - if (code != (uint8_t*) rip) { - bp[i].prev_byte = *code; - *code = 0xCC; - } - } - - // enable trap flag for single-stepping - cpu->state.EFlags |= 0x100; - - // save our precious restore point - cpu->running = true; - cpu->state.ContextFlags = 0x10000f; - RtlCaptureContext(&cpu->cont); - if (cpu->running) { - RtlRestoreContext(&cpu->state, NULL); - } - - // uninstall breakpoints - dyn_array_for(i, bp) { - uint8_t* code = bp[i].pos; - *code = bp[i].prev_byte; - } - - rip = cpu->state.Rip; - } - - uintptr_t t = (uintptr_t) &tb_jit__trampoline; - if ((rip - t) < sizeof(tb_jit__trampoline)) { - // we in the trampoline, read RAX for the return - *ret = cpu->state.Rax; - return true; - } else { - return false; - } -} - -bool tb_jit_thread_call(TB_CPUContext* cpu, void* pc, uint64_t* ret, size_t arg_count, void** args) { - // save our precious restore point - cpu->interrupted = false; - cpu->running = true; - RtlCaptureContext(&cpu->cont); - - if (cpu->running) { - // continue in JITted code - X64Params params; - - #ifdef _WIN32 - enum { ABI_GPR_COUNT = 4 }; - #else - enum { ABI_GPR_COUNT = 6 }; - #endif - - size_t i = 0; - for (; i < arg_count && i < ABI_GPR_COUNT; i++) { - memcpy(¶ms.gprs[i], &args[i], sizeof(uint64_t)); - } - - size_t stack_usage = 8 + align_up((arg_count > 0 && arg_count < 4 ? 4 : arg_count) * 8, 16); - - // go to stack top (minus whatever alloc'd param space) - uint8_t* sp = (uint8_t*) cpu; - sp += 2*1024*1024; - sp -= stack_usage; - - // fill param slots - for (; i < arg_count; i++) { - memcpy(&sp[i*8], &args[i], sizeof(uint64_t)); - } - - // install breakpoints - DynArray(TB_Breakpoint) bp = cpu->jit->breakpoints; - dyn_array_for(i, bp) { - uint8_t* code = bp[i].pos; - if (code != pc) { - bp[i].prev_byte = *code; - *code = 0xCC; - } - } - - typedef uint64_t (*Trampoline)(void* sp, void* pc, X64Params* params, TB_CPUContext* cpu); - uint64_t r = ((Trampoline)tb_jit__trampoline)(sp, pc, ¶ms, cpu); - if (ret) { *ret = r; } - - // uninstall breakpoints - dyn_array_for(i, bp) { - uint8_t* code = bp[i].pos; - *code = bp[i].prev_byte; - } - - cpu->running = false; - } - - return cpu->interrupted; -} -#endif diff --git a/vendor/tb/src/libtb.c b/vendor/tb/src/libtb.c deleted file mode 100644 index 3f431998..00000000 --- a/vendor/tb/src/libtb.c +++ /dev/null @@ -1,109 +0,0 @@ -// This is the TB unity build -void* tb_jit_stack_create(void); - -#include "tb.c" -#include "hash.c" -#include "abi.c" -#include "tb_builder.c" -#include "debug_builder.c" -#include "ir_printer.c" -#include "exporter.c" -#include "symbols.c" -#include "disasm.c" - -// JIT -#include "jit.c" - -// Optimizer -#include "opt/optimizer.c" -#include "new_builder.c" - -// Regalloc -#include "lsra.c" -#include "chaitin.c" - -// Parsers -#define TB_COFF_IMPL -#include - -// Debug -#include "debug/cv.c" -#include "debug/sdg.c" - -// Objects -#include "objects/coff.c" -#include "objects/elf64.c" -#include "objects/macho.c" -#include "objects/wasm_obj.c" - -// Linker -#include "linker/linker.c" -#include "linker/pe.c" -#include "linker/elf.c" - -// Platform layer -#if defined(_WIN32) -#pragma comment(lib, "onecore.lib") - -void* tb_platform_valloc(size_t size) { - return VirtualAlloc(NULL, size, MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE); -} - -void tb_platform_vfree(void* ptr, size_t size) { - VirtualFree(ptr, 0, MEM_RELEASE); -} - -bool tb_platform_vprotect(void* ptr, size_t size, TB_MemProtect prot) { - DWORD protect; - switch (prot) { - case TB_PAGE_RO: protect = PAGE_READONLY; break; - case TB_PAGE_RW: protect = PAGE_READWRITE; break; - case TB_PAGE_RX: protect = PAGE_EXECUTE_READ; break; - case TB_PAGE_RXW: protect = PAGE_EXECUTE_READWRITE; break; - default: return false; - } - - DWORD old_protect; - return VirtualProtect(ptr, size, protect, &old_protect); -} - -#if NTDDI_VERSION >= NTDDI_WIN10_RS4 -void* tb_jit_stack_create(void) { - size_t size = 2*1024*1024; - - // natural alignment stack because it makes it easy to always find - // the base. - MEM_EXTENDED_PARAMETER param = { - .Type = MemExtendedParameterAddressRequirements, - .Pointer = &(MEM_ADDRESS_REQUIREMENTS){ .Alignment = size } - }; - - return VirtualAlloc2(GetCurrentProcess(), NULL, size, MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE, ¶m, 1); -} -#endif /* NTDDI_VERSION >= NTDDI_WIN10_RS4 */ -#elif defined(_POSIX_C_SOURCE) -void* tb_platform_valloc(size_t size) { - return mmap(NULL, size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); -} - -void* tb_platform_valloc_guard(size_t size) { - return mmap(NULL, size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); -} - -void tb_platform_vfree(void* ptr, size_t size) { - munmap(ptr, size); -} - -bool tb_platform_vprotect(void* ptr, size_t size, TB_MemProtect prot) { - uint32_t protect; - switch (prot) { - case TB_PAGE_RO: protect = PROT_READ; break; - case TB_PAGE_RW: protect = PROT_READ | PROT_WRITE; break; - case TB_PAGE_RX: protect = PROT_READ | PROT_EXEC; break; - case TB_PAGE_RXW: protect = PROT_READ | PROT_WRITE | PROT_EXEC; break; - default: return false; - } - - return mprotect(ptr, size, protect) == 0; -} -#endif diff --git a/vendor/tb/src/linker/README.txt b/vendor/tb/src/linker/README.txt deleted file mode 100644 index 7e0f8582..00000000 --- a/vendor/tb/src/linker/README.txt +++ /dev/null @@ -1,40 +0,0 @@ -If you're interested in contributing to this code this is where you'd learn. -Essentially this is the builtin linker, it can convert TB modules, external -objects and libraries into executables. Let's walk through making a dummy -linker. - -First, create a new source file (dummy.c) in the linker directory. We're gonna -populate it with our dummy vtable (this is how the generic linker piece calls -into the format-specific details): - -``` -// in dummy.c -#define NL_STRING_MAP_IMPL -#include "linker.h" - -static void append_object(TB_Linker* l, TB_ObjectFile* obj) { - // implement this -} - -static void append_library(TB_Linker* l, TB_Slice ar_file) { - // implement this -} - -static void append_module(TB_Linker* l, TB_Module* m) { - // implement this -} - -static TB_Exports export(TB_Linker* l) { - // implement this - return (TB_Exports){ 0 }; -} - -TB_LinkerVtbl tb__linker_dummy = { - .append_object = append_object, - .append_library = append_library, - .append_module = append_module, - .export = export -}; -``` - -We can start with stubbing out `header` diff --git a/vendor/tb/src/linker/elf.c b/vendor/tb/src/linker/elf.c deleted file mode 100644 index 50936898..00000000 --- a/vendor/tb/src/linker/elf.c +++ /dev/null @@ -1,215 +0,0 @@ -#define NL_STRING_MAP_IMPL -#include "linker.h" -#include - -static void elf_append_object(TB_Linker* l, TB_LinkerThreadInfo* info, TB_Slice obj_name, TB_Slice content) { - // implement this -} - -static void elf_append_library(TB_Linker* l, TB_LinkerThreadInfo* info, TB_Slice ar_name, TB_Slice ar_file) { - // implement this -} - -static void elf_append_module(TB_Linker* l, TB_LinkerThreadInfo* info, TB_Module* m) { - CUIK_TIMED_BLOCK("layout section") { - tb_module_layout_sections(m); - } - - // We don't *really* care about this info beyond nicer errors - TB_LinkerObject* obj_file = tb_arena_alloc(info->perm_arena, sizeof(TB_LinkerObject)); - *obj_file = (TB_LinkerObject){ .module = m }; - - DynArray(TB_ModuleSection) sections = m->sections; - dyn_array_for(i, sections) { - uint32_t flags = TB_PF_R; - if (sections[i].flags & TB_MODULE_SECTION_WRITE) flags |= TB_PF_W; - if (sections[i].flags & TB_MODULE_SECTION_EXEC) flags |= TB_PF_X; - tb_linker_append_module_section(l, info, obj_file, §ions[i], flags); - } - - tb_linker_append_module_symbols(l, m); -} - -static void elf_init(TB_Linker* l) { - l->entrypoint = "_start"; -} - -#define WRITE(data, size) (memcpy(&output[write_pos], data, size), write_pos += (size)) -static TB_ExportBuffer elf_export(TB_Linker* l, TB_Arena* arena) { - CUIK_TIMED_BLOCK("GC sections") { - tb_linker_push_named(l, l->entrypoint); - tb_linker_mark_live(l); - } - - if (!tb__finalize_sections(l)) { - return (TB_ExportBuffer){ 0 }; - } - - TB_Emitter strtbl = { 0 }; - tb_out_reserve(&strtbl, 1024); - tb_out1b(&strtbl, 0); // null string in the table - - size_t final_section_count = 0; - nl_table_for(e, &l->sections) { - TB_LinkerSection* sec = e->v; - if (sec->generic_flags & TB_LINKER_SECTION_DISCARD) continue; - - // reserve space for names - sec->name_pos = tb_outs(&strtbl, tb_linker_intern_len(l, sec->name) + 1, sec->name); - tb_out1b_UNSAFE(&strtbl, 0); - - // we're keeping it for export - final_section_count += 1; - } - - TB_Elf64_Shdr strtab = { - .name = tb_outstr_nul_UNSAFE(&strtbl, ".strtab"), - .type = TB_SHT_STRTAB, - .flags = 0, - .addralign = 1, - .size = strtbl.count, - }; - - size_t size_of_headers = sizeof(TB_Elf64_Ehdr) - + (final_section_count * sizeof(TB_Elf64_Phdr)) - + ((2+final_section_count) * sizeof(TB_Elf64_Shdr)); - - size_t section_content_size = 0; - uint64_t virt_addr = size_of_headers; - CUIK_TIMED_BLOCK("layout sections") { - nl_table_for(e, &l->sections) { - TB_LinkerSection* s = e->v; - if (s->generic_flags & TB_LINKER_SECTION_DISCARD) continue; - - s->offset = size_of_headers + section_content_size; - section_content_size += s->total_size; - - s->address = virt_addr; - virt_addr += s->total_size; - // virt_addr = align_up(virt_addr + s->total_size, 4096); - } - } - - strtab.offset = size_of_headers + section_content_size; - section_content_size += strtbl.count; - - uint16_t machine = 0; - switch (l->target_arch) { - case TB_ARCH_X86_64: machine = TB_EM_X86_64; break; - case TB_ARCH_AARCH64: machine = TB_EM_AARCH64; break; - default: tb_todo(); - } - - size_t output_size = size_of_headers + section_content_size; - size_t write_pos = 0; - - TB_ExportChunk* chunk = tb_export_make_chunk(arena, output_size); - uint8_t* restrict output = chunk->data; - - TB_Elf64_Ehdr header = { - .ident = { - [TB_EI_MAG0] = 0x7F, // magic number - [TB_EI_MAG1] = 'E', - [TB_EI_MAG2] = 'L', - [TB_EI_MAG3] = 'F', - [TB_EI_CLASS] = 2, // 64bit ELF file - [TB_EI_DATA] = 1, // little-endian - [TB_EI_VERSION] = 1, // 1.0 - [TB_EI_OSABI] = 0, - [TB_EI_ABIVERSION] = 0 - }, - .type = TB_ET_DYN, // executable - .version = 1, - .machine = machine, - .entry = 0, - - .flags = 0, - - .ehsize = sizeof(TB_Elf64_Ehdr), - - .phentsize = sizeof(TB_Elf64_Phdr), - .phoff = sizeof(TB_Elf64_Ehdr), - .phnum = final_section_count, - - .shoff = sizeof(TB_Elf64_Ehdr) + (sizeof(TB_Elf64_Phdr) * final_section_count), - .shentsize = sizeof(TB_Elf64_Shdr), - .shnum = final_section_count + 2, - .shstrndx = 1, - }; - - // text section crap - TB_LinkerSection* text = tb_linker_find_section2(l, ".text"); - TB_LinkerSymbol* sym = tb_linker_find_symbol2(l, l->entrypoint); - if (text && sym) { - if (sym->tag == TB_LINKER_SYMBOL_NORMAL) { - header.entry = text->address + sym->normal.piece->offset + sym->normal.secrel; - } else if (sym->tag == TB_LINKER_SYMBOL_TB) { - header.entry = text->address + sym->tb.piece->offset + tb__get_symbol_pos(sym->tb.sym); - } else { - tb_todo(); - } - } else { - fprintf(stderr, "tblink: could not find entrypoint!\n"); - } - WRITE(&header, sizeof(header)); - - // write program headers - nl_table_for(e, &l->sections) { - TB_LinkerSection* s = e->v; - TB_Elf64_Phdr sec = { - .type = TB_PT_LOAD, - .flags = s->flags, - .offset = s->offset, - .vaddr = s->address, - .filesz = s->total_size, - .memsz = s->total_size, - .align = 1, - }; - WRITE(&sec, sizeof(sec)); - } - - // write section headers - memset(&output[write_pos], 0, sizeof(TB_Elf64_Shdr)), write_pos += sizeof(TB_Elf64_Shdr); - WRITE(&strtab, sizeof(strtab)); - nl_table_for(e, &l->sections) { - TB_LinkerSection* s = e->v; - TB_Elf64_Shdr sec = { - .name = s->name_pos, - .type = TB_SHT_PROGBITS, - .flags = TB_SHF_ALLOC | ((s->flags & TB_PF_X) ? TB_SHF_EXECINSTR : 0) | ((s->flags & TB_PF_W) ? TB_SHF_WRITE : 0), - .addralign = 1, - .size = s->total_size, - .addr = s->address, - .offset = s->offset, - }; - WRITE(&sec, sizeof(sec)); - } - - TB_LinkerSection* data = tb_linker_find_section2(l, ".data"); - TB_LinkerSection* rdata = tb_linker_find_section2(l, ".rdata"); - - // write section contents - write_pos = tb__apply_section_contents(l, output, write_pos, text, data, rdata, 1, 0); - WRITE(strtbl.data, strtbl.count); - assert(write_pos == output_size); - - // TODO(NeGate): multithread this too - /*CUIK_TIMED_BLOCK("apply final relocations") { - dyn_array_for(i, l->ir_modules) { - tb__apply_module_relocs(l, l->ir_modules[i], output); - } - - // tb__apply_external_relocs(l, output, opt_header.image_base); - }*/ - - // write section contents - return (TB_ExportBuffer){ .total = output_size, .head = chunk, .tail = chunk }; -} - -TB_LinkerVtbl tb__linker_elf = { - .init = elf_init, - .append_object = elf_append_object, - .append_library = elf_append_library, - .append_module = elf_append_module, - .export = elf_export -}; diff --git a/vendor/tb/src/linker/linker.c b/vendor/tb/src/linker/linker.c deleted file mode 100644 index 30f4555d..00000000 --- a/vendor/tb/src/linker/linker.c +++ /dev/null @@ -1,797 +0,0 @@ -#include "linker.h" - -#if __STDC_VERSION__ < 201112L || defined(__STDC_NO_ATOMICS__) -#error "Missing C11 support for stdatomic.h" -#endif - -#include - -TB_LinkerThreadInfo* linker_thread_info(TB_Linker* l) { - static thread_local TB_LinkerThreadInfo* chain; - - // almost always refers to one TB_ThreadInfo, but - // we can't assume the user has merely on TB_Module - // per thread. - TB_LinkerThreadInfo* info = chain; - while (info != NULL) { - if (info->owner == l) { - goto done; - } - info = info->next; - } - - info = tb_platform_heap_alloc(sizeof(TB_LinkerThreadInfo)); - *info = (TB_LinkerThreadInfo){ .owner = l }; - - // allocate memory for it - info->perm_arena = tb_arena_create(TB_ARENA_LARGE_CHUNK_SIZE); - info->tmp_arena = tb_arena_create(TB_ARENA_LARGE_CHUNK_SIZE); - - // thread local so it doesn't need to synchronize - info->next = chain; - if (chain != NULL) { - chain->prev = info; - } - chain = info; - - // link to the TB_Module* (we need to this to free later) - TB_LinkerThreadInfo* old_top; - do { - old_top = atomic_load(&l->first_thread_info); - info->next_in_link = old_top; - } while (!atomic_compare_exchange_strong(&l->first_thread_info, &old_top, info)); - - done: - return info; -} - -extern TB_LinkerVtbl tb__linker_pe, tb__linker_elf; - -TB_API TB_ExecutableType tb_system_executable_format(TB_System s) { - switch (s) { - case TB_SYSTEM_WINDOWS: return TB_EXECUTABLE_PE; - case TB_SYSTEM_LINUX: return TB_EXECUTABLE_ELF; - default: tb_todo(); return TB_EXECUTABLE_UNKNOWN; - } -} - -TB_API TB_Linker* tb_linker_create(TB_ExecutableType exe, TB_Arch arch) { - TB_Linker* l = tb_platform_heap_alloc(sizeof(TB_Linker)); - memset(l, 0, sizeof(TB_Linker)); - l->target_arch = arch; - - l->symbols = nl_table_alloc(256); - l->sections = nl_table_alloc(16); - - switch (exe) { - case TB_EXECUTABLE_PE: l->vtbl = tb__linker_pe; break; - // case TB_EXECUTABLE_ELF: l->vtbl = tb__linker_elf; break; - default: break; - } - - l->vtbl.init(l); - return l; -} - -TB_API void tb_linker_set_subsystem(TB_Linker* l, TB_WindowsSubsystem subsystem) { - l->subsystem = subsystem; -} - -TB_API void tb_linker_set_entrypoint(TB_Linker* l, const char* name) { - l->entrypoint = name; -} - -TB_API void tb_linker_append_object(TB_Linker* l, TB_Slice obj_name, TB_Slice content) { - CUIK_TIMED_BLOCK("append_object") { - TB_LinkerThreadInfo* info = linker_thread_info(l); - l->vtbl.append_object(l, info, obj_name, content); - } -} - -TB_API void tb_linker_append_module(TB_Linker* l, TB_Module* m) { - CUIK_TIMED_BLOCK("append_module") { - TB_LinkerThreadInfo* info = linker_thread_info(l); - l->vtbl.append_module(l, info, m); - } -} - -TB_API void tb_linker_append_library(TB_Linker* l, TB_Slice ar_name, TB_Slice ar_file) { - CUIK_TIMED_BLOCK("append_library") { - TB_LinkerThreadInfo* info = linker_thread_info(l); - l->vtbl.append_library(l, info, ar_name, ar_file); - } -} - -TB_API TB_ExportBuffer tb_linker_export(TB_Linker* l, TB_Arena* arena) { - return l->vtbl.export(l, arena); -} - -TB_API void tb_linker_destroy(TB_Linker* l) { - tb_platform_heap_free(l); -} - -void tb_linker_unresolved_sym(TB_Linker* l, TB_LinkerInternStr name) { - fprintf(stderr, "\x1b[31merror\x1b[0m: unresolved external: %s\n", name); -} - -TB_LinkerSectionPiece* tb_linker_get_piece(TB_Linker* l, TB_LinkerSymbol* restrict sym) { - if (sym && (sym->tag == TB_LINKER_SYMBOL_NORMAL || sym->tag == TB_LINKER_SYMBOL_TB)) { - return sym->normal.piece; - } - - return NULL; -} - -size_t tb__get_symbol_pos(TB_Symbol* s) { - if (s->tag == TB_SYMBOL_FUNCTION) { - return ((TB_Function*) s)->output->code_pos; - } else if (s->tag == TB_SYMBOL_GLOBAL) { - return ((TB_Global*) s)->pos; - } else { - tb_todo(); - } -} - -uint64_t tb__get_symbol_rva(TB_Linker* l, TB_LinkerSymbol* sym) { - if (sym->tag == TB_LINKER_SYMBOL_ABSOLUTE) { - return 0; - } else if (sym->tag == TB_LINKER_SYMBOL_IMAGEBASE) { - return sym->imagebase; - } - - // normal or TB - assert(sym->tag == TB_LINKER_SYMBOL_NORMAL || sym->tag == TB_LINKER_SYMBOL_TB); - TB_LinkerSectionPiece* piece = sym->normal.piece; - - uint32_t rva = piece->parent->address + piece->offset; - if (sym->tag == TB_LINKER_SYMBOL_NORMAL) { - return rva + sym->normal.secrel; - } - - TB_Symbol* s = sym->tb.sym; - if (s->tag == TB_SYMBOL_FUNCTION) { - TB_Function* f = (TB_Function*) s; - assert(f->output != NULL); - - return rva + f->output->code_pos; - } else if (s->tag == TB_SYMBOL_GLOBAL) { - return rva + ((TB_Global*) s)->pos; - } else { - tb_todo(); - } -} - -uint64_t tb__compute_rva(TB_Linker* l, TB_Module* m, const TB_Symbol* s) { - if (s->tag == TB_SYMBOL_FUNCTION) { - TB_Function* f = (TB_Function*) s; - assert(f->output != NULL); - - TB_LinkerSectionPiece* piece = m->sections[f->section].piece; - return piece->parent->address + piece->offset + f->output->code_pos; - } else if (s->tag == TB_SYMBOL_GLOBAL) { - TB_Global* g = (TB_Global*) s; - TB_LinkerSectionPiece* piece = m->sections[g->parent].piece; - return piece->parent->address + piece->offset + g->pos; - } else { - tb_todo(); - return 0; - } -} - -size_t tb__pad_file(uint8_t* output, size_t write_pos, char pad, size_t align) { - size_t align_mask = align - 1; - size_t end = (write_pos + align_mask) & ~align_mask; - if (write_pos != end) { - memset(output + write_pos, 0, end - write_pos); - write_pos = end; - } - return write_pos; -} - -void tb_linker_merge_sections(TB_Linker* linker, TB_LinkerSection* from, TB_LinkerSection* to) { - if (from == NULL || to == NULL) return; - if (from->generic_flags & TB_LINKER_SECTION_DISCARD) return; - - // remove 'from' from final output - from->generic_flags |= TB_LINKER_SECTION_DISCARD; - - if (to->last) to->last->next = from->first; - else to->first = from->first; - to->last = from->last; - - // we don't care about fixing up the offsets because they'll be properly laid out - // after this is all done - to->total_size += from->total_size; - to->piece_count += from->piece_count; - - for (TB_LinkerSectionPiece* p = from->first; p != NULL; p = p->next) { - p->parent = to; - } -} - -void tb_linker_append_module_section(TB_Linker* l, TB_LinkerThreadInfo* info, TB_LinkerObject* mod, TB_ModuleSection* section, uint32_t flags) { - assert(mod->module != NULL && "not a TB_Module's section?"); - if (section->total_size > 0) { - TB_LinkerInternStr name = tb_linker_intern_string(l, strlen(section->name), section->name); - - TB_LinkerSection* ls = tb_linker_find_or_create_section(l, name, flags); - section->piece = tb_linker_append_piece(info, ls, PIECE_MODULE_SECTION, section->total_size, section, mod); - } -} - -void tb_linker_associate(TB_Linker* l, TB_LinkerSectionPiece* a, TB_LinkerSectionPiece* b) { - assert(a->assoc == NULL); - a->assoc = b; -} - -size_t tb__apply_section_contents(TB_Linker* l, uint8_t* output, size_t write_pos, TB_LinkerSection* text, TB_LinkerSection* data, TB_LinkerSection* rdata, size_t section_alignment, size_t image_base) { - // write section contents - // TODO(NeGate): we can actually parallelize this part of linking - CUIK_TIMED_BLOCK("write sections") { - nl_table_for(e, &l->sections) { - TB_LinkerSection* s = e->v; - if (s->generic_flags & TB_LINKER_SECTION_DISCARD) continue; - - assert(s->offset == write_pos); - for (TB_LinkerSectionPiece* p = s->first; p != NULL; p = p->next) { - uint8_t* p_out = &output[write_pos]; - TB_LinkerObject* obj = p->obj; - - switch (p->kind) { - case PIECE_NORMAL: { - if (p->data == NULL) goto skip; - - memcpy(p_out, p->data, p->size); - break; - } - case PIECE_MODULE_SECTION: { - tb_helper_write_section(obj->module, 0, (TB_ModuleSection*) p->data, p_out, 0); - break; - } - case PIECE_PDATA: { - uint32_t* p_out32 = (uint32_t*) p_out; - TB_Module* m = obj->module; - - uint32_t rdata_rva = m->xdata->parent->address + m->xdata->offset; - - dyn_array_for(i, m->sections) { - DynArray(TB_FunctionOutput*) funcs = m->sections[i].funcs; - TB_LinkerSectionPiece* piece = m->sections[i].piece; - if (piece == NULL) { - continue; - } - - uint32_t rva = piece->parent->address + piece->offset; - dyn_array_for(j, funcs) { - TB_FunctionOutput* out_f = funcs[j]; - if (out_f != NULL) { - // both into the text section - *p_out32++ = rva + out_f->code_pos; - *p_out32++ = rva + out_f->code_pos + out_f->code_size; - - // refers to rdata section - *p_out32++ = rdata_rva + out_f->unwind_info; - } - } - } - break; - } - case PIECE_RELOC: { - TB_Module* m = obj->module; - assert(m != NULL); - - dyn_array_for(i, m->sections) { - DynArray(TB_Global*) globals = m->sections[i].globals; - TB_LinkerSectionPiece* piece = m->sections[i].piece; - - uint32_t data_rva = piece->parent->address + piece->offset; - uint32_t data_file = piece->parent->offset + piece->offset; - - uint32_t last_page = 0xFFFFFFFF; - uint32_t* last_block = NULL; - - dyn_array_for(j, globals) { - TB_Global* g = globals[j]; - FOREACH_N(k, 0, g->obj_count) { - size_t actual_pos = g->pos + g->objects[k].offset; - size_t actual_page = actual_pos & ~4095; - size_t page_offset = actual_pos - actual_page; - - if (g->objects[k].type != TB_INIT_OBJ_RELOC) { - continue; - } - - const TB_Symbol* s = g->objects[k].reloc; - if (last_page != actual_page) { - last_page = data_rva + actual_page; - last_block = (uint32_t*) p_out; - - last_block[0] = data_rva + actual_page; - last_block[1] = 8; // block size field (includes RVA field and itself) - p_out += 8; - } - - // compute RVA - uint32_t file_pos = data_file + actual_pos; - *((uint64_t*) &output[file_pos]) = tb__compute_rva(l, m, s) + image_base; - - // emit relocation - uint16_t payload = (10 << 12) | page_offset; // (IMAGE_REL_BASED_DIR64 << 12) | offset - *((uint16_t*) p_out) = payload, p_out += sizeof(uint16_t); - last_block[1] += 2; - } - } - } - break; - } - default: tb_todo(); - } - - write_pos += p->size; - skip:; - } - - write_pos = tb__pad_file(output, write_pos, 0x00, section_alignment); - } - } - - return write_pos; -} - -TB_LinkerSection* tb_linker_find_section(TB_Linker* l, TB_LinkerInternStr name) { - return nl_table_get(&l->sections, name); -} - -TB_LinkerSection* tb_linker_find_section2(TB_Linker* l, const char* name) { - return nl_table_get(&l->sections, tb_linker_intern_cstring(l, name)); -} - -TB_LinkerSection* tb_linker_find_or_create_section(TB_Linker* l, TB_LinkerInternStr name, uint32_t flags) { - // allocate new section if one doesn't exist already - TB_LinkerSection* s = nl_table_get(&l->sections, name); - if (s) { - return s; - } - - TB_LinkerThreadInfo* info = linker_thread_info(l); - s = tb_arena_alloc(info->perm_arena, sizeof(TB_LinkerSection)); - *s = (TB_LinkerSection){ .name = name, .flags = flags }; - - nl_table_put(&l->sections, name, s); - return s; -} - -TB_LinkerSectionPiece* tb_linker_append_piece(TB_LinkerThreadInfo* info, TB_LinkerSection* section, int kind, size_t size, const void* data, TB_LinkerObject* obj) { - // allocate some space for it, we might wanna make the total_size increment atomic - TB_LinkerSectionPiece* piece = tb_arena_alloc(info->perm_arena, sizeof(TB_LinkerSectionPiece)); - *piece = (TB_LinkerSectionPiece){ - .kind = kind, - .parent = section, - .obj = obj, - .offset = section->total_size, - .size = size, - .vsize = size, - .data = data, - }; - section->total_size += size; - section->piece_count += 1; - - if (section->last == NULL) { - section->first = section->last = piece; - } else { - section->last->next = piece; - section->last = piece; - } - return piece; -} - -TB_LinkerSymbol* tb_linker_find_symbol(TB_Linker* l, TB_LinkerInternStr name) { - return nl_table_get(&l->symbols, name); -} - -TB_LinkerSymbol* tb_linker_find_symbol2(TB_Linker* l, const char* name) { - return nl_table_get(&l->symbols, tb_linker_intern_cstring(l, name)); -} - -TB_LinkerSymbol* tb_linker_new_symbol(TB_Linker* l, TB_LinkerInternStr name) { - cuikperf_region_start("append sym", NULL); - - TB_LinkerThreadInfo* info = linker_thread_info(l); - TB_LinkerSymbol* sym = tb_arena_alloc(info->perm_arena, sizeof(TB_LinkerSymbol)); - - TB_LinkerSymbol* old; - if (old = nl_table_put(&l->symbols, name, sym), old) { - tb_arena_free(info->perm_arena, sym, sizeof(TB_LinkerSymbol)); - sym = old; - - if (old->tag != TB_LINKER_SYMBOL_UNKNOWN) { - // symbol collision if we're overriding something that's - // not a forward ref. - // __debugbreak(); - } - } else { - *sym = (TB_LinkerSymbol){ .name = name }; - } - - cuikperf_region_end(); - return sym; -} - -TB_LinkerInternStr tb_linker_intern_string(TB_Linker* l, size_t len, const char* str) { - if (l->interner.arr == NULL) { - CUIK_TIMED_BLOCK("alloc atoms") { - l->interner.exp = 20; - l->interner.cnt = 0; - l->interner.arr = cuik__valloc((1u << 20) * sizeof(TB_LinkerInternStr)); - } - } - - uint32_t mask = (1 << l->interner.exp) - 1; - uint32_t hash = tb__murmur3_32(str, len); - size_t first = hash & mask, i = first; - - do { - // linear probe - if (LIKELY(l->interner.arr[i] == NULL)) { - TB_LinkerThreadInfo* info = linker_thread_info(l); - TB_LinkerInternStr newstr = tb_arena_alloc(info->perm_arena, len + 5); - uint32_t len32 = len; - memcpy(newstr, &len32, 4); - memcpy(newstr + 4, str, len); - newstr[len + 4] = 0; - - l->interner.arr[i] = newstr + 4; - l->interner.cnt++; - return &newstr[4]; - } else if (len == tb_linker_intern_len(l, l->interner.arr[i]) && memcmp(str, l->interner.arr[i], len) == 0) { - return l->interner.arr[i]; - } - - i = (i + 1) & mask; - } while (i != first); - - log_error("atoms arena: out of memory!\n"); - abort(); -} - -TB_LinkerInternStr tb_linker_intern_cstring(TB_Linker* l, const char* str) { - return tb_linker_intern_string(l, strlen(str), str); -} - -size_t tb_linker_intern_len(TB_Linker* l, TB_LinkerInternStr str) { - return ((uint32_t*) str)[-1]; -} - -void tb_linker_append_module_symbols(TB_Linker* l, TB_Module* m) { - DynArray(TB_ModuleSection) sections = m->sections; - TB_LinkerThreadInfo* info = linker_thread_info(l); - - CUIK_TIMED_BLOCK("apply symbols") { - dyn_array_for(i, sections) { - DynArray(TB_FunctionOutput*) funcs = sections[i].funcs; - DynArray(TB_Global*) globals = sections[i].globals; - TB_LinkerSectionPiece* piece = sections[i].piece; - - dyn_array_for(i, funcs) { - const char* name = funcs[i]->parent->super.name; - TB_LinkerInternStr interned = tb_linker_intern_string(l, strlen(name), name); - - TB_LinkerSymbol* ls = tb_linker_new_symbol(l, interned); - ls->tag = TB_LINKER_SYMBOL_TB; - ls->tb.piece = piece; - ls->tb.sym = &funcs[i]->parent->super; - } - - dyn_array_for(i, globals) { - const char* name = globals[i]->super.name; - TB_LinkerInternStr interned = tb_linker_intern_string(l, strlen(name), name); - - TB_LinkerSymbol* ls = NULL; - if (globals[i]->linkage == TB_LINKAGE_PRIVATE) { - ls = tb_arena_alloc(info->perm_arena, sizeof(TB_LinkerSymbol)); - *ls = (TB_LinkerSymbol){ .name = interned }; - } else { - ls = tb_linker_new_symbol(l, interned); - } - ls->tag = TB_LINKER_SYMBOL_TB; - ls->tb.piece = piece; - ls->tb.sym = &globals[i]->super; - } - } - } -} - -void tb_linker_apply_module_relocs(TB_Linker* l, TB_Module* m, TB_LinkerSection* text, uint8_t* output) { - uint64_t trampoline_rva = text->address + l->trampoline_pos; - - dyn_array_for(i, m->sections) { - DynArray(TB_FunctionOutput*) funcs = m->sections[i].funcs; - TB_LinkerSectionPiece* piece = m->sections[i].piece; - if (piece == NULL) { - continue; - } - - uint64_t text_piece_rva = piece->parent->address + piece->offset; - uint64_t text_piece_file = piece->parent->offset + piece->offset; - - dyn_array_for(j, funcs) { - TB_FunctionOutput* out_f = funcs[j]; - for (TB_SymbolPatch* patch = out_f->first_patch; patch; patch = patch->next) { - int32_t* dst = (int32_t*) &output[text_piece_file + out_f->code_pos + patch->pos]; - size_t actual_pos = text_piece_rva + out_f->code_pos + patch->pos + 4; - - int32_t p = 0; - if (patch->target->tag == TB_SYMBOL_EXTERNAL) { - TB_LinkerSymbol* sym = patch->target->address; - if (sym->tag == TB_LINKER_SYMBOL_UNKNOWN) { - // TODO(NeGate): error for unresolved symbol - tb_todo(); - } else if (sym->tag == TB_LINKER_SYMBOL_THUNK) { - p = trampoline_rva + (sym->thunk->import.thunk_id * 6); - } else if (sym->tag == TB_LINKER_SYMBOL_IMPORT) { - p = trampoline_rva + (sym->import.thunk_id * 6); - } else { - p = tb__get_symbol_rva(l, sym); - } - - p -= actual_pos; - } else if (patch->target->tag == TB_SYMBOL_FUNCTION) { - // internal patching has already handled this - } else if (patch->target->tag == TB_SYMBOL_GLOBAL) { - TB_Global* global = (TB_Global*) patch->target; - assert(global->super.tag == TB_SYMBOL_GLOBAL); - - uint32_t flags = m->sections[global->parent].flags; - TB_LinkerSectionPiece* piece = m->sections[global->parent].piece; - uint32_t piece_rva = piece->parent->address + piece->offset; - - int32_t* dst = (int32_t*) &output[text_piece_file + out_f->code_pos + patch->pos]; - if (flags & TB_MODULE_SECTION_TLS) { - // section relative for TLS - p = piece_rva + global->pos; - } else { - p = (piece_rva + global->pos) - actual_pos; - } - } else { - tb_todo(); - } - - *dst += p; - } - } - } -} - -static TB_Slice as_filename(TB_Slice s) { - size_t last = 0; - FOREACH_N(i, 0, s.length) { - if (s.data[i] == '/' || s.data[i] == '\\') { - last = i+1; - } - } - - return (TB_Slice){ s.data + last, s.length - last }; -} - -static int compare_linker_sections(const void* a, const void* b) { - const TB_LinkerSectionPiece* sec_a = *(const TB_LinkerSectionPiece**) a; - const TB_LinkerSectionPiece* sec_b = *(const TB_LinkerSectionPiece**) b; - - return sec_a->order - sec_b->order; -} - -bool tb__finalize_sections(TB_Linker* l) { - /*if (nl_map_get_capacity(l->unresolved_symbols) > 0) { - nl_map_for_str(i, l->unresolved_symbols) { - TB_UnresolvedSymbol* u = l->unresolved_symbols[i].v; - - fprintf(stderr, "\x1b[31merror\x1b[0m: unresolved external: %.*s\n", (int) u->name.length, u->name.data); - size_t i = 0; - for (; u && i < 5; u = u->next, i++) { - // walk input stack - TB_LinkerInputHandle curr = u->reloc; - fprintf(stderr, " in "); - - int depth = 0; - while (curr != 0) { - TB_LinkerInput* input = &l->inputs[curr]; - - depth++; - if (depth) { - fprintf(stderr, "("); - } - - if (input->tag == TB_LINKER_INPUT_MODULE) { - fprintf(stderr, "\n", input->module); - } else { - TB_Slice obj_name = as_filename(input->name); - fprintf(stderr, "%.*s", (int) obj_name.length, obj_name.data); - } - - curr = input->parent; - } - - while (depth--) fprintf(stderr, ")"); - fprintf(stderr, "\n"); - } - - if (u) { - // count the rest - while (u) u = u->next, i++; - - fprintf(stderr, " ...and %zu more...\n", i - 5); - } - fprintf(stderr, "\n"); - } - - nl_map_free(l->unresolved_symbols); - return false; - }*/ - - CUIK_TIMED_BLOCK("sort sections") { - TB_LinkerSectionPiece** array_form = NULL; - size_t num = 0; - - nl_table_for(e, &l->sections) { - TB_LinkerSection* s = e->v; - if (s->generic_flags & TB_LINKER_SECTION_DISCARD) { - continue; - } - - size_t piece_count = s->piece_count; - - //////////////////////////////// - // Sort sections - //////////////////////////////// - // convert into array - size_t j = 0; - CUIK_TIMED_BLOCK("convert to array") { - assert(s->piece_count != 0); - array_form = tb_platform_heap_realloc(array_form, piece_count * sizeof(TB_LinkerSectionPiece*)); - - for (TB_LinkerSectionPiece* p = s->first; p != NULL; p = p->next) { - if (p->size != 0 && (p->flags & TB_LINKER_PIECE_LIVE)) { - array_form[j++] = p; - } - } - - // fprintf(stderr, "%.*s: %zu -> %zu\n", (int) s->name.length, s->name.data, piece_count, j); - piece_count = j; - } - - if (piece_count == 0) { - s->generic_flags |= TB_LINKER_SECTION_DISCARD; - continue; - } - - // sort - CUIK_TIMED_BLOCK("sort section") { - qsort(array_form, piece_count, sizeof(TB_LinkerSectionPiece*), compare_linker_sections); - } - - // convert back into linked list - CUIK_TIMED_BLOCK("convert into list") { - array_form[0]->offset = 0; - - size_t offset = array_form[0]->size; - for (j = 1; j < piece_count; j++) { - array_form[j]->offset = offset; - offset += array_form[j]->size; - - array_form[j-1]->next = array_form[j]; - } - s->total_size = offset; - - s->first = array_form[0]; - s->last = array_form[piece_count - 1]; - s->piece_count = piece_count; - - array_form[piece_count - 1]->next = NULL; - } - - s->number = num++; - } - tb_platform_heap_free(array_form); - } - - return true; -} - -TB_LinkerSymbol* tb_linker_get_target(TB_LinkerReloc* r) { - if (r->target->tag == TB_LINKER_SYMBOL_UNKNOWN) { - return r->alt && r->alt->tag != TB_LINKER_SYMBOL_UNKNOWN ? r->alt : NULL; - } else { - return r->target; - } -} - -void tb_linker_push_piece(TB_Linker* l, TB_LinkerSectionPiece* p) { - if (p->size == 0 || (p->flags & TB_LINKER_PIECE_LIVE) || (p->parent->generic_flags & TB_LINKER_SECTION_DISCARD)) { - return; - } - - dyn_array_put(l->worklist, p); -} - -void tb_linker_push_named(TB_Linker* l, const char* name) { - TB_LinkerInternStr str = tb_linker_intern_string(l, strlen(name), name); - TB_LinkerSymbol* sym = tb_linker_find_symbol(l, str); - if (sym != NULL) { - tb_linker_push_piece(l, tb_linker_get_piece(l, sym)); - } -} - -static TB_LinkerSymbol* resolve_external(TB_Linker* l, TB_External* ext) { - TB_LinkerInternStr str = tb_linker_intern_cstring(l, ext->super.name); - TB_LinkerSymbol* sym = tb_linker_find_symbol2(l, str); - if (sym == NULL) { - tb_todo(); - } else if (sym->tag == TB_LINKER_SYMBOL_THUNK) { - sym->thunk->flags |= TB_LINKER_SYMBOL_USED; - } - - ext->super.address = sym; - sym->flags |= TB_LINKER_SYMBOL_USED; - return sym; -} - -void tb_linker_mark_live(TB_Linker* l) { - while (dyn_array_length(l->worklist)) { - TB_LinkerSectionPiece* p = dyn_array_pop(l->worklist); - p->flags |= TB_LINKER_PIECE_LIVE; - - // associated section - if (p->assoc) { - tb_linker_push_piece(l, p->assoc); - } - - // mark module content - if (p->obj->module && !p->obj->module->visited) { - p->obj->module->visited = true; - - TB_Module* m = p->obj->module; - dyn_array_for(i, m->sections) { - if (m->sections[i].piece) { - tb_linker_push_piece(l, m->sections[i].piece); - } - } - - // associate TB externals with linker symbols - FOREACH_N(i, 0, m->exports.count) { - if (&m->exports.data[i]->super == m->chkstk_extern && m->uses_chkstk == 0) { - continue; - } - - TB_LinkerSymbol* sym = resolve_external(l, m->exports.data[i]); - TB_LinkerSectionPiece* piece = tb_linker_get_piece(l, sym); - if (piece) { - tb_linker_push_piece(l, piece); - } - } - } - - // mark any kid symbols - for (TB_LinkerSymbol* sym = p->first_sym; sym != NULL; sym = sym->next) { - if (sym) { - sym->flags |= TB_LINKER_SYMBOL_USED; - if (sym->tag == TB_LINKER_SYMBOL_NORMAL || sym->tag == TB_LINKER_SYMBOL_TB) { - tb_linker_push_piece(l, sym->normal.piece); - } - } - } - - // mark any relocations - TB_LinkerThreadInfo* info = p->info; - dyn_array_for(i, p->inputs) { - size_t j = p->inputs[i]; - TB_LinkerSymbol* sym = tb_linker_get_target(&info->relocs[j]); - - if (sym) { - sym->flags |= TB_LINKER_SYMBOL_USED; - if (sym->tag == TB_LINKER_SYMBOL_NORMAL || sym->tag == TB_LINKER_SYMBOL_TB) { - tb_linker_push_piece(l, sym->normal.piece); - } - } - } - } -} diff --git a/vendor/tb/src/linker/linker.h b/vendor/tb/src/linker/linker.h deleted file mode 100644 index b6b79d8c..00000000 --- a/vendor/tb/src/linker/linker.h +++ /dev/null @@ -1,289 +0,0 @@ -#pragma once -#include "../tb_internal.h" - -typedef struct TB_LinkerSymbol TB_LinkerSymbol; -typedef struct TB_LinkerThreadInfo TB_LinkerThreadInfo; - -typedef char* TB_LinkerInternStr; - -// basically an object file -typedef struct { - size_t len; - char* name; - - // if not-NULL, the sections for the are in a TB_Module. - TB_Module* module; -} TB_LinkerObject; - -typedef enum { - TB_LINKER_PIECE_IMMUTABLE = 1, - - // by the time GC is done, this is resolved and we can - // assume any pieces without this set are dead. - TB_LINKER_PIECE_LIVE = 2, -} TB_LinkerPieceFlags; - -// we use a linked list to store these because i couldn't be bothered to allocate -// one giant sequential region for the entire linker. -struct TB_LinkerSectionPiece { - TB_LinkerSectionPiece* next; - - enum { - // write the data buffer in this struct - PIECE_NORMAL, - // Write TB_ModuleSection - PIECE_MODULE_SECTION, - // Write the TB module's pdata section - PIECE_PDATA, - // Write the TB module's reloc section - PIECE_RELOC, - } kind; - - TB_LinkerObject* obj; - TB_LinkerSection* parent; - - TB_LinkerSymbol* first_sym; - - TB_LinkerThreadInfo* info; - DynArray(uint32_t) inputs; - - // mostly for .pdata crap - TB_LinkerSectionPiece* assoc; - - // vsize is the virtual size - size_t offset, vsize, size; - // this is for COFF $ management - uint32_t order; - TB_LinkerPieceFlags flags; - const uint8_t* data; -}; - -typedef enum { - TB_LINKER_SECTION_DISCARD = 1, - TB_LINKER_SECTION_COMDAT = 2, -} TB_LinkerSectionFlags; - -struct TB_LinkerSection { - TB_LinkerInternStr name; - - TB_LinkerSectionFlags generic_flags; - uint32_t flags; - uint32_t number; - - uint32_t name_pos; - - uint64_t address; // usually a relative virtual address. - size_t offset; // in the file. - - size_t piece_count; - size_t total_size; - TB_LinkerSectionPiece *first, *last; -}; - -typedef enum TB_LinkerSymbolTag { - TB_LINKER_SYMBOL_ABSOLUTE = 0, - - TB_LINKER_SYMBOL_UNKNOWN, - - // external linkage - TB_LINKER_SYMBOL_NORMAL, - - // used for windows stuff as "__ImageBase" - TB_LINKER_SYMBOL_IMAGEBASE, - - // TB defined - TB_LINKER_SYMBOL_TB, - - // import thunks - TB_LINKER_SYMBOL_THUNK, - - // imported from shared object (named with __imp_) - TB_LINKER_SYMBOL_IMPORT, -} TB_LinkerSymbolTag; - -typedef enum TB_LinkerSymbolFlags { - TB_LINKER_SYMBOL_WEAK = 1, - TB_LINKER_SYMBOL_COMDAT = 2, - TB_LINKER_SYMBOL_USED = 4, -} TB_LinkerSymbolFlags; - -// all symbols appended to the linker are converted into -// these and used for all kinds of relocation resolution. -struct TB_LinkerSymbol { - TB_LinkerSymbolTag tag; - TB_LinkerSymbolFlags flags; - - TB_LinkerInternStr name; - - // next symbol in the section - TB_LinkerSymbol* next; - - union { - // for normal symbols - struct { - TB_LinkerSectionPiece* piece; - uint32_t secrel; - } normal; - - uint32_t absolute; - uint32_t imagebase; - - // for IR module symbols - struct { - TB_LinkerSectionPiece* piece; - TB_Symbol* sym; - } tb; - - // for PE imports - struct { - // this is the location the thunk will call - uint32_t ds_address; - // this is the ID of the thunk - uint32_t thunk_id; - // import table ID - uint32_t id; - // TODO(NeGate): i don't remember rn - uint16_t ordinal; - } import; - - TB_LinkerSymbol* thunk; - }; -}; - -typedef struct { - TB_Slice libpath; - DynArray(TB_LinkerSymbol*) thunks; - - uint64_t *iat, *ilt; -} ImportTable; - -typedef struct TB_LinkerReloc { - uint8_t type; - uint8_t addend; - - // source - TB_LinkerSectionPiece* src_piece; - uint32_t src_offset; - - // target - TB_LinkerSymbol* target; - TB_LinkerSymbol* alt; -} TB_LinkerReloc; - -typedef struct { - TB_Slice from, to; -} TB_LinkerCmd; - -struct TB_LinkerThreadInfo { - TB_Linker* owner; - TB_LinkerThreadInfo* next_in_link; - - TB_LinkerThreadInfo* prev; - TB_LinkerThreadInfo* next; - - TB_Arena* perm_arena; - TB_Arena* tmp_arena; - - // commands - // these are generated in object files and such but won't get - // executed until export time - DynArray(TB_LinkerCmd) merges; - DynArray(TB_LinkerCmd) alternates; - - DynArray(TB_LinkerReloc) relocs; - DynArray(TB_Module*) ir_modules; -}; - -// Format-specific vtable: -typedef struct TB_LinkerVtbl { - void (*init)(TB_Linker* l); - void (*append_object)(TB_Linker* l, TB_LinkerThreadInfo* info, TB_Slice obj_name, TB_Slice content); - void (*append_library)(TB_Linker* l, TB_LinkerThreadInfo* info, TB_Slice ar_name, TB_Slice ar_file); - void (*append_module)(TB_Linker* l, TB_LinkerThreadInfo* info, TB_Module* m); - TB_ExportBuffer (*export)(TB_Linker* l, TB_Arena* dst_arena); -} TB_LinkerVtbl; - -typedef struct TB_Linker { - TB_Arch target_arch; - - const char* entrypoint; - TB_WindowsSubsystem subsystem; - - TB_LinkerVtbl vtbl; - - // we intern symbol strings to make the rest of the - // hash table work easier, it's easier to write a giant - // dumb interner. - struct { - size_t exp, cnt; - TB_LinkerInternStr* arr; - } interner; - - // we track which symbols have not been resolved yet - DynArray(TB_LinkerSectionPiece*) worklist; - - // both use intern str keys - NL_Table symbols; - NL_Table sections; - - size_t trampoline_pos; // relative to the .text section - TB_Emitter trampolines; // these are for calling imported functions - - // for relocations - _Atomic(TB_LinkerThreadInfo*) first_thread_info; - - // Windows specific: - // on windows, we use DLLs to interact with the OS so - // there needs to be a way to load these immediately, - // imports do just that. - // - // this is where all the .reloc stuff from object files goes - TB_LinkerSectionPiece* main_reloc; - uint32_t iat_pos; - DynArray(ImportTable) imports; -} TB_Linker; - -TB_LinkerThreadInfo* linker_thread_info(TB_Linker* l); - -void tb_linker_unresolved_sym(TB_Linker* l, TB_LinkerInternStr name); - -TB_LinkerSectionPiece* tb_linker_get_piece(TB_Linker* l, TB_LinkerSymbol* restrict sym); -void tb_linker_associate(TB_Linker* l, TB_LinkerSectionPiece* a, TB_LinkerSectionPiece* b); - -// TB helpers -size_t tb__get_symbol_pos(TB_Symbol* s); - -// Symbol table -TB_LinkerInternStr tb_linker_intern_string(TB_Linker* l, size_t len, const char* str); -TB_LinkerInternStr tb_linker_intern_cstring(TB_Linker* l, const char* str); -size_t tb_linker_intern_len(TB_Linker* l, TB_LinkerInternStr str); - -TB_LinkerSymbol* tb_linker_new_symbol(TB_Linker* l, TB_LinkerInternStr name); -TB_LinkerSymbol* tb_linker_find_symbol(TB_Linker* l, TB_LinkerInternStr name); -TB_LinkerSymbol* tb_linker_find_symbol2(TB_Linker* l, const char* name); - -// Sections -TB_LinkerSection* tb_linker_find_section(TB_Linker* linker, TB_LinkerInternStr name); -TB_LinkerSection* tb_linker_find_section2(TB_Linker* linker, const char* name); - -TB_LinkerSection* tb_linker_find_or_create_section(TB_Linker* linker, TB_LinkerInternStr name, uint32_t flags); -TB_LinkerSectionPiece* tb_linker_append_piece(TB_LinkerThreadInfo* info, TB_LinkerSection* section, int kind, size_t size, const void* data, TB_LinkerObject* obj); -void tb_linker_merge_sections(TB_Linker* linker, TB_LinkerSection* from, TB_LinkerSection* to); -void tb_linker_append_module_section(TB_Linker* l, TB_LinkerThreadInfo* info, TB_LinkerObject* mod, TB_ModuleSection* section, uint32_t flags); -void tb_linker_append_module_symbols(TB_Linker* l, TB_Module* m); - -uint64_t tb__compute_rva(TB_Linker* l, TB_Module* m, const TB_Symbol* s); -uint64_t tb__get_symbol_rva(TB_Linker* l, TB_LinkerSymbol* sym); - -size_t tb__pad_file(uint8_t* output, size_t write_pos, char pad, size_t align); -void tb_linker_apply_module_relocs(TB_Linker* l, TB_Module* m, TB_LinkerSection* text, uint8_t* output); -size_t tb__apply_section_contents(TB_Linker* l, uint8_t* output, size_t write_pos, TB_LinkerSection* text, TB_LinkerSection* data, TB_LinkerSection* rdata, size_t section_alignment, size_t image_base); - -TB_LinkerSymbol* tb_linker_get_target(TB_LinkerReloc* r); - -void tb_linker_push_piece(TB_Linker* l, TB_LinkerSectionPiece* p); -void tb_linker_push_named(TB_Linker* l, const char* name); -void tb_linker_mark_live(TB_Linker* l); - -// do layouting (requires GC step to complete) -bool tb__finalize_sections(TB_Linker* l); diff --git a/vendor/tb/src/linker/pe.c b/vendor/tb/src/linker/pe.c deleted file mode 100644 index 86a4b5af..00000000 --- a/vendor/tb/src/linker/pe.c +++ /dev/null @@ -1,1150 +0,0 @@ -#define NL_STRING_MAP_IMPL -#include "linker.h" -#include "../objects/coff.h" -#include "../objects/lib_parse.h" - -#include - -typedef struct { - uint16_t page_rva; - uint16_t block_size; // includes the header - uint16_t payload[]; -} BaseRelocSegment; - -enum { IMP_PREFIX_LEN = sizeof("__imp_") - 1 }; - -const static uint8_t dos_stub[] = { - // header - 0x4d,0x5a,0x78,0x00,0x01,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x00,0x00,0x00,0x00, - 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x40,0x00,0x00,0x00,0x00,0x00,0x00,0x00, - 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, - 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x78,0x00,0x00,0x00, - - // machine code - 0x0e,0x1f,0xba,0x0e,0x00,0xb4,0x09,0xcd,0x21,0xb8,0x01,0x4c,0xcd,0x21,0x54,0x68, - 0x69,0x73,0x20,0x70,0x72,0x6f,0x67,0x72,0x61,0x6d,0x20,0x63,0x61,0x6e,0x6e,0x6f, - 0x74,0x20,0x62,0x65,0x20,0x72,0x75,0x6e,0x20,0x69,0x6e,0x20,0x44,0x4f,0x53,0x20, - 0x6d,0x6f,0x64,0x65,0x2e,0x24,0x00,0x00 -}; - -static int symbol_cmp(const void* a, const void* b) { - const TB_ObjectSymbol* sym_a = (const TB_ObjectSymbol*)a; - const TB_ObjectSymbol* sym_b = (const TB_ObjectSymbol*)b; - - return sym_a->ordinal - sym_b->ordinal; -} - -// true if we replace the old one -static bool process_comdat(int select, TB_LinkerSectionPiece* old_p, TB_LinkerSectionPiece* new_p) { - switch (select) { - case 2: return false; // any - case 6: return new_p->size > old_p->size; // largest - default: tb_todo(); - } -} - -// Musl's impl for this -static int string_case_cmp(const char *_l, const char *_r, size_t n) { - const unsigned char *l=(void *)_l, *r=(void *)_r; - if (!n--) return 0; - for (; *l && *r && n && (*l == *r || tolower(*l) == tolower(*r)); l++, r++, n--); - return tolower(*l) - tolower(*r); -} - -static bool strequals(TB_Slice a, const char* b) { - return a.length == strlen(b) && memcmp(a.data, b, a.length) == 0; -} - -static bool strprefix(const char* str, const char* pre, size_t len) { - size_t prelen = strlen(pre); - return string_case_cmp(pre, str, len < prelen ? len : prelen) == 0; -} - -void pe_append_object(TB_Linker* l, TB_LinkerThreadInfo* info, TB_Slice obj_name, TB_Slice content) { - TB_COFF_Parser parser = { obj_name, content }; - tb_coff_parse_init(&parser); - - // We don't *really* care about this info beyond nicer errors - TB_LinkerObject* obj_file = tb_arena_alloc(info->perm_arena, sizeof(TB_LinkerObject)); - *obj_file = (TB_LinkerObject){ .len = obj_name.length, .name = (char*) obj_name.data }; - - TB_ArenaSavepoint sp = tb_arena_save(info->tmp_arena); - - // Apply all sections (generate lookup for sections based on ordinals) - TB_LinkerSectionPiece *text_piece = NULL, *pdata_piece = NULL; - TB_ObjectSection* sections = tb_arena_alloc(info->tmp_arena, parser.section_count * sizeof(TB_ObjectSection)); - FOREACH_N(i, 0, parser.section_count) { - TB_ObjectSection* restrict s = §ions[i]; - tb_coff_parse_section(&parser, i, s); - - // trim the dollar sign (if applies) - uint32_t order = 0; - FOREACH_N(j, 0, s->name.length) { - if (s->name.data[j] == '$') { - // convert letters into score - FOREACH_N(k, j + 1, s->name.length) { - order <<= 8; - order += s->name.data[k]; - } - - s->name.length = j; - break; - } - } - - size_t drectve_len = sizeof(".drectve")-1; - - if (s->name.length == drectve_len && memcmp(s->name.data, ".drectve", drectve_len) == 0 && s->raw_data.length > 3) { - const uint8_t* curr = s->raw_data.data + 3; - const uint8_t* end_directive = s->raw_data.data + s->raw_data.length; - - while (curr != end_directive) { - const uint8_t* end = curr; - while (end != end_directive && *end != ' ') end++; - - log_info("directive: %.*s", (int) (end - curr), curr); - - if (curr == end_directive) { - break; - } else if (strprefix((const char*) curr, "/merge:", end - curr)) { - curr += sizeof("/merge:")-1; - - const uint8_t* equals = curr; - while (*equals && *equals != '=') equals++; - - if (*equals == '=') { - TB_LinkerCmd cmd = { - .from = { curr, equals - curr }, - .to = { equals + 1, (end - equals) - 1 }, - }; - dyn_array_put(info->merges, cmd); - } - } else if (strprefix((const char*) curr, "/defaultlib:", end - curr)) { - curr += sizeof("/defaultlib:")-1; - - // TB_LinkerMsg m = { .tag = TB_LINKER_MSG_IMPORT, .import_path = { end - curr, (const char*) curr } }; - // tb_linker_send_msg(l, &m); - } else if (strprefix((const char*) curr, "/alternatename:", end - curr)) { - curr += sizeof("/alternatename:")-1; - - const uint8_t* equals = curr; - while (*equals && *equals != '=') equals++; - - if (*equals == '=') { - TB_LinkerCmd cmd = { - .from = { curr, equals - curr }, - .to = { equals + 1, (end - equals) - 1 }, - }; - dyn_array_put(info->alternates, cmd); - } - } else { - log_warn("unknown linker directive: %.*s", (int) (end - curr), curr); - break; - } - - curr = end+1; - } - - continue; - } - - // remove all the alignment flags, they don't appear in linker sections - TB_LinkerInternStr str = tb_linker_intern_string(l, s->name.length, (char*) s->name.data); - TB_LinkerSection* ls = tb_linker_find_or_create_section(l, str, s->flags & ~0x00F00000); - - if (s->flags & IMAGE_SCN_LNK_REMOVE) { - ls->generic_flags |= TB_LINKER_SECTION_DISCARD; - } - - if (s->flags & IMAGE_SCN_LNK_COMDAT) { - ls->generic_flags |= TB_LINKER_SECTION_COMDAT; - } - - const void* raw_data = s->raw_data.data; - if (s->flags & IMAGE_SCN_CNT_UNINITIALIZED_DATA) { - raw_data = NULL; - } - - TB_LinkerSectionPiece* p = tb_linker_append_piece(info, ls, PIECE_NORMAL, s->raw_data.length, raw_data, obj_file); - s->user_data = p; - - p->order = order; - p->flags = 1; - // p->vsize = s->virtual_size; - - if (s->name.length == 5 && memcmp(s->name.data, ".text", 5) == 0) { - text_piece = p; - } else if (s->name.length == 6 && memcmp(s->name.data, ".pdata", 6) == 0) { - pdata_piece = p; - } - } - - // associate the pdata with the text - if (text_piece) { - tb_linker_associate(l, text_piece, pdata_piece); - } - - // append all symbols - size_t sym_count = 0; - TB_ObjectSymbol* syms = tb_arena_alloc(info->tmp_arena, sizeof(TB_ObjectSymbol) * parser.symbol_count); - - COFF_AuxSectionSymbol* comdat_aux = NULL; - CUIK_TIMED_BLOCK("apply symbols") { - size_t i = 0; - while (i < parser.symbol_count) { - TB_ObjectSymbol* restrict sym = &syms[sym_count++]; - i += tb_coff_parse_symbol(&parser, i, sym); - - TB_LinkerInternStr name = tb_linker_intern_string(l, sym->name.length, (const char*) sym->name.data); - TB_LinkerSymbol* s = NULL; - - if (sym->section_num > 0) { - TB_ObjectSection* sec = §ions[sym->section_num - 1]; - if (sec->name.length == sym->name.length && memcmp(sym->name.data, sec->name.data, sym->name.length) == 0) { - // we're a section symbol - // COMDAT is how linkers handle merging of inline functions in C++ - if ((sec->flags & IMAGE_SCN_LNK_COMDAT)) { - // the next symbol is the actual COMDAT symbol - comdat_aux = sym->extra; - } - - // sections without a piece are ok - if (sec->user_data == NULL) { - continue; - } - } - - TB_LinkerSectionPiece* p = sec->user_data; - assert(p != NULL); - if (sym->type == TB_OBJECT_SYMBOL_STATIC) { - s = tb_arena_alloc(info->perm_arena, sizeof(TB_LinkerSymbol)); - *s = (TB_LinkerSymbol){ .name = name }; - s->tag = TB_LINKER_SYMBOL_NORMAL; - s->normal.piece = p; - s->normal.secrel = sym->value; - } else if (sym->type == TB_OBJECT_SYMBOL_WEAK_EXTERN) { - if (comdat_aux) { - assert(0 && "COMDAT and weak external?"); - comdat_aux = NULL; - } - - s = tb_linker_new_symbol(l, name); - s->tag = TB_LINKER_SYMBOL_NORMAL; - s->normal.piece = p; - s->normal.secrel = sym->value; - } else if (sym->type == TB_OBJECT_SYMBOL_EXTERN) { - if (comdat_aux) { - // check if it already exists as a COMDAT - s = tb_linker_find_symbol(l, name); - if (s && (s->flags & TB_LINKER_SYMBOL_COMDAT)) { - assert(s->tag == TB_LINKER_SYMBOL_NORMAL); - - bool replace = process_comdat(comdat_aux->selection, s->normal.piece, p); - if (replace) { - s->normal.piece->size = 0; - } else { - p->size = 0; - } - } - - if (s == NULL) { - s = tb_linker_new_symbol(l, name); - s->tag = TB_LINKER_SYMBOL_NORMAL; - s->flags |= TB_LINKER_SYMBOL_COMDAT; - s->normal.piece = p; - s->normal.secrel = sym->value; - } - - comdat_aux = NULL; - } else { - s = tb_linker_find_symbol(l, name); - if (s == NULL) { - s = tb_linker_new_symbol(l, name); - s->tag = TB_LINKER_SYMBOL_NORMAL; - } else if (s->tag == TB_LINKER_SYMBOL_UNKNOWN) { - s->tag = TB_LINKER_SYMBOL_NORMAL; - } - - assert(s->tag == TB_LINKER_SYMBOL_NORMAL); - s->normal.piece = p; - s->normal.secrel = sym->value; - } - } - - // add to the section piece's symbol list - if (s) { - s->next = p->first_sym; - p->first_sym = s; - } - } else if (sym->type == TB_OBJECT_SYMBOL_EXTERN || sym->type == TB_OBJECT_SYMBOL_WEAK_EXTERN) { - // symbols without a section number are proper externals (ones defined somewhere - // else that we might want) - s = tb_linker_find_symbol(l, name); - if (s == NULL) { - s = tb_linker_new_symbol(l, name); - s->tag = TB_LINKER_SYMBOL_UNKNOWN; - } - } else { - log_debug("skipped %s", name); - } - - sym->user_data = s; - } - } - - CUIK_TIMED_BLOCK("parse relocations") FOREACH_N(i, 1, parser.section_count) { - TB_ObjectSection* restrict s = §ions[i]; - TB_LinkerSectionPiece* restrict p = s->user_data; - - assert(p->info == NULL); - p->info = info; - - // some relocations point to sections within the same object file, we resolve - // their symbols early. - FOREACH_N(j, 0, s->relocation_count) { - TB_ObjectReloc* restrict reloc = &s->relocations[j]; - - // resolve address used in relocation, symbols are sorted so we can binary search - TB_ObjectSymbol key = { .ordinal = reloc->symbol_index }; - TB_ObjectSymbol* restrict src_symbol = bsearch(&key, syms, sym_count, sizeof(TB_ObjectSymbol), symbol_cmp); - if (src_symbol->user_data == NULL) { - continue; - } - - TB_LinkerReloc r = { - .type = reloc->type, - .target = src_symbol->user_data, - .src_piece = p, - .src_offset = reloc->virtual_address, - .addend = reloc->addend, - }; - - if (reloc->type == TB_OBJECT_RELOC_ADDR64) { - if (src_symbol->type == TB_OBJECT_SYMBOL_WEAK_EXTERN) { - // weak aux - uint32_t* weak_sym = src_symbol->extra; - - TB_ObjectSymbol* restrict alt_sym = bsearch( - &(TB_ObjectSymbol){ .ordinal = *weak_sym }, - syms, sym_count, sizeof(TB_ObjectSymbol), - symbol_cmp - ); - - r.alt = alt_sym->user_data; - } - } else { - if (src_symbol->type == TB_OBJECT_SYMBOL_WEAK_EXTERN) { - tb_todo(); - } - } - - dyn_array_put(p->inputs, dyn_array_length(info->relocs)); - dyn_array_put(info->relocs, r); - } - } - - tb_arena_restore(info->tmp_arena, sp); -} - -static void pe_append_library(TB_Linker* l, TB_LinkerThreadInfo* info, TB_Slice ar_name, TB_Slice ar_file) { - log_debug("linking against %.*s", (int) ar_name.length, ar_name.data); - - TB_ArchiveFileParser ar_parser = { 0 }; - if (!tb_archive_parse(ar_file, &ar_parser)) { - return; - } - - TB_Arena* perm_arena = info->perm_arena; - TB_Arena* arena = info->tmp_arena; - TB_ArenaSavepoint sp = tb_arena_save(arena); - - TB_ArchiveEntry* entries = tb_arena_alloc(arena, ar_parser.member_count * sizeof(TB_ArchiveEntry)); - size_t new_count; - CUIK_TIMED_BLOCK("parse_entries") { - new_count = tb_archive_parse_entries(&ar_parser, 0, ar_parser.member_count, entries); - } - - FOREACH_N(i, 0, new_count) { - TB_ArchiveEntry* restrict e = &entries[i]; - - if (e->import_name.length) { - // import from DLL - TB_Slice libname = e->name; - ptrdiff_t import_index = -1; - dyn_array_for(j, l->imports) { - ImportTable* table = &l->imports[j]; - - if (table->libpath.length == libname.length && - memcmp(table->libpath.data, libname.data, libname.length) == 0) { - import_index = j; - break; - } - } - - if (import_index < 0) { - // we haven't used this DLL yet, make an import table for it - import_index = dyn_array_length(l->imports); - - ImportTable t = { - .libpath = libname, - .thunks = dyn_array_create(TB_LinkerSymbol*, 4096) - }; - dyn_array_put(l->imports, t); - } - - // make __imp_ form which refers to raw address - size_t newlen = e->import_name.length + sizeof("__imp_") - 1; - uint8_t* newstr = tb_arena_alloc(perm_arena, newlen); - memcpy(newstr, "__imp_", sizeof("__imp_")); - memcpy(newstr + sizeof("__imp_") - 1, e->import_name.data, e->import_name.length); - newstr[newlen] = 0; - - TB_LinkerInternStr name = tb_linker_intern_string(l, newlen, (char*) newstr); - CUIK_TIMED_BLOCK_ARGS("archive", name) { - TB_LinkerSymbol* import_sym = tb_linker_new_symbol(l, name); - import_sym->tag = TB_LINKER_SYMBOL_IMPORT; - import_sym->import.id = import_index; - import_sym->import.ordinal = e->ordinal; - - // make the thunk-like symbol - TB_LinkerSymbol* sym = tb_linker_new_symbol(l, tb_linker_intern_string(l, e->import_name.length, (char*) e->import_name.data)); - sym->tag = TB_LINKER_SYMBOL_THUNK; - sym->thunk = import_sym; - - TB_ArchiveEntry* entries = tb_arena_alloc(arena, ar_parser.member_count * sizeof(TB_ArchiveEntry)); - dyn_array_put(l->imports[import_index].thunks, import_sym); - } - } else { - CUIK_TIMED_BLOCK("append object file") { - pe_append_object(l, info, e->name, e->content); - } - } - } - - tb_arena_restore(arena, sp); -} - -static void pe_append_module(TB_Linker* l, TB_LinkerThreadInfo* info, TB_Module* m) { - m->visited = false; - - // Also resolves internal call patches which is cool - ExportList exports; - CUIK_TIMED_BLOCK("layout section") { - m->exports = tb_module_layout_sections(m); - } - - dyn_array_put(info->ir_modules, m); - - // We don't *really* care about this info beyond nicer errors - TB_LinkerObject* obj_file = tb_arena_alloc(info->perm_arena, sizeof(TB_LinkerObject)); - *obj_file = (TB_LinkerObject){ .module = m }; - - // Convert module into sections which we can then append to the output - DynArray(TB_ModuleSection) sections = m->sections; - dyn_array_for(i, sections) { - uint32_t flags = TB_COFF_SECTION_READ; - if (sections[i].flags & TB_MODULE_SECTION_WRITE) flags |= TB_COFF_SECTION_WRITE; - if (sections[i].flags & TB_MODULE_SECTION_EXEC) flags |= TB_COFF_SECTION_EXECUTE | IMAGE_SCN_CNT_CODE; - if (sections[i].comdat.type != 0) flags |= TB_COFF_SECTION_COMDAT; - - tb_linker_append_module_section(l, info, obj_file, §ions[i], flags); - } - - const ICodeGen* restrict code_gen = tb__find_code_generator(m); - if (m->compiled_function_count > 0 && code_gen->emit_win64eh_unwind_info) { - TB_LinkerInternStr rdata_name = tb_linker_intern_cstring(l, ".rdata"); - TB_LinkerSection* rdata = tb_linker_find_or_create_section(l, rdata_name, IMAGE_SCN_MEM_READ | IMAGE_SCN_CNT_INITIALIZED_DATA); - - CUIK_TIMED_BLOCK("generate xdata") { - TB_Emitter xdata = { 0 }; - - dyn_array_for(i, sections) { - DynArray(TB_FunctionOutput*) funcs = sections[i].funcs; - dyn_array_for(j, funcs) { - TB_FunctionOutput* out_f = funcs[j]; - out_f->unwind_info = xdata.count; - code_gen->emit_win64eh_unwind_info(&xdata, out_f, out_f->stack_usage); - } - } - - TB_LinkerSectionPiece* x = tb_linker_append_piece(info, rdata, PIECE_NORMAL, xdata.count, xdata.data, obj_file); - - TB_LinkerInternStr pdata_name = tb_linker_intern_cstring(l, ".pdata"); - TB_LinkerSection* pdata = tb_linker_find_or_create_section(l, pdata_name, IMAGE_SCN_MEM_READ | IMAGE_SCN_CNT_INITIALIZED_DATA); - TB_LinkerSectionPiece* pdata_piece = tb_linker_append_piece(info, pdata, PIECE_PDATA, m->compiled_function_count * 12, NULL, obj_file); - - tb_linker_associate(l, pdata_piece, x); - - dyn_array_for(i, sections) { - DynArray(TB_FunctionOutput*) funcs = sections[i].funcs; - if (dyn_array_length(funcs) && sections[i].piece) { - tb_linker_associate(l, sections[i].piece, pdata_piece); - } - } - m->xdata = x; - } - } - - CUIK_TIMED_BLOCK(".reloc") { - uint32_t last_page = UINT32_MAX, reloc_size = 0; - dyn_array_for(i, m->sections) { - DynArray(TB_Global*) globals = m->sections[i].globals; - dyn_array_for(j, globals) { - TB_Global* g = globals[j]; - FOREACH_N(k, 0, g->obj_count) { - size_t actual_page = g->pos + g->objects[k].offset; - - if (g->objects[k].type == TB_INIT_OBJ_RELOC) { - if (last_page != actual_page) { - last_page = actual_page; - reloc_size += 8; - } - - reloc_size += 2; - } - } - } - - if (reloc_size > 0) { - TB_LinkerInternStr reloc_name = tb_linker_intern_cstring(l, ".reloc"); - TB_LinkerSection* reloc = tb_linker_find_or_create_section(l, reloc_name, IMAGE_SCN_MEM_READ | IMAGE_SCN_CNT_INITIALIZED_DATA); - TB_LinkerSectionPiece* reloc_piece = tb_linker_append_piece(info, reloc, PIECE_RELOC, reloc_size, NULL, obj_file); - - tb_linker_associate(l, sections[i].piece, reloc_piece); - } - } - } - - tb_linker_append_module_symbols(l, m); -} - -// just run whatever reloc function from the spec -static int32_t resolve_reloc(TB_LinkerSymbol* sym, TB_ObjectRelocType type, uint32_t source_pos, uint32_t target_rva, int addend) { - switch (type) { - case TB_OBJECT_RELOC_ADDR32NB: - return target_rva; - - case TB_OBJECT_RELOC_SECTION: - return sym->normal.piece->parent->number; - - case TB_OBJECT_RELOC_SECREL: - return sym->normal.piece->parent->address; - - case TB_OBJECT_RELOC_REL32: - return target_rva - (source_pos + addend); - - default: - tb_todo(); - } -} - -static void apply_external_relocs(TB_Linker* l, uint8_t* output, uint64_t image_base) { - TB_LinkerSection* text = tb_linker_find_section2(l, ".text"); - uint32_t trampoline_rva = text->address + l->trampoline_pos; - uint32_t iat_pos = l->iat_pos; - - // TODO(NeGate): we can multithread this code with a job stealing queue - for (TB_LinkerThreadInfo* restrict info = l->first_thread_info; info; info = info->next) { - dyn_array_for(i, info->ir_modules) { - tb_linker_apply_module_relocs(l, info->ir_modules[i], text, output); - } - - // relative relocations - dyn_array_for(i, info->relocs) { - TB_LinkerReloc* restrict rel = &info->relocs[i]; - if (rel->type == TB_OBJECT_RELOC_ADDR64) continue; - if ((rel->src_piece->flags & TB_LINKER_PIECE_LIVE) == 0) continue; - - // resolve source location - uint32_t target_rva = 0; - TB_LinkerSymbol* sym = tb_linker_get_target(rel); - if (sym == NULL) { - tb_linker_unresolved_sym(l, rel->target->name); - continue; - } - - assert(sym->tag != TB_LINKER_SYMBOL_UNKNOWN); - if (sym->tag == TB_LINKER_SYMBOL_IMPORT) { - target_rva = iat_pos + (sym->import.thunk_id * 8); - } else if (sym->tag == TB_LINKER_SYMBOL_THUNK) { - TB_LinkerSymbol* import_sym = sym->thunk; - target_rva = trampoline_rva + (import_sym->import.thunk_id * 6); - } else { - target_rva = tb__get_symbol_rva(l, sym); - } - - // find patch position - TB_LinkerSectionPiece* restrict p = rel->src_piece; - TB_LinkerSection* restrict s = p->parent; - - int32_t* dst = (int32_t*) &output[s->offset + p->offset + rel->src_offset]; - uint32_t actual_pos = s->address + p->offset + rel->src_offset; - - *dst += resolve_reloc(sym, rel->type, actual_pos, target_rva, rel->addend); - } - } - - // this part will probably stay single threaded for simplicity - if (l->main_reloc) { - uint8_t* p_start = (uint8_t*) &output[l->main_reloc->parent->offset + l->main_reloc->offset]; - uint8_t* p_out = p_start; - - uint32_t last_page = 0xFFFFFFFF; - uint32_t* last_block = NULL; - TB_LinkerSectionPiece* last_piece = NULL; - for (TB_LinkerThreadInfo* restrict info = l->first_thread_info; info; info = info->next) { - dyn_array_for(i, info->relocs) { - TB_LinkerReloc* restrict r = &info->relocs[i]; - if (r->type != TB_OBJECT_RELOC_ADDR64) continue; - - TB_LinkerSectionPiece* restrict p = r->src_piece; - if ((p->flags & TB_LINKER_PIECE_LIVE) == 0) continue; - - TB_LinkerSection* restrict s = p->parent; - if (s->generic_flags & TB_LINKER_SECTION_DISCARD) continue; - - // patch RVA - size_t actual_pos = p->offset + r->src_offset; - uint32_t file_pos = s->offset + actual_pos; - uint64_t* reloc = (uint64_t*) &output[file_pos]; - - TB_LinkerSymbol* sym = tb_linker_get_target(r); - if (sym == NULL) { - tb_linker_unresolved_sym(l, r->target->name); - continue; - } - - assert(sym->tag != TB_LINKER_SYMBOL_UNKNOWN); - if (sym->tag == TB_LINKER_SYMBOL_ABSOLUTE) { - *reloc += sym->absolute; - continue; - } else if (sym->tag == TB_LINKER_SYMBOL_IMPORT) { - *reloc += iat_pos + (sym->import.thunk_id * 8); - } else if (sym->tag == TB_LINKER_SYMBOL_THUNK) { - TB_LinkerSymbol* import_sym = sym->thunk; - *reloc += trampoline_rva + (import_sym->import.thunk_id * 6); - } else { - *reloc += image_base + tb__get_symbol_rva(l, sym); - } - - // generate relocation - size_t actual_page = actual_pos & ~4095; - size_t page_offset = actual_pos - actual_page; - - if (last_piece != p && last_page != actual_page) { - last_piece = p; - last_page = actual_page; - last_block = (uint32_t*) p_out; - - last_block[0] = s->address + actual_page; - last_block[1] = 8; // block size field (includes RVA field and itself) - p_out += 8; - } - - // emit relocation - uint16_t payload = (10 << 12) | page_offset; // (IMAGE_REL_BASED_DIR64 << 12) | offset - *((uint16_t*) p_out) = payload, p_out += 2; - last_block[1] += 2; - } - } - - size_t actual_reloc_size = p_out - p_start; - (void) actual_reloc_size; - assert(actual_reloc_size == l->main_reloc->size); - } -} - -// returns the two new section pieces for the IAT and ILT -static COFF_ImportDirectory* gen_imports(TB_Linker* l, PE_ImageDataDirectory* imp_dir, PE_ImageDataDirectory* iat_dir) { - // cull unused imports - size_t j = 0; - size_t import_entry_count = 0; - dyn_array_for(i, l->imports) { - DynArray(TB_LinkerSymbol*) tbl = l->imports[j].thunks; - - size_t cnt = 0; - dyn_array_for(k, tbl) if (tbl[k]->flags & TB_LINKER_SYMBOL_USED) { - tbl[cnt++] = tbl[k]; - } - dyn_array_set_length(tbl, cnt); - l->imports[i].thunks = tbl; - - if (dyn_array_length(l->imports[i].thunks) != 0) { - l->imports[j++] = l->imports[i]; - - // there's an extra NULL terminator for the import entry lists - import_entry_count += dyn_array_length(l->imports[i].thunks) + 1; - } - } - - if (l->imports) { - dyn_array_set_length(l->imports, j); // trimmed - } - - if (import_entry_count == 0) { - *imp_dir = (PE_ImageDataDirectory){ 0 }; - *iat_dir = (PE_ImageDataDirectory){ 0 }; - return NULL; - } - - // Generate import thunks - uint32_t thunk_id_counter = 0; - l->trampolines = (TB_Emitter){ 0 }; - dyn_array_for(i, l->imports) { - ImportTable* imp = &l->imports[i]; - - dyn_array_for(j, imp->thunks) { - imp->thunks[j]->import.ds_address = l->trampolines.count; - imp->thunks[j]->import.thunk_id = thunk_id_counter++; - - // TODO(NeGate): This trampoline is x64 specific, we should - // implement a system to separate this from the core PE export - // code. - tb_out1b(&l->trampolines, 0xFF); - tb_out1b(&l->trampolines, 0x25); - // we're gonna relocate this to map onto an import thunk later - tb_out4b(&l->trampolines, 0); - } - } - - //////////////////////////////// - // Generate import table - //////////////////////////////// - size_t import_dir_size = (1 + dyn_array_length(l->imports)) * sizeof(COFF_ImportDirectory); - size_t iat_size = import_entry_count * sizeof(uint64_t); - size_t total_size = import_dir_size + 2*iat_size; - dyn_array_for(i, l->imports) { - ImportTable* imp = &l->imports[i]; - total_size += imp->libpath.length + 1; - - dyn_array_for(j, imp->thunks) { - TB_LinkerSymbol* t = imp->thunks[j]; - total_size += tb_linker_intern_len(l, t->name) + 3; - } - } - - TB_LinkerThreadInfo* info = linker_thread_info(l); - uint8_t* output = tb_arena_alloc(info->perm_arena, total_size); - - COFF_ImportDirectory* import_dirs = (COFF_ImportDirectory*) &output[0]; - uint64_t* iat = (uint64_t*) &output[import_dir_size]; - uint64_t* ilt = (uint64_t*) &output[import_dir_size + iat_size]; - size_t strtbl_pos = import_dir_size + iat_size*2; - - // We put both the IAT and ILT into the rdata, the PE loader doesn't care but it - // means the user can't edit these... at least not easily - TB_LinkerInternStr rdata_name = tb_linker_intern_cstring(l, ".rdata"); - TB_LinkerSection* rdata = tb_linker_find_or_create_section(l, rdata_name, IMAGE_SCN_MEM_READ | IMAGE_SCN_CNT_INITIALIZED_DATA); - TB_LinkerSectionPiece* import_piece = tb_linker_append_piece(info, rdata, PIECE_NORMAL, total_size, output, 0); - - *imp_dir = (PE_ImageDataDirectory){ import_piece->offset, import_dir_size }; - *iat_dir = (PE_ImageDataDirectory){ import_piece->offset + import_dir_size, iat_size }; - - size_t p = 0; - dyn_array_for(i, l->imports) { - ImportTable* imp = &l->imports[i]; - COFF_ImportDirectory* header = &import_dirs[i]; - TB_Slice lib = imp->libpath; - - // after we resolve RVAs we need to backpatch stuff - imp->iat = &iat[p], imp->ilt = &ilt[p]; - - *header = (COFF_ImportDirectory){ - .import_lookup_table = import_piece->offset + import_dir_size + iat_size + p*sizeof(uint64_t), - .import_address_table = import_piece->offset + import_dir_size + p*sizeof(uint64_t), - .name = import_piece->offset + strtbl_pos, - }; - - memcpy(&output[strtbl_pos], lib.data, lib.length), strtbl_pos += lib.length; - output[strtbl_pos++] = 0; - - dyn_array_for(j, imp->thunks) { - TB_LinkerSymbol* t = imp->thunks[j]; - - const char* name = t->name + IMP_PREFIX_LEN; - size_t len = tb_linker_intern_len(l, t->name) - IMP_PREFIX_LEN; - - // import-by-name - uint64_t value = import_piece->offset + strtbl_pos; - memcpy(&output[strtbl_pos], &t->import.ordinal, sizeof(uint16_t)), strtbl_pos += 2; - memcpy(&output[strtbl_pos], name, len), strtbl_pos += len; - output[strtbl_pos++] = 0; - - // both the ILT and IAT are practically identical at this point - iat[p] = ilt[p] = value, p++; - } - - // NULL terminated - iat[p] = ilt[p] = 0, p++; - } - assert(p == import_entry_count); - - // add NULL import directory at the end - import_dirs[dyn_array_length(l->imports)] = (COFF_ImportDirectory){ 0 }; - - // trampolines require .text section - if (l->trampolines.count) { - TB_LinkerSection* text = tb_linker_find_section(l, tb_linker_intern_cstring(l, ".text")); - assert(text != NULL); - - TB_LinkerSectionPiece* piece = tb_linker_append_piece(info, text, PIECE_NORMAL, l->trampolines.count, l->trampolines.data, 0); - l->trampoline_pos = piece->offset; - } - - return import_dirs; -} - -static void* gen_reloc_section(TB_Linker* l) { - // generates .reloc for all object files - uint32_t reloc_sec_size = 0; - uint32_t last_page = UINT32_MAX; - TB_LinkerSectionPiece* last_piece = NULL; - - for (TB_LinkerThreadInfo* restrict info = l->first_thread_info; info; info = info->next) { - dyn_array_for(i, info->relocs) { - TB_LinkerReloc* restrict r = &info->relocs[i]; - if (r->type == TB_OBJECT_RELOC_ADDR64) continue; - if ((r->src_piece->flags & TB_LINKER_PIECE_LIVE) == 0) continue; - - TB_LinkerSection* s = r->src_piece->parent; - if (s->generic_flags & TB_LINKER_SECTION_DISCARD) continue; - if (r->target == NULL || r->target->tag == TB_LINKER_SYMBOL_ABSOLUTE) continue; - - size_t actual_pos = r->src_piece->offset + r->src_offset; - size_t actual_page = actual_pos & ~4095; - - if (last_piece != r->src_piece && last_page != actual_page) { - last_piece = r->src_piece; - last_page = actual_page; - - reloc_sec_size += 8; - } - - reloc_sec_size += 2; - } - } - - if (reloc_sec_size) { - TB_LinkerThreadInfo* info = linker_thread_info(l); - void* reloc = tb_arena_alloc(info->perm_arena, reloc_sec_size); - - TB_LinkerSection* reloc_sec = tb_linker_find_or_create_section(l, ".reloc", IMAGE_SCN_MEM_READ | IMAGE_SCN_CNT_INITIALIZED_DATA); - l->main_reloc = tb_linker_append_piece(info, reloc_sec, PIECE_NORMAL, reloc_sec_size, reloc, 0); - - return reloc; - } else { - return NULL; - } -} - -#define CSTRING(str) { sizeof(str)-1, (const uint8_t*) str } -static void pe_init(TB_Linker* l) { - l->entrypoint = "mainCRTStartup"; - l->subsystem = TB_WIN_SUBSYSTEM_CONSOLE; - - #define symbol(name_) tb_linker_new_symbol(l, tb_linker_intern_cstring(l, name_)) - symbol("__ImageBase")->tag = TB_LINKER_SYMBOL_IMAGEBASE; - - // This is practically just ripped from LLD - // https://github.com/llvm/llvm-project/blob/3d0a5bf7dea509f130c51868361a38daeee7816a/lld/COFF/Driver.cpp#L2192 - symbol("__AbsoluteZero"); - // add_abs("__volatile_metadata"); - // add_abs("__guard_memcpy_fptr"); - symbol("__guard_fids_count"); - symbol("__guard_fids_table"); - symbol("__guard_flags"); - symbol("__guard_iat_count"); - symbol("__guard_iat_table"); - symbol("__guard_longjmp_count"); - symbol("__guard_longjmp_table"); - // Needed for MSVC 2017 15.5 CRT. - symbol("__enclave_config"); - // Needed for MSVC 2019 16.8 CRT. - symbol("__guard_eh_cont_count"); - symbol("__guard_eh_cont_table"); - #undef symbol -} - -#define WRITE(data, size) (memcpy(&output[write_pos], data, size), write_pos += (size)) -static TB_ExportBuffer pe_export(TB_Linker* l, TB_Arena* arena) { - PE_ImageDataDirectory imp_dir, iat_dir; - COFF_ImportDirectory* import_dirs; - - for (TB_LinkerThreadInfo* restrict info = l->first_thread_info; info; info = info->next) { - dyn_array_for(i, info->alternates) { - TB_Slice to = info->alternates[i].to; - TB_Slice from = info->alternates[i].from; - - TB_LinkerInternStr to_str = tb_linker_intern_string(l, to.length, (const char*) to.data); - TB_LinkerInternStr from_str = tb_linker_intern_string(l, from.length, (const char*) from.data); - - TB_LinkerSymbol* old = tb_linker_find_symbol(l, to_str); - if (old) { - TB_LinkerSymbol* sym = tb_linker_find_symbol(l, from_str); - assert(sym->tag == TB_LINKER_SYMBOL_UNKNOWN); - - *sym = *old; - sym->name = from_str; - } - } - - /* dyn_array_for(i, info->merges) { - NL_Slice to_name = { info->merges[i].to.length, info->merges[i].to.data }; - ptrdiff_t to = nl_map_get(l->sections, to_name); - if (to < 0) continue; - - NL_Slice from_name = { info->merges[i].from.length, info->merges[i].from.data }; - ptrdiff_t from = nl_map_get(l->sections, from_name); - if (from < 0) continue; - - tb__merge_sections(l, l->sections[from].v, l->sections[to].v); - } */ - } - - // personally like merging these - TB_LinkerSection* rdata = tb_linker_find_section2(l, ".rdata"); - tb_linker_merge_sections(l, tb_linker_find_section2(l, ".00cfg"), rdata); - tb_linker_merge_sections(l, tb_linker_find_section2(l, ".idata"), rdata); - tb_linker_merge_sections(l, tb_linker_find_section2(l, ".xdata"), rdata); - - // this will resolve the sections, GC any pieces which aren't used and - // resolve symbols. - CUIK_TIMED_BLOCK("Resolve & GC") { - tb_linker_push_named(l, l->entrypoint); - tb_linker_push_named(l, "_tls_used"); - tb_linker_push_named(l, "_load_config_used"); - tb_linker_mark_live(l); - } - - if (!tb__finalize_sections(l)) { - return (TB_ExportBuffer){ 0 }; - } - - CUIK_TIMED_BLOCK("generate imports") { - import_dirs = gen_imports(l, &imp_dir, &iat_dir); - } - - CUIK_TIMED_BLOCK("generate .reloc") { - gen_reloc_section(l); - } - - size_t final_section_count = 0; - nl_table_for(e, &l->sections) { - TB_LinkerSection* sec = e->v; - final_section_count += (sec->generic_flags & TB_LINKER_SECTION_DISCARD) == 0; - } - - size_t size_of_headers = sizeof(dos_stub) - + sizeof(uint32_t) // PE magic number - + sizeof(COFF_FileHeader) - + sizeof(PE_OptionalHeader64) - + (final_section_count * sizeof(PE_SectionHeader)); - - size_of_headers = align_up(size_of_headers, 512); - - size_t pe_code_size = 0; // bytes in total marked as IMAGE_SCN_CNT_CODE - size_t pe_init_size = 0; // bytes in total marked as IMAGE_SCN_CNT_INITIALIZED_DATA - size_t pe_uninit_size = 0; // bytes in total marked as IMAGE_SCN_CNT_UNINITIALIZED_DATA - - size_t section_content_size = 0; - uint64_t virt_addr = align_up(size_of_headers, 4096); // this area is reserved for the PE header stuff - CUIK_TIMED_BLOCK("layout sections") { - nl_table_for(e, &l->sections) { - TB_LinkerSection* s = e->v; - if (s->generic_flags & TB_LINKER_SECTION_DISCARD) continue; - - if (s->flags & IMAGE_SCN_CNT_CODE) pe_code_size += s->total_size; - if (s->flags & IMAGE_SCN_CNT_INITIALIZED_DATA) pe_init_size += s->total_size; - if (s->flags & IMAGE_SCN_CNT_UNINITIALIZED_DATA) pe_uninit_size += s->total_size; - - s->offset = size_of_headers + section_content_size; - if ((s->flags & IMAGE_SCN_CNT_UNINITIALIZED_DATA) == 0) { - section_content_size += align_up(s->total_size, 512); - } - - s->address = virt_addr; - virt_addr += align_up(s->total_size, 4096); - } - } - - TB_LinkerSection* text = tb_linker_find_section2(l, ".text"); - TB_LinkerSection* data = tb_linker_find_section2(l, ".data"); - - if (import_dirs != NULL) { - iat_dir.virtual_address += rdata->address; - imp_dir.virtual_address += rdata->address; - CUIK_TIMED_BLOCK("relocate imports and trampolines") { - dyn_array_for(i, l->imports) { - ImportTable* imp = &l->imports[i]; - COFF_ImportDirectory* header = &import_dirs[i]; - header->import_lookup_table += rdata->address; - header->import_address_table += rdata->address; - header->name += rdata->address; - - uint64_t *ilt = imp->ilt, *iat = imp->iat; - uint64_t iat_rva = header->import_address_table; - uint64_t trampoline_rva = text->address + l->trampoline_pos; - - dyn_array_for(j, imp->thunks) { - if (iat[j] != 0) { - iat[j] += rdata->address; - ilt[j] += rdata->address; - } - - // Relocate trampoline entries to point into the IAT, the PE loader will - // fill these slots in with absolute addresses of the symbols. - int32_t* trampoline_dst = (int32_t*) &l->trampolines.data[imp->thunks[j]->import.ds_address + 2]; - assert(*trampoline_dst == 0 && "We set this earlier... why isn't it here?"); - - uint32_t target_rva = iat_rva + j*8; - uint32_t source_pos = trampoline_rva + imp->thunks[j]->import.ds_address + 6; - (*trampoline_dst) += target_rva - source_pos; - } - } - } - l->iat_pos = iat_dir.virtual_address; - } - - size_t output_size = size_of_headers + section_content_size; - COFF_FileHeader header = { - .machine = 0x8664, - .section_count = final_section_count, - .timestamp = 1056582000u, // my birthday since that's consistent :P - .symbol_table = 0, - .symbol_count = 0, - .optional_header_size = sizeof(PE_OptionalHeader64), - .flags = 0x2 | 0x0020 /* executable, >2GB */ - }; - - if (l->subsystem == TB_WIN_SUBSYSTEM_EFI_APP) { - header.flags |= 0x2000; // DLL - } - - static const uint32_t subsys[] = { - [TB_WIN_SUBSYSTEM_WINDOWS] = IMAGE_SUBSYSTEM_WINDOWS_GUI, - [TB_WIN_SUBSYSTEM_CONSOLE] = IMAGE_SUBSYSTEM_WINDOWS_CUI, - [TB_WIN_SUBSYSTEM_EFI_APP] = IMAGE_SUBSYSTEM_EFI_APPLICATION, - }; - - PE_OptionalHeader64 opt_header = { - .magic = 0x20b, - .section_alignment = 0x1000, - .file_alignment = 0x200, - - .image_base = 0x140000000, - - .size_of_code = pe_code_size, - .size_of_initialized_data = pe_init_size, - .size_of_uninitialized_data = pe_uninit_size, - - .size_of_image = virt_addr, - .size_of_headers = (size_of_headers + 0x1FF) & ~0x1FF, - .subsystem = subsys[l->subsystem], - .dll_characteristics = 0x40 | 0x20 | 0x0100 | 0x8000, /* dynamic base, high entropy, nx compat, terminal server aware */ - - .size_of_stack_reserve = 2 << 20, - .size_of_stack_commit = 4096, - - .rva_size_count = IMAGE_NUMBEROF_DIRECTORY_ENTRIES, - .data_directories = { - [IMAGE_DIRECTORY_ENTRY_IMPORT] = imp_dir, - [IMAGE_DIRECTORY_ENTRY_IAT] = iat_dir, - } - }; - - if (l->subsystem != TB_WIN_SUBSYSTEM_EFI_APP) { - opt_header.major_os_ver = 6; - opt_header.minor_os_ver = 0; - opt_header.major_subsystem_ver = 6; - opt_header.minor_subsystem_ver = 0; - } - - TB_LinkerSymbol* tls_used_sym = tb_linker_find_symbol2(l, "_tls_used"); - if (tls_used_sym) { - opt_header.data_directories[IMAGE_DIRECTORY_ENTRY_TLS] = (PE_ImageDataDirectory){ tb__get_symbol_rva(l, tls_used_sym), sizeof(PE_TLSDirectory) }; - } - - TB_LinkerSymbol* load_config_used_sym = tb_linker_find_symbol2(l, "_load_config_used"); - if (load_config_used_sym) { - opt_header.data_directories[IMAGE_DIRECTORY_ENTRY_LOAD_CONFIG] = (PE_ImageDataDirectory){ tb__get_symbol_rva(l, load_config_used_sym), 0x140 }; - } - - TB_LinkerSection* pdata = tb_linker_find_section2(l, ".pdata"); - if (pdata) { - opt_header.data_directories[IMAGE_DIRECTORY_ENTRY_EXCEPTION] = (PE_ImageDataDirectory){ pdata->address, pdata->total_size }; - } - - TB_LinkerSection* reloc = tb_linker_find_section2(l, ".reloc"); - if (reloc) { - opt_header.data_directories[IMAGE_DIRECTORY_ENTRY_BASERELOC] = (PE_ImageDataDirectory){ reloc->address, reloc->total_size }; - } - - // text section crap - if (text) { - opt_header.base_of_code = text->address; - opt_header.size_of_code = align_up(text->total_size, 4096); - - TB_LinkerSymbol* sym = tb_linker_find_symbol2(l, l->entrypoint); - if (sym) { - if (sym->tag == TB_LINKER_SYMBOL_NORMAL) { - opt_header.entrypoint = text->address + sym->normal.piece->offset + sym->normal.secrel; - } else if (sym->tag == TB_LINKER_SYMBOL_TB) { - opt_header.entrypoint = text->address + sym->tb.piece->offset + tb__get_symbol_pos(sym->tb.sym); - } else { - tb_todo(); - } - } else { - printf("tblink: could not find entrypoint! %s\n", l->entrypoint); - } - } - - size_t write_pos = 0; - TB_ExportChunk* chunk = tb_export_make_chunk(arena, output_size); - uint8_t* restrict output = chunk->data; - - uint32_t pe_magic = 0x00004550; - WRITE(dos_stub, sizeof(dos_stub)); - WRITE(&pe_magic, sizeof(pe_magic)); - WRITE(&header, sizeof(header)); - WRITE(&opt_header, sizeof(opt_header)); - - nl_table_for(e, &l->sections) { - TB_LinkerSection* s = e->v; - if (s->generic_flags & TB_LINKER_SECTION_DISCARD) continue; - - PE_SectionHeader sec_header = { - .virtual_size = align_up(s->total_size, 4096), - .virtual_address = s->address, - .characteristics = s->flags, - }; - - if ((s->flags & IMAGE_SCN_CNT_UNINITIALIZED_DATA) == 0) { - sec_header.pointer_to_raw_data = s->offset; - sec_header.size_of_raw_data = s->total_size; - } - - size_t len = tb_linker_intern_len(l, s->name); - memcpy(sec_header.name, s->name, len > 8 ? 8 : len); - WRITE(&sec_header, sizeof(sec_header)); - } - write_pos = tb__pad_file(output, write_pos, 0x00, 0x200); - - tb__apply_section_contents(l, output, write_pos, text, data, rdata, 512, opt_header.image_base); - - TB_ExportBuffer chk = { .total = output_size, .head = chunk, .tail = chunk }; - - // TODO(NeGate): multithread this too - CUIK_TIMED_BLOCK("apply final relocations") { - apply_external_relocs(l, output, opt_header.image_base); - } - - return chk; -} - -TB_LinkerVtbl tb__linker_pe = { - .init = pe_init, - .append_object = pe_append_object, - .append_library = pe_append_library, - .append_module = pe_append_module, - .export = pe_export -}; diff --git a/vendor/tb/src/lsra.c b/vendor/tb/src/lsra.c deleted file mode 100644 index 1084e11f..00000000 --- a/vendor/tb/src/lsra.c +++ /dev/null @@ -1,942 +0,0 @@ -// Linear scan register allocator: -// https://ssw.jku.at/Research/Papers/Wimmer04Master/Wimmer04Master.pdf -#include "codegen.h" - -#define FOREACH_SET(it, set) \ -FOREACH_N(_i, 0, ((set).capacity + 63) / 64) FOREACH_BIT(it, _i*64, (set).data[_i]) - -typedef struct { - Ctx* ctx; - TB_Arena* arena; - - int spills; - int num_classes; - int* num_regs; - uint64_t* callee_saved; - - // time when the physical registers will be free again - int* free_pos; - int* block_pos; - - size_t stack_slot_cap; - - // waiting to get registers, sorted such that the top most item is the youngest - DynArray(LiveInterval*) unhandled; - DynArray(LiveInterval*) inactive; - DynArray(CalleeSpill) callee_spills; - RegMask* normie_mask; - - Set active_set[MAX_REG_CLASSES]; - LiveInterval** active[MAX_REG_CLASSES]; - LiveInterval* fixed[MAX_REG_CLASSES]; -} LSRA; - -// Forward decls... yay -static void insert_split_move(LSRA* restrict ra, int t, LiveInterval* old_it, LiveInterval* new_it); -static void cuiksort_defs(LiveInterval** intervals, ptrdiff_t lo, ptrdiff_t hi); -static bool update_interval(LSRA* restrict ra, LiveInterval* interval, bool is_active, int time, int inactive_index); -static ptrdiff_t allocate_free_reg(LSRA* restrict ra, LiveInterval* interval); -static LiveInterval* split_intersecting(LSRA* restrict ra, int pos, LiveInterval* interval, RegMask new_mask); -static void move_to_active(LSRA* restrict ra, LiveInterval* interval); - -static const char* GPR_NAMES[] = { "RAX", "RCX", "RDX", "RBX", "RSP", "RBP", "RSI", "RDI", "R8", "R9", "R10", "R11", "R12", "R13", "R14", "R15" }; - -// static const char* GPR_NAMES[] = { "X0", "X1", "X2", "X3", "X4", "X5", "X6", "X7", "X8", "X9", "X10", "X11", "X12", "X13", "X14", "X15" }; -static void print_reg_name(int rg, int num) { - if (rg == 1) { - printf("R%d", num); - // printf("%s", GPR_NAMES[num]); - } else if (rg == 2) { - printf("XMM%d", num); - } else if (rg == REG_CLASS_STK) { - printf("[sp + %d]", num*8); - } else { - tb_todo(); - } -} - -// Helpers -static int interval_start(LiveInterval* l) { return l->ranges[l->range_count - 1].start; } -static int interval_end(LiveInterval* l) { return l->ranges[1].end; } -static int interval_class(LiveInterval* l) { return l->mask.class; } - -static LiveInterval* split_interval_at(LiveInterval* interval, int pos) { - if (interval == NULL) { - return interval; - } - - // skip past previous intervals - while (interval->split_kid && pos > interval_end(interval)) { - interval = interval->split_kid; - } - - // assert(interval->reg >= 0 || pos <= interval_end(interval)); - return interval; -} - -static int range_intersect(LiveRange* a, LiveRange* b) { - if (b->start <= a->end && a->start <= b->end) { - return a->start > b->start ? a->start : b->start; - } else { - return -1; - } -} - -static int interval_intersect(LiveInterval* a, LiveInterval* b) { - FOREACH_REVERSE_N(i, 1, a->active_range+1) { - FOREACH_REVERSE_N(j, 1, b->active_range+1) { - int t = range_intersect(&a->ranges[i], &b->ranges[j]); - if (t >= 0) { - return t; - } - } - } - - return -1; -} - -static void add_use_pos(LSRA* restrict ra, LiveInterval* interval, int t, bool may_spill) { - if (interval->use_cap == interval->use_count) { - if (interval->use_cap == 0) { - interval->use_cap = 4; - } else { - interval->use_cap *= 2; - } - interval->uses = tb_arena_realloc(ra->arena, interval->uses, interval->use_cap * sizeof(UsePos)); - } - - interval->uses[interval->use_count++] = (UsePos){ may_spill, t }; -} - -static void add_range(LSRA* restrict ra, LiveInterval* interval, int start, int end) { - assert(start <= end); - assert(interval->range_count > 0); - - if (interval->ranges[interval->range_count - 1].start <= end) { - LiveRange* top = &interval->ranges[interval->range_count - 1]; - - // coalesce - top->start = TB_MIN(top->start, start); - top->end = TB_MAX(top->end, end); - } else { - if (interval->range_cap == interval->range_count) { - interval->range_cap *= 2; - interval->ranges = tb_arena_realloc(ra->arena, interval->ranges, interval->range_cap * sizeof(LiveRange)); - } - - interval->active_range = interval->range_count; - interval->ranges[interval->range_count++] = (LiveRange){ start, end }; - } -} - -LiveInterval* gimme_interval_for_mask(Ctx* restrict ctx, TB_Arena* arena, LSRA* restrict ra, RegMask mask, TB_DataType dt) { - // not so fixed interval? we need a unique interval then - int reg = fixed_reg_mask(mask); - if (reg >= 0) { - return &ctx->fixed[mask.class][reg]; - } else { - LiveInterval* interval = tb_arena_alloc(arena, sizeof(LiveInterval)); - *interval = (LiveInterval){ - .id = ctx->interval_count++, - .mask = mask, - .dt = dt, - .hint = NULL, - .reg = -1, - .assigned = -1, - .range_cap = 4, .range_count = 1, - .ranges = tb_arena_alloc(arena, 4 * sizeof(LiveRange)) - }; - interval->ranges[0] = (LiveRange){ INT_MAX, INT_MAX }; - return interval; - } -} - -static Tile* tile_at_time(LSRA* restrict ra, int t) { - // find which BB - MachineBB* mbb = NULL; - FOREACH_N(i, 0, ra->ctx->bb_count) { - mbb = &ra->ctx->machine_bbs[i]; - if (t <= mbb->end->time) break; - } - - Tile* curr = mbb->start; - while (curr != NULL) { - if (curr->time > t) { - return curr; - } - curr = curr->next; - } - - return NULL; -} - -void tb__lsra(Ctx* restrict ctx, TB_Arena* arena) { - LSRA ra = { .ctx = ctx, .arena = arena, .spills = ctx->num_regs[0] }; - int initial_spills = ctx->num_regs[0]; - - // create timeline & insert moves - CUIK_TIMED_BLOCK("insert legalizing moves") { - int timeline = 4; - FOREACH_N(i, 0, ctx->bb_count) { - MachineBB* mbb = &ctx->machine_bbs[i]; - - for (Tile* t = mbb->start; t; t = t->next) { - // insert input copies - FOREACH_N(j, 0, t->in_count) { - LiveInterval* in_def = t->ins[j].src; - RegMask in_mask = t->ins[j].mask; - int hint = fixed_reg_mask(in_mask); - - if (in_def == NULL) { - continue; - } - - RegMask in_def_mask = in_def->mask; - if (hint >= 0) { - in_def->hint = &ctx->fixed[in_mask.class][hint]; - } - - bool both_fixed = hint >= 0 && in_def_mask.mask == in_mask.mask; - - // we resolve def-use conflicts with a spill move, either when: - // * the use and def classes don't match. - // * the use mask is more constrained than the def. - // * it's on both ends to avoid stretching fixed intervals. - if (in_def_mask.class != in_mask.class || (in_def_mask.mask & in_mask.mask) != in_def_mask.mask || both_fixed) { - if (both_fixed) { - in_def_mask = ctx->normie_mask[in_def_mask.class]; - } - - TB_OPTDEBUG(REGALLOC)(printf(" TEMP "), tb__print_regmask(in_def_mask), printf(" -> "), tb__print_regmask(in_mask), printf("\n")); - - // construct copy (either to a fixed interval or a new masked interval) - Tile* tmp = tb_arena_alloc(arena, sizeof(Tile) + sizeof(LiveInterval*)); - *tmp = (Tile){ - .prev = t->prev, - .next = t, - .tag = TILE_SPILL_MOVE, - .time = timeline - }; - assert(in_def->dt.raw); - tmp->spill_dt = in_def->dt; - tmp->ins = tb_arena_alloc(tmp_arena, sizeof(Tile*)); - tmp->in_count = 1; - tmp->ins[0].src = in_def; - tmp->ins[0].mask = in_def_mask; - if (t->prev == NULL) { - mbb->start = tmp; - } else { - t->prev->next = tmp; - } - t->prev = tmp; - - // replace use site with temporary that legalized the constraint - tmp->out_count = 1; - tmp->outs[0] = gimme_interval_for_mask(ctx, arena, &ra, in_mask, in_def->dt); - t->ins[j].src = tmp->outs[0]; - - timeline += 2; - } - } - - // place on timeline - t->time = timeline; - // 2addr ops will have a bit of room for placing the hypothetical copy - timeline += t->n && ctx->_2addr(t->n) ? 4 : 2; - - // insert copy if we're writing to a fixed interval - FOREACH_N(j, 0, t->out_count) if (t->outs[j]) { - LiveInterval* interval = t->outs[j]; - - // if we're writing to a fixed interval, insert copy - // such that we only guarentee a fixed location at the - // def site. - int reg = fixed_reg_mask(interval->mask); - if (reg >= 0 && t->tag != TILE_SPILL_MOVE) { - RegMask rm = interval->mask; - LiveInterval* fixed = &ctx->fixed[rm.class][reg]; - - // interval: FIXED(t) => NORMIE_MASK - interval->mask = ctx->normie_mask[rm.class]; - interval->hint = fixed; - - // insert copy such that the def site is the only piece which "requires" - // the fixed range. - Tile* tmp = tb_arena_alloc(arena, sizeof(Tile) + sizeof(LiveInterval*)); - *tmp = (Tile){ - .prev = t, - .next = t->next, - .tag = TILE_SPILL_MOVE, - .time = timeline, - }; - assert(interval->dt.raw); - tmp->spill_dt = interval->dt; - tmp->ins = tb_arena_alloc(tmp_arena, sizeof(Tile*)); - tmp->in_count = 1; - tmp->ins[0].src = fixed; - tmp->ins[0].mask = rm; - t->next->prev = tmp; - t->next = tmp; - - // replace def site with fixed interval - tmp->out_count = 1; - tmp->outs[0] = t->outs[j]; - t->outs[j] = fixed; - - // skip this move op - t = tmp; - timeline += 2; - } - } - } - - timeline += 2; - } - } - - // build intervals from dataflow - CUIK_TIMED_BLOCK("build intervals") { - Set visited = set_create_in_arena(arena, ctx->interval_count); - - FOREACH_REVERSE_N(i, 0, ctx->bb_count) { - MachineBB* mbb = &ctx->machine_bbs[i]; - - int bb_start = mbb->start->time; - int bb_end = mbb->end->time + 2; - - // live outs define a full range across the BB (if they're defined - // in the block, the later reverse walk will fix that up) - Set* live_out = &mbb->live_out; - FOREACH_N(j, 0, (ctx->interval_count + 63) / 64) { - uint64_t bits = live_out->data[j]; - if (bits == 0) continue; - - FOREACH_N(k, 0, 64) if (bits & (1ull << k)) { - add_range(&ra, ctx->id2interval[j*64 + k], bb_start, bb_end); - } - } - - for (Tile* t = mbb->end; t; t = t->prev) { - int time = t->time; - - // mark output - FOREACH_N(j, 0, t->out_count) { - LiveInterval* interval = t->outs[j]; - assert(interval->mask.mask); - - if (!set_get(&visited, interval->id)) { - set_put(&visited, interval->id); - if (interval->reg < 0) { - dyn_array_put(ra.unhandled, interval); - } - } - - if (interval->range_count == 1) { - add_range(&ra, interval, time, time); - } else { - interval->ranges[interval->range_count - 1].start = time; - } - } - - // 2 address ops will interfere with their own inputs (except for - // shared dst/src) - bool _2addr = t->tag == TILE_SPILL_MOVE || (t->n && ctx->_2addr(t->n)); - - // mark inputs - FOREACH_N(j, 0, t->in_count) { - LiveInterval* in_def = t->ins[j].src; - RegMask in_mask = t->ins[j].mask; - int hint = fixed_reg_mask(in_mask); - - // clobber fixed input - if (in_def == NULL) { - LiveInterval* tmp; - if (hint >= 0) { - tmp = &ctx->fixed[in_mask.class][hint]; - } else { - tmp = gimme_interval_for_mask(ctx, arena, &ra, in_mask, TB_TYPE_I64); - dyn_array_put(ra.unhandled, tmp); - t->ins[j].src = tmp; - } - - add_range(&ra, tmp, time, time + 1); - continue; - } - - RegMask in_def_mask = in_def->mask; - if (hint >= 0) { - in_def->hint = &ctx->fixed[in_mask.class][hint]; - } - - int use_time = time; - if (_2addr) { - if (j != 0) { - // extend - use_time += 2; - } else { - assert(t->out_count >= 1 && "2addr ops need destinations"); - - // hint as copy - if (t->outs[0]->hint == NULL) { - t->outs[0]->hint = in_def; - } - } - } - - add_range(&ra, in_def, bb_start, use_time); - add_use_pos(&ra, in_def, use_time, in_mask.may_spill); - } - } - } - } - - int max_regs_in_class = 0; - CUIK_TIMED_BLOCK("pre-pass on fixed intervals") { - ra.num_classes = ctx->num_classes; - ra.num_regs = ctx->num_regs; - ra.normie_mask = ctx->normie_mask; - ra.callee_saved = ctx->callee_saved; - - FOREACH_N(i, 0, ctx->num_classes) { - if (max_regs_in_class < ctx->num_regs[i]) { - max_regs_in_class = ctx->num_regs[i]; - } - - // add range at beginning such that all fixed intervals are "awake" - FOREACH_N(j, 0, ctx->num_regs[i]) { - add_range(&ra, &ctx->fixed[i][j], 0, 1); - dyn_array_put(ra.unhandled, &ctx->fixed[i][j]); - } - - size_t cap = ctx->num_regs[i]; - if (i == REG_CLASS_STK) { - // stack slots have infinite colors we so need to make sure we can resize things - if (cap < 128) cap = 128; - else cap = tb_next_pow2(cap + 1); - - ra.stack_slot_cap = cap; - } - - ra.active_set[i] = set_create_in_arena(arena, cap); - ra.active[i] = tb_arena_alloc(arena, cap * sizeof(LiveInterval*)); - ra.fixed[i] = ctx->fixed[i]; - memset(ra.active[i], 0, ctx->num_regs[i] * sizeof(LiveInterval*)); - } - - // only need enough to store for the biggest register class - ra.free_pos = TB_ARENA_ARR_ALLOC(tmp_arena, max_regs_in_class, int); - ra.block_pos = TB_ARENA_ARR_ALLOC(tmp_arena, max_regs_in_class, int); - } - - // sort intervals: - CUIK_TIMED_BLOCK("sort intervals") { - cuiksort_defs(ra.unhandled, 0, dyn_array_length(ra.unhandled) - 1); - } - - // linear scan: - // expire old => allocate free or spill/split => rinse & repeat. - CUIK_TIMED_BLOCK("linear scan") { - while (dyn_array_length(ra.unhandled)) { - LiveInterval* interval = dyn_array_pop(ra.unhandled); - - int time = interval_start(interval); - int end = interval_end(interval); - - assert(time != INT_MAX); - - #if TB_OPTDEBUG_REGALLOC - printf(" # v%-4d t=[%-4d - %4d) ", interval->id, time, end); - tb__print_regmask(interval->mask); - printf(" "); - if (interval->tile && interval->tile->n) { - print_node_sexpr(interval->tile->n, 0); - } - printf("\n"); - #endif - - // update intervals (inactive <-> active along with expiring) - FOREACH_N(rc, 0, ctx->num_classes) { - FOREACH_SET(reg, ra.active_set[rc]) { - update_interval(&ra, ra.active[rc][reg], true, time, -1); - } - } - - for (size_t i = 0; i < dyn_array_length(ra.inactive);) { - LiveInterval* inactive = ra.inactive[i]; - if (update_interval(&ra, inactive, false, time, i)) { - continue; - } - i++; - } - - // allocate free register (or stack slot) - ptrdiff_t reg = interval->reg; - if (reg < 0) { - if (interval->mask.class == REG_CLASS_STK) { - assert(interval->mask.mask == 0); - - // add new stack slot - if (reg < 0) { - int next_stk_regs = ra.num_regs[0]++; - if (next_stk_regs == ra.stack_slot_cap) { - size_t old_cap = ra.stack_slot_cap; - ra.stack_slot_cap *= 2; - - Set new_set = set_create_in_arena(arena, ra.stack_slot_cap); - set_copy(&new_set, &ra.active_set[REG_CLASS_STK]); - - LiveInterval** new_active = tb_arena_alloc(arena, ra.stack_slot_cap * sizeof(LiveInterval*)); - memcpy(new_active, ra.active, ra.stack_slot_cap * sizeof(LiveInterval*)); - - ra.active_set[REG_CLASS_STK] = new_set; - ra.active[REG_CLASS_STK] = new_active; - ra.active[REG_CLASS_STK][next_stk_regs] = NULL; - } - - reg = ra.spills++; - } - - TB_OPTDEBUG(REGALLOC)(printf(" # assign to [SP + %"PRId64"]\n", reg*8)); - } else { - reg = allocate_free_reg(&ra, interval); - assert(reg >= 0 && "despair"); - } - } - - // add to active set - if (reg >= 0) { - interval->class = interval->mask.class; - interval->assigned = reg; - move_to_active(&ra, interval); - } - - // display active set - #if TB_OPTDEBUG_REGALLOC - static const char* classes[] = { "STK", "GPR", "VEC" }; - FOREACH_N(rc, 0, ctx->num_classes) { - printf(" \x1b[32m%s { ", classes[rc]); - FOREACH_SET(reg, ra.active_set[rc]) { - LiveInterval* l = ra.active[rc][reg]; - printf("v%d:", l->id); - print_reg_name(rc, reg); - printf(" "); - } - printf("}\x1b[0m\n"); - } - #endif - } - } - - // move resolver: - // when a split happens, all indirect paths that cross the split will have - // moves inserted. - CUIK_TIMED_BLOCK("move resolver") { - FOREACH_N(i, 0, ctx->bb_count) { - MachineBB* mbb = &ctx->machine_bbs[i]; - TB_Node* end_node = mbb->end_n; - int terminator = mbb->end->time; - - for (User* u = end_node->users; u; u = u->next) { - if (cfg_is_control(u->n) && !cfg_is_endpoint(u->n)) { - TB_Node* succ = end_node->type == TB_BRANCH ? cfg_next_bb_after_cproj(u->n) : u->n; - MachineBB* target = node_to_bb(ctx, succ); - int start_time = target->start->time; - - // for all live-ins, we should check if we need to insert a move - FOREACH_SET(k, target->live_in) { - LiveInterval* interval = ctx->id2interval[k]; - - // if the value changes across the edge, insert move - LiveInterval* start = split_interval_at(interval, terminator); - LiveInterval* end = split_interval_at(interval, start_time); - - if (start != end) { - tb_todo(); - - /* if (start->spill > 0) { - assert(end->spill == start->spill && "TODO: both can't be spills yet"); - insert_split_move(&ra, start_time, start, end); - } else { - insert_split_move(&ra, terminator - 1, start, end); - } */ - } - } - } - } - } - } - - // resolve all split interval references - CUIK_TIMED_BLOCK("split resolver") { - FOREACH_REVERSE_N(i, 0, ctx->bb_count) { - MachineBB* mbb = &ctx->machine_bbs[i]; - - for (Tile* t = mbb->start; t; t = t->next) { - int pos = t->time; - - FOREACH_N(j, 0, t->out_count) { - t->outs[j] = split_interval_at(t->outs[j], pos - 1); - } - - FOREACH_N(i, 0, t->in_count) { - t->ins[i].src = split_interval_at(t->ins[i].src, pos); - } - } - } - } - - ctx->stack_usage += (ra.spills - initial_spills) * 8; - ctx->initial_spills = initial_spills; - ctx->callee_spills = ra.callee_spills; -} - -static void compute_free_pos(LSRA* restrict ra, LiveInterval* interval, int class, uint64_t mask) { - // callee saved will be biased to have nearer free positions to avoid incurring - // a spill on them early. - int half_free = interval_end(interval); - FOREACH_N(i, 0, ra->num_regs[class]) { - int p = 0; - if (mask == 0 || (mask & (1u << i))) { - // it's in use rn, stop trying to steal - if (!set_get(&ra->active_set[class], i)) { - p = INT_MAX; - if (ra->callee_saved[class] & (1ull << i)) { - p = half_free; - } - } - } - - ra->free_pos[i] = p; - } - - // for each inactive which intersects current - dyn_array_for(i, ra->inactive) { - LiveInterval* other = ra->inactive[i]; - int fp = ra->free_pos[other->assigned]; - if (fp > 0) { - int p = interval_intersect(interval, other); - if (p >= 0 && p < fp) { - ra->free_pos[other->assigned] = p; - } - } - } -} - -// returns -1 if no registers are available -static ptrdiff_t allocate_free_reg(LSRA* restrict ra, LiveInterval* interval) { - int rc = interval->mask.class; - compute_free_pos(ra, interval, interval->mask.class, interval->mask.mask); - - int highest = -1; - int hint_reg = interval->hint ? interval->hint->assigned : -1; - - // it's better in the long run to aggressively split based on hints - if (hint_reg >= 0 && interval_end(interval) <= ra->free_pos[hint_reg]) { - highest = hint_reg; - } - - // pick highest free pos - if (highest < 0) { - highest = 0; - FOREACH_N(i, 1, ra->num_regs[rc]) if (ra->free_pos[i] > ra->free_pos[highest]) { - highest = i; - } - } - - int pos = ra->free_pos[highest]; - if (UNLIKELY(pos == 0)) { - int reg = -1; - FOREACH_N(i, 0, ra->num_regs[rc]) { - if (set_get(&ra->active_set[rc], i) && ra->active[rc][i]->reg < 0) { - reg = i; - break; - } - } - assert(reg >= 0 && "no way they're all in fixed-use lmao"); - - // alloc failure, split any - LiveInterval* active_user = ra->active[rc][reg]; - set_remove(&ra->active_set[rc], reg); - - // split whatever is using the interval right now - split_intersecting(ra, interval_start(interval) - 3, active_user, REGMASK(STK, 0)); - return reg; - } else { - if (UNLIKELY(ra->callee_saved[rc] & (1ull << highest))) { - ra->callee_saved[rc] &= ~(1ull << highest); - - TB_OPTDEBUG(REGALLOC)(printf(" # spill callee saved register "), print_reg_name(rc, highest), printf("\n")); - LiveInterval* fixed = &ra->fixed[rc][highest]; - - // mark callee move - CalleeSpill c = { fixed->class, fixed->reg, ra->spills++ }; - dyn_array_put(ra->callee_spills, c); - } - - if (interval_end(interval) <= pos) { - // we can steal it completely - TB_OPTDEBUG(REGALLOC)(printf(" # assign to "), print_reg_name(rc, highest)); - - if (hint_reg >= 0) { - if (highest == hint_reg) { - TB_OPTDEBUG(REGALLOC)(printf(" (HINTED)\n")); - } else { - TB_OPTDEBUG(REGALLOC)(printf(" (FAILED HINT "), print_reg_name(rc, hint_reg), printf(")\n")); - } - } else { - TB_OPTDEBUG(REGALLOC)(printf("\n")); - } - } else { - // TODO(NeGate): split at optimal position before current - interval->class = interval->mask.class; - interval->assigned = highest; - split_intersecting(ra, pos - 3, interval, REGMASK(STK, 0)); - TB_OPTDEBUG(REGALLOC)(printf(" # stole "), print_reg_name(rc, highest), printf("\n")); - } - - return highest; - } -} - -static void insert_split_move(LSRA* restrict ra, int t, LiveInterval* old_it, LiveInterval* new_it) { - // find which BB - MachineBB* mbb = NULL; - FOREACH_N(i, 0, ra->ctx->bb_count) { - mbb = &ra->ctx->machine_bbs[i]; - if (t <= mbb->end->time) break; - } - - Tile *prev = NULL, *curr = mbb->start; - while (curr != NULL) { - if (curr->time > t) { - break; - } - prev = curr, curr = curr->next; - } - - Tile* move = tb_arena_alloc(ra->arena, sizeof(Tile) + sizeof(LiveInterval*)); - *move = (Tile){ - .tag = TILE_SPILL_MOVE, - }; - assert(old_it->dt.raw); - move->spill_dt = old_it->dt; - move->ins = tb_arena_alloc(ra->arena, sizeof(Tile*)); - move->in_count = 1; - move->ins[0].src = old_it; - move->ins[0].mask = old_it->mask; - move->out_count = 1; - move->outs[0] = new_it; - if (prev) { - move->time = prev->time + 1; - move->prev = prev; - move->next = prev->next; - if (prev->next == NULL) { - prev->next = move; - mbb->end = move; - } else { - prev->next->prev = move; - prev->next = move; - } - } else { - move->time = t; - move->next = mbb->start; - mbb->start->prev = move; - mbb->start = move; - } -} - -static LiveInterval* split_intersecting(LSRA* restrict ra, int pos, LiveInterval* interval, RegMask new_mask) { - cuikperf_region_start("split", NULL); - int rc = interval->class; - - LiveInterval* restrict new_it = TB_ARENA_ALLOC(ra->arena, LiveInterval); - *new_it = *interval; - - assert(interval->mask.class != new_mask.class); - new_it->mask = new_mask; - - if (interval->assigned >= 0) { - TB_OPTDEBUG(REGALLOC)(printf(" \x1b[33m# v%d: split ", interval->id), print_reg_name(rc, interval->assigned), printf(" at t=%d\x1b[0m\n", pos)); - } else { - TB_OPTDEBUG(REGALLOC)(printf(" \x1b[33m# v%d: split *undecided* at t=%d\x1b[0m\n", interval->id, pos)); - } - - // split lifetime - new_it->assigned = new_it->reg = -1; - new_it->uses = NULL; - new_it->split_kid = NULL; - new_it->range_count = 0; - - assert(interval->split_kid == NULL && "cannot spill while spilled"); - interval->split_kid = new_it; - - { - // since the split is starting at pos and pos is at the top of the - // unhandled list... we can push this to the top wit no problem - size_t i = 0, count = dyn_array_length(ra->unhandled); - for (; i < count; i++) { - if (pos > interval_start(ra->unhandled[i])) break; - } - - // we know where to insert - dyn_array_put(ra->unhandled, NULL); - memmove(&ra->unhandled[i + 1], &ra->unhandled[i], (count - i) * sizeof(LiveInterval*)); - ra->unhandled[i] = new_it; - } - - // split ranges - size_t end = interval->range_count; - FOREACH_REVERSE_N(i, 1, end) { - LiveRange* range = &interval->ranges[i]; - if (range->end > pos) { - bool clean_split = pos < range->start; - - LiveRange old = interval->ranges[interval->active_range]; - - new_it->range_count = new_it->range_cap = i + 1; - new_it->active_range = new_it->range_count - 1; - new_it->ranges = interval->ranges; - - // move interval up, also insert INT_MAX and potentially - size_t start = new_it->range_count - !clean_split; - - interval->range_count = interval->range_cap = (end - start) + 1; - interval->ranges = tb_arena_alloc(ra->arena, interval->range_count * sizeof(LiveRange)); - interval->active_range -= start - 1; - interval->ranges[0] = (LiveRange){ INT_MAX, INT_MAX }; - - FOREACH_N(j, start, end) { - assert(j - start + 1 < interval->range_count); - interval->ranges[j - start + 1] = new_it->ranges[j]; - } - - // assert(interval->ranges[interval->active_range].start == old.start); - // assert(interval->ranges[interval->active_range].end == old.end); - - if (range->start <= pos) { - interval->ranges[1].end = pos; - new_it->ranges[new_it->range_count - 1].start = pos; - } - break; - } - } - assert(new_it->range_count != 0); - - // split uses - size_t i = 0, use_count = interval->use_count; - while (i < use_count + 1) { - size_t split_count = use_count - i; - if (i == use_count || interval->uses[i].pos < pos) { - new_it->use_count = new_it->use_cap = i; - new_it->uses = interval->uses; - - interval->use_count = interval->use_cap = split_count; - interval->uses = &interval->uses[i]; - break; - } - i++; - } - - // insert move (the control flow aware moves are inserted later) - insert_split_move(ra, pos, interval, new_it); - - // reload before next use that requires the original regclass - if (new_mask.class == REG_CLASS_STK) { - FOREACH_REVERSE_N(i, 0, new_it->use_count) { - if (!new_it->uses[i].may_spill) { - split_intersecting(ra, new_it->uses[i].pos - 3, new_it, interval->mask); - break; - } - } - } - - cuikperf_region_end(); - return new_it; -} - -// update active range to match where the position is currently -static bool update_interval(LSRA* restrict ra, LiveInterval* interval, bool is_active, int time, int inactive_index) { - // get to the right range first - while (time >= interval->ranges[interval->active_range].end) { - assert(interval->active_range > 0); - interval->active_range -= 1; - } - - int hole_end = interval->ranges[interval->active_range].start; - int active_end = interval->ranges[interval->active_range].end; - bool is_now_active = time >= hole_end; - - int rc = interval_class(interval); - int reg = interval->assigned; - - if (interval->active_range == 0) { // expired - if (is_active) { - TB_OPTDEBUG(REGALLOC)(printf(" # active "), print_reg_name(rc, reg), printf(" has expired at t=%d (v%d)\n", interval_end(interval), interval->id)); - set_remove(&ra->active_set[rc], reg); - } else { - TB_OPTDEBUG(REGALLOC)(printf(" # inactive "), print_reg_name(rc, reg), printf(" has expired at t=%d (v%d)\n", interval_end(interval), interval->id)); - dyn_array_remove(ra->inactive, inactive_index); - return true; - } - } else if (is_now_active != is_active) { // if we moved, change which list we're in - if (is_now_active) { // inactive -> active - TB_OPTDEBUG(REGALLOC)(printf(" # inactive "), print_reg_name(rc, reg), printf(" is active again (until t=%d, v%d)\n", active_end, interval->id)); - - move_to_active(ra, interval); - dyn_array_remove(ra->inactive, inactive_index); - return true; - } else { // active -> inactive - TB_OPTDEBUG(REGALLOC)(printf(" # active "), print_reg_name(rc, reg), printf(" is going quiet for now (until t=%d, v%d)\n", active_end, interval->id)); - - set_remove(&ra->active_set[rc], reg); - dyn_array_put(ra->inactive, interval); - } - } - - return false; -} - -static void move_to_active(LSRA* restrict ra, LiveInterval* interval) { - int rc = interval_class(interval), reg = interval->assigned; - if (set_get(&ra->active_set[rc], reg)) { - tb_panic("v%d: interval v%d should never be forced out, we should've accomodated them in the first place", interval->id, ra->active[rc][reg]->id); - } - - set_put(&ra->active_set[rc], reg); - ra->active[rc][reg] = interval; -} - -//////////////////////////////// -// Sorting unhandled list -//////////////////////////////// -static size_t partition(LiveInterval** intervals, ptrdiff_t lo, ptrdiff_t hi) { - int pivot = interval_start(intervals[(hi - lo) / 2 + lo]); // middle - - ptrdiff_t i = lo - 1, j = hi + 1; - for (;;) { - // Move the left index to the right at least once and while the element at - // the left index is less than the pivot - do { i += 1; } while (interval_start(intervals[i]) > pivot); - - // Move the right index to the left at least once and while the element at - // the right index is greater than the pivot - do { j -= 1; } while (interval_start(intervals[j]) < pivot); - - // If the indices crossed, return - if (i >= j) return j; - - // Swap the elements at the left and right indices - SWAP(LiveInterval*, intervals[i], intervals[j]); - } -} - -static void cuiksort_defs(LiveInterval** intervals, ptrdiff_t lo, ptrdiff_t hi) { - if (lo >= 0 && hi >= 0 && lo < hi) { - // get pivot - size_t p = partition(intervals, lo, hi); - - // sort both sides - cuiksort_defs(intervals, lo, p); - cuiksort_defs(intervals, p + 1, hi); - } -} diff --git a/vendor/tb/src/mips/mips_insts.inc b/vendor/tb/src/mips/mips_insts.inc deleted file mode 100644 index 117da839..00000000 --- a/vendor/tb/src/mips/mips_insts.inc +++ /dev/null @@ -1,38 +0,0 @@ -// special -R(sllv, 0b000000, 0b000100) -R(srlv, 0b000000, 0b000110) -R(srav, 0b000000, 0b000111) -R(sll, 0b000000, 0b000000) -R(srl, 0b000000, 0b000010) -R(sra, 0b000000, 0b000011) -R(add, 0b000000, 0b100000) -R(addu, 0b000000, 0b100001) -R(sub, 0b000000, 0b100010) -R(subu, 0b000000, 0b100011) -R(and, 0b000000, 0b100100) -R(or, 0b000000, 0b100101) -R(xor, 0b000000, 0b100110) -R(nor, 0b000000, 0b100111) -R(jr, 0b000000, 0b001000) -// special2 -R(mul, 0b011100, 0b000010) -// i-types -I(addi, 0b001000) -I(andi, 0b001100) -I(ori, 0b001101) -I(xori, 0b001110) -I(lui, 0b001111) -// loads -I(lb, 0b100000) -I(lw, 0b100011) -I(ld, 0b110111) -// stores -I(sb, 0b101000) -I(sh, 0b101001) -I(sw, 0b101011) -// branches -I(beq, 0b000100) -J(jal, 0b000011) -#undef R -#undef I -#undef J \ No newline at end of file diff --git a/vendor/tb/src/mips/mips_target.c b/vendor/tb/src/mips/mips_target.c deleted file mode 100644 index ba101020..00000000 --- a/vendor/tb/src/mips/mips_target.c +++ /dev/null @@ -1,910 +0,0 @@ -#include "../tb_internal.h" - -#ifdef TB_HAS_MIPS -// NOTE(NeGate): THIS IS VERY INCOMPLETE -#include "../emitter.h" - -enum { - // OP reg, imm - TILE_HAS_IMM = 1, -}; - -enum { - // register classes - REG_CLASS_GPR = 1, - REG_CLASS_COUNT, -}; - -typedef struct { - TB_Node* base; - int offset; -} AuxAddress; - -typedef enum { - ZR, // zero reg. - AT, // reserved for assembler. - V0, V1, // returns. - A0, A1, A2, A3, // call params. - T0, T1, T2, T3, T4, T5, T6, T7, // temporaries (volatile) - S0, S1, S2, S3, S4, S5, S6, S7, // temporaries (non-volatile) - T8, T9, // temporaries (volatile) - K0, K1, // kernel regs. - GP, // global ptr - SP, // stack ptr - FP, // frame ptr - RA, // return addr -} GPR; - -static const char* gpr_names[32] = { - "zr", - "at", - "v0", "v1", - "a0", "a1", "a2", "a3", - "t0", "t1", "t2", "t3", "t4", "t5", "t6", "t7", - "s0", "s1", "s2", "s3", "s4", "s5", "s6", "s7", - "t8", "t9", - "k0", "k1", - "gp", - "sp", - "fp", - "ra", -}; - -#include "../codegen_impl.h" - -static bool fits_into_int16(uint64_t x) { - uint64_t hi = x >> 16ull; - return hi == 0 || hi == 0xFFFFFFFFFFFF; -} - -static bool try_for_imm16(int bits, TB_Node* n, int32_t* out_x) { - if (n->type != TB_INTEGER_CONST) { - return false; - } - - TB_NodeInt* i = TB_NODE_GET_EXTRA(n); - if (bits > 16) { - bool sign = (i->value >> 15ull) & 1; - uint64_t top = i->value >> 16ull; - - // if the sign matches the rest of the top bits, we can sign extend just fine - if (top != (sign ? 0xFFFFFFFF : 0)) { - return false; - } - } - - *out_x = i->value; - return true; -} - -static void init_ctx(Ctx* restrict ctx, TB_ABI abi) { - TB_Arch arch = ctx->module->target_arch; - assert(arch == TB_ARCH_MIPS32 || arch == TB_ARCH_MIPS64); - - ctx->sched = greedy_scheduler; - ctx->regalloc = tb__lsra; - ctx->abi_index = arch == TB_ARCH_MIPS64; - ctx->stack_slot_size = arch == TB_ARCH_MIPS64 ? 8 : 4; - - uint32_t not_tmps = (1 << ZR) | (1 << AT) | (1 << SP) | (1 << K0) | (1 << K1) | (1 << GP); - - ctx->num_regs[REG_CLASS_GPR] = 32; - ctx->normie_mask[REG_CLASS_GPR] = REGMASK(GPR, UINT32_MAX & ~not_tmps); - - uint32_t volatile_gprs = ((UINT32_MAX >> 7) << 1) & ~(0xFF << S0); - volatile_gprs |= 1u << RA; - ctx->callee_saved[REG_CLASS_GPR] = ~volatile_gprs; -} - -static int bits_in_type(Ctx* restrict ctx, TB_DataType dt) { - switch (dt.type) { - case TB_INT: return dt.data; - case TB_PTR: return 64; - default: tb_todo(); - } -} - -static RegMask normie_mask(Ctx* restrict ctx, TB_DataType dt) { - return ctx->normie_mask[REG_CLASS_GPR]; -} - -static bool is_regpair(Ctx* restrict ctx, TB_DataType dt) { - return ctx->abi_index == 0 && dt.type == TB_INT && dt.data > 32; -} - -static int reg_count(Ctx* restrict ctx, TB_Node* n) { - if (n->dt.type == TB_INT) { - assert(n->dt.data <= 64); - return ctx->abi_index == 0 && n->dt.data > 32 ? 2 : 1; - } else if (n->dt.type == TB_PTR) { - return 1; - } else if (n->dt.type == TB_FLOAT) { - return 1; - } else { - return 0; - } -} - -#define OUT1(m) (dst->outs[0]->dt = n->dt, dst->outs[0]->mask = (m)) -static void isel_node(Ctx* restrict ctx, Tile* dst, TB_Node* n) { - switch (n->type) { - // 0-input normie ops - case TB_REGION: - case TB_NATURAL_LOOP: - case TB_AFFINE_LOOP: - case TB_REGION: - case TB_ROOT: - case TB_TRAP: - case TB_CALLGRAPH: - case TB_SPLITMEM: - case TB_MERGEMEM: - case TB_UNREACHABLE: - case TB_DEBUGBREAK: - case TB_INTEGER_CONST: - break; - - case TB_LOCAL: - try_init_stack_slot(ctx, n); - break; - - case TB_PHI: - if (n->dt.type == TB_INT || n->dt.type == TB_PTR || n->dt.type == TB_FLOAT) { - RegMask rm = normie_mask(ctx, n->dt); - rm.may_spill = true; - OUT1(rm); - } - return; - - case TB_PROJ: { - if (dst->out_count) { - RegMask rm = { 0 }; - int i = TB_NODE_GET_EXTRA_T(n, TB_NodeProj)->index; - - if (n->inputs[0]->type == TB_ROOT) { - if (i == 2) { - // RPC is the RA reg - rm = REGMASK(GPR, 1u << RA); - } else if (i >= 3) { - rm = REGMASK(GPR, 1u << ((i - 3) + A0)); - } - } else if (n->inputs[0]->type == TB_CALL) { - if (n->dt.type == TB_FLOAT) { - tb_todo(); - } else if (i >= 2) { - rm = REGMASK(GPR, 1 << (V0+i)); - } - } else if (n->inputs[0]->type == TB_BRANCH) { - } else if (n->inputs[0]->type == TB_SPLITMEM) { - } else { - tb_todo(); - } - - OUT1(rm); - } - return; - } - - case TB_RETURN: { - int rets = (n->input_count - 3); - bool pair = false; - if (rets > 0 && is_regpair(ctx, n->inputs[3]->dt)) { - assert(rets == 1 && "At most 1 doubleword return :("); - pair = true; - rets = 2; - } - - dst->ins = tb_arena_alloc(tmp_arena, (1+rets) * sizeof(TileInput)); - dst->in_count = 1+rets; - dst->ins[0].mask = REGMASK(GPR, 1u << RA); - dst->ins[0].src = get_interval(ctx, n->inputs[2], 0); - - if (pair) { - dst->ins[1].mask = REGMASK(GPR, 1u << V0); - dst->ins[1].src = get_interval(ctx, n->inputs[3], 0); - dst->ins[2].mask = REGMASK(GPR, 1u << V1); - dst->ins[2].src = get_interval(ctx, n->inputs[3], 1); - } else { - assert(rets <= 2 && "At most 2 return values :("); - if (rets >= 1) { - dst->ins[1].mask = REGMASK(GPR, 1u << V0); - dst->ins[1].src = get_interval(ctx, n->inputs[3], 0); - } - - if (rets >= 2) { - dst->ins[2].mask = REGMASK(GPR, 1u << V1); - dst->ins[2].src = get_interval(ctx, n->inputs[4], 0); - } - } - break; - } - - case TB_AND: - case TB_OR: - case TB_XOR: - case TB_ADD: - case TB_SUB: - case TB_MUL: - case TB_SAR: - case TB_SHL: - case TB_SHR: { - int32_t x; - if (n->type != TB_MUL && try_for_imm16(n->dt.data, n->inputs[2], &x)) { - tile_broadcast_ins(ctx, dst, n, 1, 2, ctx->normie_mask[REG_CLASS_GPR]); - dst->flags |= TILE_HAS_IMM; - } else { - tile_broadcast_ins(ctx, dst, n, 1, 3, ctx->normie_mask[REG_CLASS_GPR]); - } - break; - } - - // no rotate ops, we'll emulate it: - case TB_ROL: - case TB_ROR: { - int32_t x; - bool fits_16 = try_for_imm16(n->dt.data, n->inputs[2], &x); - - dst->in_count = fits_16 ? 3 : 2; - dst->ins = tb_arena_alloc(tmp_arena, dst->in_count * sizeof(TileInput)); - dst->ins[0].src = get_interval(ctx, n->inputs[1], 0); - dst->ins[0].mask = ctx->normie_mask[REG_CLASS_GPR]; - dst->ins[1].src = NULL; - dst->ins[1].mask = ctx->normie_mask[REG_CLASS_GPR]; - if (fits_16) { - dst->flags |= TILE_HAS_IMM; - } else { - dst->ins[2].src = get_interval(ctx, n->inputs[1], 0); - dst->ins[2].mask = ctx->normie_mask[REG_CLASS_GPR]; - } - break; - } - - case TB_SIGN_EXT: - case TB_ZERO_EXT: { - tile_broadcast_ins(ctx, dst, n, 1, n->input_count, ctx->normie_mask[REG_CLASS_GPR]); - break; - } - - case TB_ARRAY_ACCESS: { - tile_broadcast_ins(ctx, dst, n, 1, n->input_count, ctx->normie_mask[REG_CLASS_GPR]); - break; - } - - case TB_LOAD: { - TB_Node* base = n->inputs[2]; - int16_t offset = 0; - - if (base->type == TB_MEMBER_ACCESS) { - uint64_t raw = TB_NODE_GET_EXTRA_T(n, TB_NodeMember)->offset; - if (fits_into_int16(raw)) { - base = base->inputs[1]; - offset = raw; - } - } - - if (base->type == TB_LOCAL) { - try_init_stack_slot(ctx, base); - - dst->ins = NULL; - dst->in_count = 0; - } else { - dst->ins = tb_arena_alloc(tmp_arena, 1 * sizeof(TileInput)); - dst->in_count = 1; - dst->ins[0].mask = ctx->normie_mask[REG_CLASS_GPR]; - dst->ins[0].src = get_interval(ctx, base, 0); - } - - AuxAddress* aux = tb_arena_alloc(tmp_arena, sizeof(AuxAddress)); - aux->base = base; - aux->offset = offset; - dst->aux = aux; - break; - } - - case TB_STORE: { - TB_Node* base = n->inputs[2]; - int16_t offset = 0; - - if (base->type == TB_MEMBER_ACCESS) { - uint64_t raw = TB_NODE_GET_EXTRA_T(n, TB_NodeMember)->offset; - if (fits_into_int16(raw)) { - base = base->inputs[1]; - offset = raw; - } - } - - if (base->type == TB_LOCAL) { - try_init_stack_slot(ctx, base); - - dst->ins = tb_arena_alloc(tmp_arena, 1 * sizeof(TileInput)); - dst->in_count = 1; - } else { - dst->ins = tb_arena_alloc(tmp_arena, 2 * sizeof(TileInput)); - dst->in_count = 2; - dst->ins[1].mask = ctx->normie_mask[REG_CLASS_GPR]; - dst->ins[1].src = get_interval(ctx, base, 0); - } - dst->ins[0].mask = ctx->normie_mask[REG_CLASS_GPR]; - dst->ins[0].src = get_interval(ctx, n->inputs[3], 0); - - AuxAddress* aux = tb_arena_alloc(tmp_arena, sizeof(AuxAddress)); - aux->base = base; - aux->offset = offset; - dst->aux = aux; - break; - } - - case TB_CALL: { - uint32_t volatile_gprs = ((UINT32_MAX >> 7) << 1) & ~(0xFF << S0); - volatile_gprs |= 1u << RA; - - int param_count = n->input_count - 3; - if (ctx->num_regs[0] < param_count) { - ctx->num_regs[0] = param_count; - } - - FOREACH_N(i, 0, param_count > 4 ? 4 : param_count) { - volatile_gprs &= ~(1u << (A0 + i)); - } - - size_t clobber_count = tb_popcount(volatile_gprs); - size_t input_start = n->inputs[2]->type == TB_SYMBOL ? 3 : 2; - size_t input_count = (n->input_count - input_start) + clobber_count; - - TileInput* ins; - if (n->inputs[2]->type == TB_SYMBOL) { - // CALL symbol - ins = dst->ins = tb_arena_alloc(tmp_arena, input_count * sizeof(TileInput)); - dst->in_count = input_count; - } else { - // CALL r/m - ins = dst->ins = tb_arena_alloc(tmp_arena, input_count * sizeof(TileInput)); - dst->in_count = input_count; - - ins[0].src = get_interval(ctx, n->inputs[2], 0); - ins[0].mask = ctx->normie_mask[REG_CLASS_GPR]; - ins += 1; - } - - FOREACH_N(i, 0, param_count) { - ins[i].src = get_interval(ctx, n->inputs[i + 3], 0); - - if (i < 4) { - if (TB_IS_FLOAT_TYPE(n->inputs[i + 3]->dt)) { - tb_todo(); - } else { - ins[i].mask = REGMASK(GPR, 1u << (A0 + i)); - } - } else { - // stack slots go into [RSP + 8i] - ins[i].mask = REGMASK(STK, i); - } - } - - int j = param_count; - FOREACH_N(i, 0, ctx->num_regs[1]) { - if (volatile_gprs & (1u << i)) { - ins[j].src = NULL; - ins[j].mask = REGMASK(GPR, 1u << i); - j++; - } - } - - assert(j == input_count - (n->inputs[2]->type != TB_SYMBOL)); - return; - } - - case TB_BRANCH: { - TB_Node* cmp = n->inputs[1]; - TB_NodeBranch* br = TB_NODE_GET_EXTRA(n); - - if (br->succ_count > 2) { - tb_todo(); - } else { - dst->ins = tb_arena_alloc(tmp_arena, 1 * sizeof(TileInput)); - dst->in_count = 1; - dst->ins[0].src = get_interval(ctx, cmp, 0); - dst->ins[0].mask = ctx->normie_mask[REG_CLASS_GPR]; - } - break; - } - - default: tb_todo(); - } - - if (dst->out_count == 1) { - dst->outs[0]->dt = n->dt; - dst->outs[0]->mask = normie_mask(ctx, n->dt); - } else if (dst->out_count == 2) { - dst->outs[0]->dt = TB_TYPE_I32; - dst->outs[0]->mask = normie_mask(ctx, n->dt); - dst->outs[1]->dt = TB_TYPE_I32; - dst->outs[1]->mask = normie_mask(ctx, n->dt); - } else if (dst->out_count != 0) { - tb_todo(); - } -} - -enum { - #define R(name, op, funct) name, - #define I(name, op) name, - #define J(name, op) name, - #include "mips_insts.inc" - - INST_MAX, -}; - -#define __(t, op, ...) t ## type(e, op, __VA_ARGS__) -#define COMMENT(...) (e->has_comments ? tb_emit_comment(e, tmp_arena, __VA_ARGS__) : (void)0) - -static uint32_t insts[INST_MAX] = { - #define R(name, op, funct) [name] = ((op<<26) | (funct)), - #define I(name, op) [name] = (op<<26), - #define J(name, op) [name] = (op<<26), - #include "mips_insts.inc" -}; - -static void nop(TB_CGEmitter* e) { - EMIT4(e, 0); -} - -static void jtype(TB_CGEmitter* e, int op, uint32_t imm) { - EMIT4(e, insts[op] | (imm & 0x3FFFFFF)); -} - -static void rtype(TB_CGEmitter* e, int op, uint32_t rd, uint32_t rs, uint32_t rt, uint32_t shamt) { - assert(op >= 0 && op < INST_MAX); - EMIT4(e, insts[op] | (rs<<21) | (rt<<16) | (rd<<11) | (shamt<<6)); -} - -static void itype(TB_CGEmitter* e, int op, uint32_t rt, uint32_t rs, uint32_t imm) { - assert(op >= 0 && op < INST_MAX); - EMIT4(e, insts[op] | (rs<<21) | (rt<<16) | (imm&0xFFFF)); -} - -static void tb_emit_rel16(TB_CGEmitter* restrict e, uint32_t* head, uint32_t pos) { - uint32_t curr = *head; - if (curr & 0x80000000) { - // the label target is resolved, we need to do the relocation now - uint32_t target = curr & 0x7FFFFFFF; - int32_t rel = target - (pos + 4); - PATCH2(e, pos+2, rel*4); - } else { - PATCH2(e, pos+2, curr); - *head = pos; - } -} - -static void tb_resolve_rel16(TB_CGEmitter* restrict e, uint32_t* head, uint32_t target) { - // walk previous relocations - uint32_t curr = *head; - while (curr != 0 && (curr & 0x80000000) == 0) { - uint32_t next; - memcpy(&next, &e->data[curr], 4); - int32_t rel = target - (curr + 4); - PATCH2(e, curr+2, rel*4); - curr = next; - } - - // store the target and mark it as resolved - *head = 0x80000000 | target; -} - -static void branch(TB_CGEmitter* e, int op, uint32_t rt, uint32_t rs, int id) { - assert(op == beq); - EMIT4(e, insts[op] | (rs<<21) | (rt<<16)); - EMIT4(e, 0); // TODO(NeGate): delay slots - - tb_emit_rel16(e, &e->labels[id], GET_CODE_POS(e) - 8); -} - -static void on_basic_block(Ctx* restrict ctx, TB_CGEmitter* e, int bbid) { - tb_resolve_rel16(e, &e->labels[bbid], e->count); -} - -static void loadimm(TB_CGEmitter* e, int dst, uint32_t imm) { - GPR curr = ZR; - if (imm >> 16ull) { - itype(e, lui, dst, curr, imm >> 16ull); - curr = dst; - } - - itype(e, ori, dst, curr, imm & 0xFFFF); -} - -static int stk_offset(Ctx* ctx, int reg) { - int pos = reg * ctx->stack_slot_size; - if (reg >= ctx->num_regs[0]) { - return (ctx->stack_usage - pos) + ctx->stack_slot_size; - } else { - return pos; - } -} - -static void pre_emit(Ctx* restrict ctx, TB_CGEmitter* e, TB_Node* n) { - // TODO(NeGate): optionally preserve the frame pointer - TB_FunctionPrototype* proto = ctx->f->prototype; - - size_t stack_usage = (ctx->num_regs[0] + proto->param_count) * ctx->stack_slot_size; - ctx->stack_usage = stack_usage; - - if (stack_usage > 0) { - __(i, addi, SP, SP, -stack_usage); - } - - FOREACH_N(i, 0, dyn_array_length(ctx->callee_spills)) { - int pos = stk_offset(ctx, ctx->callee_spills[i].stk); - int rc = ctx->callee_spills[i].class; - assert(rc == REG_CLASS_GPR); - - GPR src = ctx->callee_spills[i].reg; - __(i, sw, src, SP, stk_offset(ctx, pos)); - } - - ctx->prologue_length = ctx->emit.count; -} - -static GPR gpr_at(LiveInterval* l) { assert(l->class == REG_CLASS_GPR); return l->assigned; } -static void emit_tile(Ctx* restrict ctx, TB_CGEmitter* e, Tile* t) { - if (t->tag == TILE_SPILL_MOVE) { - if (t->outs[0]->class == REG_CLASS_STK && t->ins[0].src->class == REG_CLASS_GPR) { - COMMENT("spill v%d -> v%d", t->outs[0]->id, t->ins[0].src->id); - GPR src = gpr_at(t->ins[0].src); - int dst = t->outs[0]->assigned; - itype(e, sw, src, SP, stk_offset(ctx, dst)); - } else if (t->outs[0]->class == REG_CLASS_GPR && t->ins[0].src->class == REG_CLASS_STK) { - COMMENT("reload v%d -> v%d", t->outs[0]->id, t->ins[0].src->id); - GPR dst = gpr_at(t->outs[0]); - int src = t->ins[0].src->assigned; - itype(e, lw, dst, SP, stk_offset(ctx, src)); - } else { - GPR dst = gpr_at(t->outs[0]); - GPR src = gpr_at(t->ins[0].src); - if (dst != src) { - rtype(e, or, dst, src, 0, 0); - } - } - } else if (t->tag == TILE_GOTO) { - MachineBB* mbb = node_to_bb(ctx, t->succ); - if (ctx->fallthrough != mbb->id) { - branch(e, beq, ZR, ZR, mbb->id); - } - } else { - TB_Node* n = t->n; - switch (n->type) { - // projections don't manage their own work, that's the - // TUPLE node's job. - case TB_PROJ: - case TB_REGION: - case TB_NATURAL_LOOP: - case TB_AFFINE_LOOP: - case TB_PHI: - case TB_ROOT: - case TB_POISON: - case TB_UNREACHABLE: - case TB_SPLITMEM: - case TB_MERGEMEM: - case TB_CALLGRAPH: - break; - - case TB_RETURN: { - // return can run in parallel to the stack free - __(r, jr, 0, RA, 0, 0); - if (ctx->stack_usage > 0) { - __(i, addi, SP, SP, ctx->stack_usage); - } else { - nop(e); - } - break; - } - - case TB_BRANCH: { - GPR src = gpr_at(t->ins[0].src); - __debugbreak(); - // branch(e, beq, src, ZR, 0); - // branch(e, beq, src, ZR, mbb->id); - break; - } - - case TB_CALL: { - if (n->inputs[2]->type == TB_SYMBOL) { - TB_Symbol* sym = TB_NODE_GET_EXTRA_T(n->inputs[2], TB_NodeSymbol)->sym; - - __(j, jal, 0); - tb_emit_symbol_patch(e->output, sym, e->count - 4); - } else { - tb_todo(); - } - nop(e); // TODO(NeGate): delay slots - break; - } - - case TB_LOAD: { - AuxAddress* aux = t->aux; - GPR dst = gpr_at(t->outs[0]); - int32_t offset = aux->offset; - - GPR base; - if (aux->base->type == TB_LOCAL) { - offset += get_stack_slot(ctx, aux->base); - base = SP; - } else { - base = gpr_at(t->ins[0].src); - } - itype(e, lw, dst, base, offset); - break; - } - - case TB_STORE: { - AuxAddress* aux = t->aux; - GPR src = gpr_at(t->ins[0].src); - int32_t offset = aux->offset; - - GPR base; - if (aux->base->type == TB_LOCAL) { - offset += get_stack_slot(ctx, aux->base); - base = SP; - } else { - base = gpr_at(t->ins[1].src); - } - itype(e, sw, src, base, offset); - break; - } - - case TB_LOCAL: { - GPR dst = gpr_at(t->outs[0]); - int pos = get_stack_slot(ctx, n); - itype(e, addi, dst, SP, pos); - break; - } - - case TB_INTEGER_CONST: { - GPR dst = gpr_at(t->outs[0]); - uint64_t imm = TB_NODE_GET_EXTRA_T(n, TB_NodeInt)->value; - - loadimm(e, dst, imm); - if (t->out_count == 2) { - GPR dst2 = gpr_at(t->outs[1]); - loadimm(e, dst2, imm >> 32ull); - } - break; - } - - case TB_ARRAY_ACCESS: { - intptr_t offset = (intptr_t) t->aux; - GPR dst = gpr_at(t->outs[0]); - GPR base = gpr_at(t->ins[0].src); - GPR idx = gpr_at(t->ins[1].src); - int64_t stride = TB_NODE_GET_EXTRA_T(n, TB_NodeArray)->stride; - - uint64_t log2 = tb_ffs(stride) - 1; - if (UINT64_C(1) << log2 == stride) { - rtype(e, sll, dst, 0, idx, log2); - } else { - tb_todo(); - } - rtype(e, add, dst, base, offset, 0); - break; - } - - case TB_AND: - case TB_OR: - case TB_XOR: - case TB_ADD: - case TB_SUB: - case TB_MUL: { - const static int rops[] = { and, or, xor, add, sub, mul }; - const static int iops[] = { andi, ori, xori, addi, addi, -1 }; - - GPR dst = gpr_at(t->outs[0]); - GPR lhs = gpr_at(t->ins[0].src); - if (t->flags & TILE_HAS_IMM) { - assert(n->inputs[2]->type == TB_INTEGER_CONST); - uint64_t i = TB_NODE_GET_EXTRA_T(n->inputs[2], TB_NodeInt)->value; - if (n->type == TB_SUB) { - // no SUBI, we just negate and do ADDI - i = -i; - } - - itype(e, iops[n->type - TB_AND], dst, lhs, i); - } else { - GPR rhs = gpr_at(t->ins[1].src); - rtype(e, rops[n->type - TB_AND], dst, lhs, rhs, 0); - } - break; - } - - case TB_ZERO_EXT: - case TB_SIGN_EXT: { - GPR dst = gpr_at(t->outs[0]); - GPR src = gpr_at(t->ins[0].src); - int op = n->type == TB_SIGN_EXT ? sra : srl; - - int32_t x = 64; - int32_t y = x - bits_in_type(ctx, n->dt); - rtype(e, sll, dst, 0, src, x); - rtype(e, op, dst, 0, src, y); - break; - } - - case TB_SAR: - case TB_SHL: - case TB_SHR: { - const static int rops[] = { sllv, srlv, srav }; - const static int iops[] = { sll, srl, sra }; - - GPR dst = gpr_at(t->outs[0]); - GPR lhs = gpr_at(t->ins[0].src); - if (t->flags & TILE_HAS_IMM) { - assert(n->inputs[2]->type == TB_INTEGER_CONST); - uint64_t x = TB_NODE_GET_EXTRA_T(n->inputs[2], TB_NodeInt)->value; - rtype(e, iops[n->type - TB_SHL], dst, 0, lhs, x); - } else { - GPR rhs = gpr_at(t->ins[1].src); - rtype(e, rops[n->type - TB_SHL], dst, lhs, rhs, 0); - } - break; - } - - case TB_ROL: { - GPR dst = gpr_at(t->outs[0]); - GPR src = gpr_at(t->ins[0].src); - GPR tmp = gpr_at(t->ins[1].src); - - if (t->flags & TILE_HAS_IMM) { - assert(n->inputs[2]->type == TB_INTEGER_CONST); - uint64_t x = TB_NODE_GET_EXTRA_T(n->inputs[2], TB_NodeInt)->value; - - int32_t y = 32 - x; - rtype(e, sll, tmp, 0, src, x); // sll tmp,src,X - rtype(e, srl, dst, 0, src, y); // srl dst,src,Y - rtype(e, or, dst, dst, tmp, 0); // or dst,dst,tmp - } else { - tb_todo(); - } - break; - } - - default: tb_todo(); - } - } -} - -static void post_emit(Ctx* restrict ctx, TB_CGEmitter* e) { -} - -typedef struct { - const char* name; - int op; - enum { ITYPE, RTYPE, JTYPE } family; -} Op; - -static Op decode(uint32_t inst) { - #define R(name, op, funct) if ((inst >> 26) == op && (inst & 0b111111) == funct) { return (Op){ #name, name, RTYPE }; } - #define I(name, op) if ((inst >> 26) == op) { return (Op){ #name, name, ITYPE }; } - #define J(name, op) if ((inst >> 26) == op) { return (Op){ #name, name, JTYPE }; } - #include "mips_insts.inc" - - return (Op){ "error", RTYPE }; -} - -#define E(fmt, ...) tb_asm_print(e, fmt, ## __VA_ARGS__) -static void disassemble(TB_CGEmitter* e, Disasm* restrict d, int bb, size_t pos, size_t end) { - if (bb >= 0) { - E(".bb%d:\n", bb); - } - - while (pos < end) { - while (d->loc != d->end && d->loc->pos == pos) { - E(" // %s : line %d\n", d->loc->file->path, d->loc->line); - d->loc++; - } - - uint32_t inst; - memcpy(&inst, &e->data[pos], sizeof(uint32_t)); - pos += 4; - - // special case - uint64_t line_start = e->total_asm; - if (inst == 0) { - E(" nop"); - } else if ((inst >> 26) == 0 && (inst & 0b111111) == 0b100101 && ((inst >> 16) & 0b11111) == 0) { - int a = (inst >> 11) & 0b11111; - int c = (inst >> 21) & 0b11111; - E(" move $%s, $%s", gpr_names[a], gpr_names[c]); - } else if ((inst >> 26) == 0b001101 && ((inst >> 21) & 0b11111) == 0) { - int a = (inst >> 16) & 0b11111; - E(" lui $%s, %d", gpr_names[a], inst & 0xFFFF); - } else { - // figure out what instruction type it is (everything else is trivial from there) - Op op = decode(inst); - E(" %s ", op.name); - switch (op.family) { - case RTYPE: { - int a = (inst >> 11) & 0b11111; - int b = (inst >> 16) & 0b11111; - int c = (inst >> 21) & 0b11111; - E("$%s, $%s, $%s", gpr_names[a], gpr_names[b], gpr_names[c]); - break; - } - - case ITYPE: { - int b = (inst >> 16) & 0b11111; - int c = (inst >> 21) & 0b11111; - if (op.op >= lb && op.op <= sw) { - E("$%s, $%s(%d)", gpr_names[b], gpr_names[c], tb__sxt(inst & 0xFFFF, 16, 64)); - } else { - E("$%s, $%s, %d", gpr_names[b], gpr_names[c], tb__sxt(inst & 0xFFFF, 16, 64)); - } - break; - } - - case JTYPE: { - int32_t disp = (inst & 0x3FFFFFF) * 2; - if (d->patch && d->patch->pos == pos - 4) { - const TB_Symbol* target = d->patch->target; - d->patch = d->patch->next; - - if (target->name[0] == 0) { - E("sym%p", target); - } else { - E("%s", target->name); - } - - if (disp) { - E(" + %"PRIu64, disp); - } - } else { - E("%"PRIu64, disp); - } - break; - } - - default: tb_todo(); - } - } - - int offset = e->total_asm - line_start; - if (d->comment && d->comment->pos == pos) { - TB_OPTDEBUG(ANSI)(E("\x1b[32m")); - E(" // "); - bool out_of_line = false; - do { - if (out_of_line) { - // tack on a newline - E("%*s // ", offset, ""); - } - - E("%.*s\n", d->comment->line_len, d->comment->line); - d->comment = d->comment->next; - out_of_line = true; - } while (d->comment && d->comment->pos == pos); - TB_OPTDEBUG(ANSI)(E("\x1b[0m")); - } else { - E("\n"); - } - } -} -#undef E - -static size_t emit_call_patches(TB_Module* restrict m, TB_FunctionOutput* out_f) { - return 0; -} - -ICodeGen tb__mips32_codegen = { - .minimum_addressable_size = 8, - .pointer_size = 32, - .emit_win64eh_unwind_info = NULL, - .emit_call_patches = emit_call_patches, - .get_data_type_size = get_data_type_size, - .compile_function = compile_function, -}; - -ICodeGen tb__mips64_codegen = { - .minimum_addressable_size = 8, - .pointer_size = 64, - .emit_win64eh_unwind_info = NULL, - .emit_call_patches = emit_call_patches, - .get_data_type_size = get_data_type_size, - .compile_function = compile_function, -}; -#endif diff --git a/vendor/tb/src/mips/targets.txt b/vendor/tb/src/mips/targets.txt deleted file mode 100644 index 3a348c28..00000000 --- a/vendor/tb/src/mips/targets.txt +++ /dev/null @@ -1,23 +0,0 @@ -This is *sorta* the tutorial on writing codegen targets. - -Section 1: stubbing out the necessary functions. - codegen_impl.h acts as the glue code for a target, copy them - into your code to get started. - -``` -// in your cpu_target.c -#include "../tb_internal.h" - -#include "../codegen_impl.h" - -// stubbed out -static void init_ctx(Ctx* restrict ctx, TB_ABI abi) { tb_todo(); } -static RegMask isel_node(Ctx* restrict ctx, Tile* dst, TB_Node* n) { tb_todo(); } -static void emit_tile(Ctx* restrict ctx, TB_CGEmitter* e, Tile* t) { tb_todo(); } -static void pre_emit(Ctx* restrict ctx, TB_CGEmitter* e, TB_Node* n) { tb_todo(); } -static void post_emit(Ctx* restrict ctx, TB_CGEmitter* e) { tb_todo(); } -static void disassemble(TB_CGEmitter* e, Disasm* restrict d, int bb, size_t pos, size_t end) { tb_todo(); } -``` - - - diff --git a/vendor/tb/src/new_builder.c b/vendor/tb/src/new_builder.c deleted file mode 100644 index 6d25b360..00000000 --- a/vendor/tb/src/new_builder.c +++ /dev/null @@ -1,639 +0,0 @@ -// I'm trying to transition most people over since it's a bit nicer. - -typedef struct TB_GraphCtrl TB_GraphCtrl; -struct TB_GraphCtrl { - TB_GraphCtrl* prev; - - int pos; // where the stack head was before doing the code - bool has_else; - bool is_loop; - - // LOOP: [0] body, [1] break - // IF: [0] true, [1] false - TB_Node* paths[2]; - - // loop header - TB_Node* header; - TB_Node* join; - - // preserved names - TB_ArenaSavepoint sp; - int val_cnt; - TB_Node* vals[]; -}; - -struct TB_GraphBuilder { - TB_Function* f; - TB_Arena* arena; - - // active control flow - TB_Node *top_ctrl, *bot_ctrl; - - // ctrl stack - TB_GraphCtrl* top; - - // value stack - int val_cap, val_cnt; - TB_Node** vals; -}; - -static TB_Node* xfer_ctrl(TB_GraphBuilder* g, TB_Node* n) { - TB_Node* prev = g->bot_ctrl; - g->bot_ctrl = n; - return prev; -} - -static void push(TB_GraphBuilder* g, TB_Node* n) { - if (g->val_cnt == g->val_cap) { - g->val_cap *= 2; - g->vals = tb_platform_heap_realloc(g->vals, g->val_cap * sizeof(TB_Node*)); - } - - // we don't wanna GVN phis too early, they might be incomplete - if (n->type != TB_PHI) { - n = tb_pass_gvn_node(g->f, n); - } - - g->vals[g->val_cnt++] = n; -} - -static TB_Node* pop(TB_GraphBuilder* g) { - assert(g->val_cnt > 0 && "nothing on the value stack!!!"); - return g->vals[--g->val_cnt]; -} - -TB_GraphBuilder* tb_builder_enter(TB_Function* f, TB_Arena* arena) { - TB_GraphBuilder* g = tb_arena_alloc(arena, sizeof(TB_GraphBuilder)); - *g = (TB_GraphBuilder){ .f = f, .arena = arena, .val_cap = 32 }; - g->vals = tb_platform_heap_alloc(g->val_cap * sizeof(TB_Node*)); - - // both RPC and memory are mutable vars - assert(g->val_cap >= 2 + f->param_count); - g->val_cnt = 2 + f->param_count; - FOREACH_N(i, 0, 2 + f->param_count) { - g->vals[i] = f->params[1 + i]; - } - - g->top_ctrl = g->bot_ctrl = f->params[0]; - return g; -} - -void tb_builder_exit(TB_GraphBuilder* g) { -} - -int tb_builder_save(TB_GraphBuilder* g) { - return g->val_cnt; -} - -void tb_builder_restore(TB_GraphBuilder* g, int v) { - assert(g->val_cnt >= v); - g->val_cnt = v; -} - -void tb_builder_push(TB_GraphBuilder* g, TB_Node* n) { - push(g, n); -} - -TB_Node* tb_builder_pop(TB_GraphBuilder* g) { - return pop(g); -} - -void tb_builder_uint(TB_GraphBuilder* g, TB_DataType dt, uint64_t x) { - assert(TB_IS_POINTER_TYPE(dt) || TB_IS_INTEGER_TYPE(dt)); - if (dt.type == TB_INT && dt.data < 64) { - uint64_t mask = ~UINT64_C(0) >> (64 - dt.data); - x &= mask; - } - - TB_Node* n = tb_alloc_node(g->f, TB_INTEGER_CONST, dt, 1, sizeof(TB_NodeInt)); - set_input(g->f, n, g->f->root_node, 0); - TB_NODE_SET_EXTRA(n, TB_NodeInt, .value = x); - push(g, n); -} - -void tb_builder_sint(TB_GraphBuilder* g, TB_DataType dt, int64_t x) { - assert(TB_IS_POINTER_TYPE(dt) || TB_IS_INTEGER_TYPE(dt)); - - TB_Node* n = tb_alloc_node(g->f, TB_INTEGER_CONST, dt, 1, sizeof(TB_NodeInt)); - set_input(g->f, n, g->f->root_node, 0); - TB_NODE_SET_EXTRA(n, TB_NodeInt, .value = x); - push(g, n); -} - -void tb_builder_float32(TB_GraphBuilder* g, float imm) { - TB_Node* n = tb_alloc_node(g->f, TB_FLOAT32_CONST, TB_TYPE_F32, 1, sizeof(TB_NodeFloat32)); - set_input(g->f, n, g->f->root_node, 0); - TB_NODE_SET_EXTRA(n, TB_NodeFloat32, .value = imm); - push(g, n); -} - -void tb_builder_float64(TB_GraphBuilder* g, double imm) { - TB_Node* n = tb_alloc_node(g->f, TB_FLOAT64_CONST, TB_TYPE_F64, 1, sizeof(TB_NodeFloat64)); - set_input(g->f, n, g->f->root_node, 0); - TB_NODE_SET_EXTRA(n, TB_NodeFloat64, .value = imm); - push(g, n); -} - -void tb_builder_string(TB_GraphBuilder* g, ptrdiff_t len, const char* str) { - if (len < 0) len = strlen(str) + 1; - - TB_Function* f = g->f; - TB_Global* dummy = tb_global_create(f->super.module, 0, NULL, NULL, TB_LINKAGE_PRIVATE); - tb_global_set_storage(f->super.module, tb_module_get_rdata(f->super.module), dummy, len, 1, 1); - - char* dst = tb_global_add_region(f->super.module, dummy, 0, len); - memcpy(dst, str, len); - - push(g, tb_inst_get_symbol_address(f, (TB_Symbol*) dummy)); -} - -void tb_builder_cast(TB_GraphBuilder* g, TB_DataType dt, int type) { - assert(type >= TB_TRUNCATE && type <= TB_BITCAST); - TB_Node* a = pop(g); - - TB_Function* f = g->f; - TB_Node* n = tb_alloc_node(f, type, dt, 2, 0); - set_input(f, n, a, 1); - push(g, n); -} - -void tb_builder_binop_int(TB_GraphBuilder* g, int type, TB_ArithmeticBehavior ab) { - assert(type >= TB_AND && type <= TB_SMOD); - TB_Node* b = pop(g); - TB_Node* a = pop(g); - - TB_Function* f = g->f; - TB_Node* n = tb_alloc_node(f, type, a->dt, 3, sizeof(TB_NodeBinopInt)); - set_input(f, n, a, 1); - set_input(f, n, b, 2); - TB_NODE_SET_EXTRA(n, TB_NodeBinopInt, .ab = ab); - push(g, n); -} - -void tb_builder_cmp(TB_GraphBuilder* g, int type, bool flip, TB_DataType dt) { - TB_Node* b = pop(g); - TB_Node* a = pop(g); - tb_assert(type >= TB_CMP_EQ && type <= TB_CMP_FLE, "'type' wasn't a comparison node type (see TB_NodeTypeEnum)"); - tb_assert(TB_DATA_TYPE_EQUALS(a->dt, b->dt), "datatype mismatch"); - - TB_Function* f = g->f; - TB_Node* n = tb_alloc_node(f, type, TB_TYPE_BOOL, 3, sizeof(TB_NodeCompare)); - if (flip) { - set_input(f, n, b, 1); - set_input(f, n, a, 2); - } else { - set_input(f, n, a, 1); - set_input(f, n, b, 2); - } - TB_NODE_SET_EXTRA(n, TB_NodeCompare, .cmp_dt = a->dt); - push(g, n); -} - -void tb_builder_array(TB_GraphBuilder* g, int64_t stride) { - TB_Node* index = pop(g); - TB_Node* base = pop(g); - tb_assert(base->dt.type == TB_PTR, "base on ARRAY must be an integer"); - tb_assert(index->dt.type == TB_INT, "index on ARRAY must be an integer"); - - TB_Function* f = g->f; - TB_Node* n = tb_alloc_node(f, TB_ARRAY_ACCESS, TB_TYPE_PTR, 3, sizeof(TB_NodeArray)); - set_input(f, n, base, 1); - set_input(f, n, index, 2); - TB_NODE_SET_EXTRA(n, TB_NodeArray, .stride = stride); - push(g, n); -} - -void tb_builder_member(TB_GraphBuilder* g, int64_t offset) { - if (offset == 0) { return; } - TB_Node* base = pop(g); - - TB_Function* f = g->f; - TB_Node* n = tb_alloc_node(f, TB_MEMBER_ACCESS, TB_TYPE_PTR, 2, sizeof(TB_NodeMember)); - set_input(f, n, base, 1); - TB_NODE_SET_EXTRA(n, TB_NodeMember, .offset = offset); - push(g, n); -} - -void tb_builder_load(TB_GraphBuilder* g, int mem_var, bool ctrl_dep, TB_DataType dt, int32_t offset, TB_CharUnits alignment) { - TB_Function* f = g->f; - - TB_Node* addr = pop(g); - if (offset) { - TB_Node* n = tb_alloc_node(f, TB_MEMBER_ACCESS, TB_TYPE_PTR, 2, sizeof(TB_NodeMember)); - set_input(f, n, addr, 1); - TB_NODE_SET_EXTRA(n, TB_NodeMember, .offset = offset); - addr = n; - } - - TB_Node* n = tb_alloc_node(f, TB_LOAD, dt, 3, sizeof(TB_NodeMemAccess)); - if (ctrl_dep) { - set_input(f, n, g->bot_ctrl, 0); - } - set_input(f, n, g->vals[mem_var], 1); - set_input(f, n, addr, 2); - TB_NODE_SET_EXTRA(n, TB_NodeMemAccess, .align = alignment); - - push(g, n); -} - -void tb_builder_store(TB_GraphBuilder* g, int mem_var, int32_t offset, TB_CharUnits alignment) { - TB_Function* f = g->f; - - TB_Node* val = pop(g); - TB_Node* addr = pop(g); - assert(g->vals[mem_var]->dt.type == TB_MEMORY); - - if (offset) { - TB_Node* n = tb_alloc_node(f, TB_MEMBER_ACCESS, TB_TYPE_PTR, 2, sizeof(TB_NodeMember)); - set_input(f, n, addr, 1); - TB_NODE_SET_EXTRA(n, TB_NodeMember, .offset = offset); - addr = n; - } - - TB_Node* n = tb_alloc_node(f, TB_STORE, TB_TYPE_MEMORY, 4, sizeof(TB_NodeMemAccess)); - set_input(f, n, f->trace.bot_ctrl, 0); - set_input(f, n, g->vals[mem_var], 1); - set_input(f, n, addr, 2); - set_input(f, n, val, 3); - TB_NODE_SET_EXTRA(n, TB_NodeMemAccess, .align = alignment); - - // assign ROOT memory - g->vals[mem_var] = n; -} - -void tb_builder_debugbreak(TB_GraphBuilder* g) { - TB_Node* n = tb_alloc_node(g->f, TB_DEBUGBREAK, TB_TYPE_CONTROL, 1, 0); - set_input(g->f, n, xfer_ctrl(g, n), 0); -} - -int tb_builder_var(TB_GraphBuilder* g) { - return g->val_cnt - 1; -} - -void tb_builder_get_var(TB_GraphBuilder* g, int id) { - assert(id < g->val_cnt); - push(g, g->vals[id]); -} - -void tb_builder_set_var(TB_GraphBuilder* g, int id) { - assert(id < g->val_cnt - 1); - g->vals[id] = pop(g); -} - -void tb_builder_if(TB_GraphBuilder* g, int total_hits, int taken) { - TB_Function* f = g->f; - - TB_ArenaSavepoint sp = tb_arena_save(g->arena); - TB_GraphCtrl* ctrl = tb_arena_alloc(g->arena, sizeof(TB_GraphCtrl) + g->val_cnt*sizeof(TB_Node*)); - *ctrl = (TB_GraphCtrl){ 0 }; - - TB_Node* cond = pop(g); - - // clone incoming state so we can diff & insert phis later - ctrl->sp = sp; - ctrl->val_cnt = g->val_cnt; - FOREACH_N(i, 0, g->val_cnt) { - ctrl->vals[i] = g->vals[i]; - } - - ctrl->prev = g->top; - g->top = ctrl; - - // generate branch op - { - TB_Node* n = tb_alloc_node(f, TB_BRANCH, TB_TYPE_TUPLE, 2, sizeof(TB_NodeBranch) + sizeof(int64_t)); - set_input(f, n, xfer_ctrl(g, NULL), 0); - set_input(f, n, cond, 1); - - ctrl->paths[0] = tb__make_proj(f, TB_TYPE_CONTROL, n, 0); - ctrl->paths[1] = tb__make_proj(f, TB_TYPE_CONTROL, n, 1); - - TB_NodeBranch* br = TB_NODE_GET_EXTRA(n); - br->total_hits = total_hits; - br->succ_count = 2; - br->keys[0].key = 0; - br->keys[0].taken = total_hits - taken; - } - - // add merge region (no phis yet, they'll be back) - ctrl->header = NULL; - ctrl->join = tb_alloc_node_dyn(f, TB_REGION, TB_TYPE_CONTROL, 0, 2, sizeof(TB_NodeRegion)); - TB_NODE_SET_EXTRA(ctrl->join, TB_NodeRegion); - - // goto the true path - g->top_ctrl = g->bot_ctrl = ctrl->paths[0]; -} - -void tb_builder_else(TB_GraphBuilder* g) { - assert(g->top && "we're not inside of an if block"); - TB_GraphCtrl* ctrl = g->top; - - if (g->bot_ctrl) { - // goto true => join - add_input_late(g->f, ctrl->join, g->bot_ctrl); - } - - // goto the false path - ctrl->has_else = true; - g->top_ctrl = g->bot_ctrl = ctrl->paths[1]; -} - -void tb_builder_endif(TB_GraphBuilder* g) { - assert(g->top_ctrl && "we're not inside of an if block"); - TB_GraphCtrl* ctrl = g->top; - g->top = ctrl->prev; - - if (!ctrl->has_else) { - // goto false => join - add_input_late(g->f, ctrl->join, ctrl->paths[1]); - } - - if (g->bot_ctrl) { - // fallthru - add_input_late(g->f, ctrl->join, g->bot_ctrl); - } - - // insert phis - TB_Node* join = ctrl->join; - assert(join->type == TB_REGION); - assert(g->val_cnt == ctrl->val_cnt && "paths on branch mismatch in outgoing variables?"); - - if (join->input_count > 0) { - FOREACH_N(i, 0, g->val_cnt) { - if (g->vals[i] != ctrl->vals[i]) { - TB_Node* n = tb_alloc_node_dyn(g->f, TB_PHI, g->vals[i]->dt, 3, 3, 0); - set_input(g->f, n, join, 0); - set_input(g->f, n, ctrl->vals[i], 1); - set_input(g->f, n, g->vals[i], 2); - g->vals[i] = n; - } - } - - g->top_ctrl = g->bot_ctrl = join; - } else { - tb_pass_kill_node(g->f, join); - g->top_ctrl = g->bot_ctrl = NULL; - } - tb_arena_restore(g->arena, ctrl->sp); -} - -static void block_jmp(TB_GraphBuilder* g, TB_Node* bot, int depth) { - TB_GraphCtrl* ctrl = g->top; - while (depth--) { ctrl = ctrl->prev; } - - // add exit path - if (!ctrl->is_loop && ctrl->join == NULL) { - ctrl->join = tb_alloc_node_dyn(g->f, TB_REGION, TB_TYPE_CONTROL, 0, 2, sizeof(TB_NodeRegion)); - TB_NODE_SET_EXTRA(ctrl->join, TB_NodeRegion); - } - - TB_Node* target = ctrl->is_loop ? ctrl->header : ctrl->join; - - assert(ctrl->val_cnt == g->val_cnt); - add_input_late(g->f, target, bot); - - // add edges to phis - if (ctrl->is_loop) { - FOREACH_N(i, 0, ctrl->val_cnt) { - add_input_late(g->f, ctrl->vals[i], g->vals[i]); - } - } else { - FOREACH_N(i, 0, ctrl->val_cnt) { - TB_Node* val = ctrl->vals[i]; - if (val == NULL) { - ctrl->vals[i] = g->vals[i]; - } else if (val->type == TB_PHI && val->inputs[0] == target) { - add_input_late(g->f, val, g->vals[i]); - } else { - // add phi - TB_Node* n = tb_alloc_node_dyn(g->f, TB_PHI, g->vals[i]->dt, 3, 3, 0); - set_input(g->f, n, target, 0); - set_input(g->f, n, val, 1); - set_input(g->f, n, g->vals[i], 2); - ctrl->vals[i] = n; - } - } - } -} - -void tb_builder_loop(TB_GraphBuilder* g) { - TB_Function* f = g->f; - - TB_ArenaSavepoint sp = tb_arena_save(g->arena); - TB_GraphCtrl* ctrl = tb_arena_alloc(g->arena, sizeof(TB_GraphCtrl) + g->val_cnt*sizeof(TB_Node*)); - *ctrl = (TB_GraphCtrl){ 0 }; - - // add header region - ctrl->header = tb_alloc_node_dyn(f, TB_NATURAL_LOOP, TB_TYPE_CONTROL, 0, 2, sizeof(TB_NodeRegion)); - ctrl->join = NULL; - ctrl->is_loop = true; - TB_NODE_SET_EXTRA(ctrl->header, TB_NodeRegion); - - if (g->bot_ctrl) { - // fallthru - add_input_late(g->f, ctrl->header, g->bot_ctrl); - } - - // since we don't know which values are mutated, we'll just clone all of them - ctrl->sp = sp; - ctrl->val_cnt = g->val_cnt; - FOREACH_N(i, 0, g->val_cnt) { - TB_Node* n = tb_alloc_node_dyn(g->f, TB_PHI, g->vals[i]->dt, 2, 4, 0); - set_input(g->f, n, ctrl->header, 0); - set_input(g->f, n, g->vals[i], 1); - g->vals[i] = ctrl->vals[i] = n; - } - - ctrl->prev = g->top; - g->top = ctrl; - - // goto the loop body - g->top_ctrl = g->bot_ctrl = ctrl->header; -} - -void tb_builder_endloop(TB_GraphBuilder* g) { - TB_GraphCtrl* ctrl = g->top; - g->top = ctrl->prev; - - TB_Node* header = ctrl->header; - FOREACH_N(i, 0, ctrl->val_cnt) { - // phis are complete now, we can fold them out if unused - TB_Node* src = ctrl->vals[i]; - /* if (src->type == TB_PHI && src->inputs[0] == header && src->input_count > 1) { - TB_Node* k = identity_phi(NULL, g->f, src); - if (k != src) { - // phi => same - subsume_node(g->f, src, k); - g->vals[i] = k; - continue; - } - } */ - - g->vals[i] = ctrl->vals[i]; - } - - tb_arena_restore(g->arena, ctrl->sp); -} - -void tb_builder_br(TB_GraphBuilder* g, int depth) { - block_jmp(g, g->bot_ctrl, depth); - g->top_ctrl = g->bot_ctrl = NULL; -} - -void tb_builder_br_if(TB_GraphBuilder* g, int depth) { - TB_Function* f = g->f; - - TB_Node* cond = pop(g); - TB_Node* n = tb_alloc_node(f, TB_BRANCH, TB_TYPE_TUPLE, 2, sizeof(TB_NodeBranch) + sizeof(int64_t)); - set_input(f, n, xfer_ctrl(g, NULL), 0); - set_input(f, n, cond, 1); - - TB_Node* leave = tb__make_proj(f, TB_TYPE_CONTROL, n, 0); - TB_Node* fallthru = tb__make_proj(f, TB_TYPE_CONTROL, n, 1); - - TB_NodeBranch* br = TB_NODE_GET_EXTRA(n); - br->total_hits = 100; - br->succ_count = 2; - br->keys[0].key = 0; - br->keys[0].taken = 50; - - block_jmp(g, leave, depth); - - // goto fallthru path - g->top_ctrl = g->bot_ctrl = fallthru; -} - -void tb_builder_block(TB_GraphBuilder* g) { - TB_Function* f = g->f; - - TB_ArenaSavepoint sp = tb_arena_save(g->arena); - TB_GraphCtrl* ctrl = tb_arena_alloc(g->arena, sizeof(TB_GraphCtrl) + sizeof(TB_Node*)*g->val_cnt); - *ctrl = (TB_GraphCtrl){ 0 }; - ctrl->sp = sp; - - ctrl->val_cnt = g->val_cnt; - memset(ctrl->vals, 0, sizeof(TB_Node*)*g->val_cnt); - - ctrl->prev = g->top; - g->top = ctrl; -} - -void tb_builder_endblock(TB_GraphBuilder* g) { - if (g->bot_ctrl != NULL) { - block_jmp(g, g->bot_ctrl, 0); - } - - TB_GraphCtrl* ctrl = g->top; - g->top = ctrl->prev; - - FOREACH_N(i, 0, ctrl->val_cnt) { - g->vals[i] = ctrl->vals[i]; - } - - g->top_ctrl = g->bot_ctrl = ctrl->join; - tb_arena_restore(g->arena, ctrl->sp); -} - -void tb_builder_static_call(TB_GraphBuilder* g, TB_FunctionPrototype* proto, TB_Symbol* target, int mem_var, int nargs) { - assert(g->val_cnt >= nargs); - TB_Function* f = g->f; - - TB_Node* target_node; - { - target_node = tb_alloc_node(f, TB_SYMBOL, TB_TYPE_PTR, 1, sizeof(TB_NodeSymbol)); - set_input(f, target_node, f->root_node, 0); - TB_NODE_SET_EXTRA(target_node, TB_NodeSymbol, .sym = target); - - target_node = tb_pass_gvn_node(g->f, target_node); - } - - // construct call op - g->val_cnt -= nargs; - { - size_t proj_count = 2 + (proto->return_count > 1 ? proto->return_count : 1); - - TB_Node* n = tb_alloc_node(f, TB_CALL, TB_TYPE_TUPLE, 3 + nargs, sizeof(TB_NodeCall) + (sizeof(TB_Node*)*proj_count)); - set_input(f, n, target_node, 2); - FOREACH_N(i, 0, nargs) { - set_input(f, n, g->vals[g->val_cnt + i], i + 3); - } - - TB_NodeCall* c = TB_NODE_GET_EXTRA(n); - c->proj_count = proj_count; - c->proto = proto; - - // control proj - TB_Node* cproj = tb__make_proj(f, TB_TYPE_CONTROL, n, 0); - set_input(f, n, xfer_ctrl(g, cproj), 0); - - // memory proj - TB_Node* mproj = tb__make_proj(f, TB_TYPE_MEMORY, n, 1); - set_input(f, n, g->vals[mem_var], 1); - - // create data projections - TB_PrototypeParam* rets = TB_PROTOTYPE_RETURNS(proto); - FOREACH_N(i, 0, proto->return_count) { - c->projs[i + 2] = tb__make_proj(f, rets[i].dt, n, i + 2); - } - - // we'll slot a NULL so it's easy to tell when it's empty - if (proto->return_count == 0) { - c->projs[2] = NULL; - } - - c->projs[0] = cproj; - c->projs[1] = mproj; - - g->vals[mem_var] = mproj; - add_input_late(f, get_callgraph(f), n); - - // push all returns - assert(proto->return_count < 2); - FOREACH_N(i, 0, proto->return_count) { - push(g, c->projs[2 + i]); - } - } -} - -void tb_builder_ret(TB_GraphBuilder* g, int count) { - TB_Function* f = g->f; - TB_Node* mem_state = g->vals[0]; - - assert(g->val_cnt >= count); - g->val_cnt -= count; - TB_Node** args = &g->vals[g->val_cnt]; - - // allocate return node - TB_Node* ret = f->ret_node; - TB_Node* ctrl = ret->inputs[0]; - assert(ctrl->type == TB_REGION); - - // add to PHIs - assert(ret->input_count >= 3 + count); - add_input_late(f, ret->inputs[1], mem_state); - - size_t i = 3; - for (; i < count + 3; i++) { - TB_Node* v = args[i - 3]; - assert(ret->inputs[i]->dt.raw == v->dt.raw && "datatype mismatch"); - add_input_late(f, ret->inputs[i], v); - } - - size_t phi_count = ret->input_count; - for (; i < phi_count; i++) { - // put poison in the leftovers? - log_warn("%s: ir: generated poison due to inconsistent number of returned values", f->super.name); - - TB_Node* poison = tb_alloc_node(f, TB_POISON, ret->inputs[i]->dt, 1, 0); - set_input(f, poison, f->ret_node, 0); - - poison = tb__gvn(f, poison, 0); - add_input_late(f, ret->inputs[i], poison); - } - - // basically just tb_inst_goto without the memory PHI (we did it earlier) - TB_Node* n = g->bot_ctrl; - g->bot_ctrl = NULL; - - add_input_late(f, ctrl, n); -} diff --git a/vendor/tb/src/objects/coff.c b/vendor/tb/src/objects/coff.c deleted file mode 100644 index 36d83971..00000000 --- a/vendor/tb/src/objects/coff.c +++ /dev/null @@ -1,669 +0,0 @@ -#include "coff.h" - -static int compare_relocs(const void* a, const void* b) { - const COFF_ImageReloc* aa = (const COFF_ImageReloc*) a; - const COFF_ImageReloc* bb = (const COFF_ImageReloc*) b; - - return aa->VirtualAddress - bb->VirtualAddress; -} - -static int compare_pdata(const void* a, const void* b) { - const uint32_t* sym_a = (const uint32_t*) a; - const uint32_t* sym_b = (const uint32_t*) b; - - return (sym_a[0] > sym_b[0]) - (sym_a[0] < sym_b[0]); -} - -static COFF_Symbol section_sym(const char* name, int num, int sc) { - COFF_Symbol s = { .section_number = num, .storage_class = sc, .aux_symbols_count = 1 }; - strncpy((char*) s.short_name, name, 8); - return s; -} - -static COFF_AuxSectionSymbol section_aux_sym(COFF_SectionHeader* s, int num) { - return (COFF_AuxSectionSymbol){ - .length = s->raw_data_size, - .reloc_count = s->num_reloc, - .number = num, - }; -} - -typedef struct { - size_t pos, size; -} StringTable; - -// COFF unwind info is two sections, .pdata and .xdata -struct COFF_UnwindInfo { - size_t section_num, patch_count; - - TB_ExportChunk* xdata_chunk; - TB_ExportChunk* pdata_chunk; - TB_ExportChunk* pdata_relocs; - - COFF_SectionHeader xdata_header; - COFF_SectionHeader pdata_header; -}; - -static void generate_unwind_info(TB_Arena* dst_arena, COFF_UnwindInfo* restrict u, const ICodeGen* restrict code_gen, size_t xdata_section, TB_ModuleSection* section) { - size_t count = dyn_array_length(section->funcs); - *u = (COFF_UnwindInfo){ .patch_count = count * 3, .section_num = (xdata_section+1) * 2 }; - - // generate pdata - u->pdata_chunk = tb_export_make_chunk(dst_arena, count * 3 * sizeof(uint32_t)); - uint32_t* pdata = (uint32_t*) u->pdata_chunk->data; - - bool overflow = count * 3 >= 0xFFFF; - u->pdata_relocs = tb_export_make_chunk(dst_arena, (overflow + count*3) * sizeof(COFF_ImageReloc)); - COFF_ImageReloc* relocs = (COFF_ImageReloc*) u->pdata_relocs->data; - - if (overflow) { - *relocs++ = (COFF_ImageReloc){ .VirtualAddress = count * 3 }; - } - - // generate xdata - TB_Emitter xdata = { 0 }; - DynArray(TB_FunctionOutput*) funcs = section->funcs; - dyn_array_for(i, funcs) { - TB_FunctionOutput* out_f = funcs[i]; - - out_f->unwind_info = xdata.count; - code_gen->emit_win64eh_unwind_info(&xdata, out_f, out_f->stack_usage); - out_f->unwind_size = xdata.count - out_f->unwind_info; - - // write pdata - size_t j = i*3; - pdata[j+0] = 0; - pdata[j+1] = out_f->code_size - out_f->nop_pads; - pdata[j+2] = out_f->unwind_info; - - // pdata has relocations - uint32_t sym = out_f->parent->super.symbol_id; - relocs[j + 0] = (COFF_ImageReloc){ - .Type = IMAGE_REL_AMD64_ADDR32NB, - .SymbolTableIndex = sym, - .VirtualAddress = j * 4 - }; - - relocs[j + 1] = (COFF_ImageReloc){ - .Type = IMAGE_REL_AMD64_ADDR32NB, - .SymbolTableIndex = sym, - .VirtualAddress = (j * 4) + 4 - }; - - relocs[j + 2] = (COFF_ImageReloc){ - .Type = IMAGE_REL_AMD64_ADDR32NB, - .SymbolTableIndex = u->section_num, // xdata section - .VirtualAddress = (j * 4) + 8 - }; - } - - u->xdata_chunk = tb_export_make_chunk(dst_arena, xdata.count); - memcpy(u->xdata_chunk->data, xdata.data, xdata.count); - tb_platform_heap_free(xdata.data); - - // generate COFF headers - u->pdata_header = (COFF_SectionHeader){ - .name = ".pdata", - .characteristics = COFF_CHARACTERISTICS_RODATA | (u->patch_count >= 0xFFFF ? IMAGE_SCN_LNK_NRELOC_OVFL : 0), - .raw_data_size = u->pdata_chunk->size, - .num_reloc = (u->patch_count >= 0xFFFF ? 0xFFFF : u->patch_count), - }; - - u->xdata_header = (COFF_SectionHeader){ - .name = ".xdata", - .characteristics = COFF_CHARACTERISTICS_RODATA, - .raw_data_size = u->xdata_chunk->size, - }; -} - -#define WRITE(data, size) (memcpy(&output[write_pos], data, size), write_pos += (size)) -TB_ExportBuffer tb_coff_write_output(TB_Module* m, TB_Arena* dst_arena, const IDebugFormat* dbg) { - TB_Arena* arena = get_temporary_arena(m); - - ExportList exports; - CUIK_TIMED_BLOCK("layout section") { - exports = tb_module_layout_sections(m); - } - - // accumulate all sections - DynArray(TB_ModuleSection) sections = m->sections; - - // mark correct flags on sections - #define ON(mask, a, b) flags |= (sections[i].flags & mask ? (a) : (b)) - dyn_array_for(i, sections) { - uint32_t flags = TB_COFF_SECTION_READ; - ON(TB_MODULE_SECTION_WRITE, TB_COFF_SECTION_WRITE, 0); - ON(TB_MODULE_SECTION_EXEC, TB_COFF_SECTION_EXECUTE | TB_COFF_SECTION_CODE, TB_COFF_SECTION_INIT); - if (sections[i].comdat.type != 0) flags |= TB_COFF_SECTION_COMDAT; - sections[i].export_flags = flags; - } - #undef ON - - int dbg_section_count = (dbg ? dbg->number_of_debug_sections(m) : 0); - int section_count = dyn_array_length(sections) + dbg_section_count; - const ICodeGen* restrict code_gen = tb__find_code_generator(m); - - // mark each with a unique id - size_t unique_id_counter = section_count * 2; - CUIK_TIMED_BLOCK("alloc symbol IDs") { - dyn_array_for(i, sections) { - // unwind info - if (code_gen->emit_win64eh_unwind_info && sections[i].funcs) { - unique_id_counter += 4; - } - } - - dyn_array_for(i, sections) { - DynArray(TB_FunctionOutput*) funcs = sections[i].funcs; - DynArray(TB_Global*) globals = sections[i].globals; - - dyn_array_for(i, funcs) { - funcs[i]->parent->super.symbol_id = unique_id_counter++; - } - - dyn_array_for(i, globals) { - globals[i]->super.symbol_id = unique_id_counter++; - } - } - - FOREACH_N(i, 0, exports.count) { - exports.data[i]->super.symbol_id = unique_id_counter++; - } - } - - dyn_array_for(i, sections) { - sections[i].section_num = 1 + i; - - // make unwind info for each section with functions - if (code_gen->emit_win64eh_unwind_info && sections[i].funcs) { - COFF_UnwindInfo* u = tb_arena_alloc(arena, sizeof(COFF_UnwindInfo)); - generate_unwind_info(dst_arena, u, code_gen, section_count, §ions[i]); - - sections[i].unwind = u; - section_count += 2; // .pdata + .xdata - } - } - - size_t string_table_cap = unique_id_counter; - const char** string_table = tb_arena_alloc(arena, string_table_cap * sizeof(char*)); - - // COFF file header & section headers - COFF_FileHeader header = { - .section_count = section_count, - .timestamp = 1056582000u, // my birthday since that's consistent :P - .symbol_count = unique_id_counter, - .symbol_table = 0, - .flags = IMAGE_FILE_LINE_NUMS_STRIPPED - }; - - switch (m->target_arch) { - case TB_ARCH_X86_64: header.machine = COFF_MACHINE_AMD64; break; - case TB_ARCH_AARCH64: header.machine = COFF_MACHINE_ARM64; break; - default: tb_todo(); - } - - // calculate output size & layout object - size_t output_size = sizeof(COFF_FileHeader); - output_size += section_count * sizeof(COFF_SectionHeader); - dyn_array_for(i, sections) { - sections[i].raw_data_pos = output_size; - output_size += sections[i].total_size; - - COFF_UnwindInfo* unwind = sections[i].unwind; - if (unwind != NULL) { - unwind->pdata_header.raw_data_pos = output_size; - output_size += unwind->pdata_header.raw_data_size; - - unwind->xdata_header.raw_data_pos = output_size; - output_size += unwind->xdata_header.raw_data_size; - - unwind->pdata_header.pointer_to_reloc = output_size; - output_size += unwind->pdata_relocs->size; - } - } - - TB_SectionGroup debug_sections = { 0 }; - if (dbg) CUIK_TIMED_BLOCK("generate debug") { - debug_sections = dbg->generate_debug_info(m, arena); - } - - FOREACH_N(i, 0, debug_sections.length) { - debug_sections.data[i].virtual_address = output_size; - output_size += debug_sections.data[i].raw_data.length; - } - - // calculate relocation layout - output_size = tb__layout_relocations(m, sections, code_gen, output_size, sizeof(COFF_ImageReloc)); - - FOREACH_N(i, 0, debug_sections.length) { - size_t reloc_count = debug_sections.data[i].relocation_count; - - // we're storing the relocation array's file pos here - debug_sections.data[i].user_data = (void*) (uintptr_t) output_size; - output_size += reloc_count * sizeof(COFF_ImageReloc); - } - - header.symbol_table = output_size; - output_size += header.symbol_count * sizeof(COFF_Symbol); - - // first 4 bytes of the table are the size of the table - StringTable strtbl = { .pos = output_size, .size = 4 }; - - // compute string table size - dyn_array_for(i, sections) { - DynArray(TB_FunctionOutput*) funcs = sections[i].funcs; - DynArray(TB_Global*) globals = sections[i].globals; - - // is a man not entitled to the dogshit of his cache locality? - dyn_array_for(i, funcs) { - size_t name_len = strlen(funcs[i]->parent->super.name); - if (name_len >= 8) strtbl.size += name_len + 1; - } - - dyn_array_for(i, globals) { - size_t name_len = strlen(globals[i]->super.name); - if (name_len >= 8) strtbl.size += name_len + 1; - } - } - - FOREACH_N(i, 0, exports.count) { - size_t name_len = strlen(exports.data[i]->super.name); - if (name_len >= 8) strtbl.size += name_len + 1; - } - - output_size += strtbl.size; - - //////////////////////////////// - // write output - //////////////////////////////// - TB_ExportBuffer buffer = { 0 }; - - CUIK_TIMED_BLOCK("write output") { - CUIK_TIMED_BLOCK("write headers") { - TB_ExportChunk* headers = tb_export_make_chunk(dst_arena, sizeof(COFF_FileHeader) + (sizeof(COFF_SectionHeader) * section_count)); - - // write COFF header - COFF_FileHeader* file = (COFF_FileHeader*) headers->data; - *file = header; - - // write sections headers - COFF_SectionHeader* sec_headers = (COFF_SectionHeader*) (file + 1); - dyn_array_for(i, sections) { - COFF_SectionHeader header = { - .characteristics = sections[i].export_flags, - .raw_data_size = sections[i].total_size, - .raw_data_pos = sections[i].raw_data_pos, - .pointer_to_reloc = sections[i].reloc_pos - }; - - size_t len = strlen(sections[i].name); - memcpy(header.name, sections[i].name, len > 8 ? 8 : len); - - if (sections[i].reloc_count >= 0xFFFF) { - header.num_reloc = 0xFFFF; - header.characteristics |= IMAGE_SCN_LNK_NRELOC_OVFL; - } else { - header.num_reloc = sections[i].reloc_count; - } - - *sec_headers++ = header; - } - - FOREACH_N(i, 0, debug_sections.length) { - COFF_SectionHeader header = { - .characteristics = COFF_CHARACTERISTICS_DEBUG, - .raw_data_size = debug_sections.data[i].raw_data.length, - .raw_data_pos = debug_sections.data[i].virtual_address, - .pointer_to_reloc = (uintptr_t) debug_sections.data[i].user_data - }; - - /*if (dbg == &tb__codeview_debug_format) { - header.characteristics |= IMAGE_SCN_MEM_DISCARDABLE; - }*/ - - TB_Slice name = debug_sections.data[i].name; - assert(name.length <= 8); - memcpy(header.name, name.data, name.length); - - size_t reloc_count = debug_sections.data[i].relocation_count; - if (reloc_count >= 0xFFFF) { - header.num_reloc = 0xFFFF; - header.characteristics |= IMAGE_SCN_LNK_NRELOC_OVFL; - } else { - header.num_reloc = reloc_count; - } - - *sec_headers++ = header; - } - - dyn_array_for(i, sections) { - // pdata then xdata - if (sections[i].unwind) { - memcpy(sec_headers + 0, §ions[i].unwind->pdata_header, sizeof(COFF_SectionHeader)); - memcpy(sec_headers + 1, §ions[i].unwind->xdata_header, sizeof(COFF_SectionHeader)); - sec_headers += 2; - } - } - - tb_export_append_chunk(&buffer, headers); - } - - // write raw data - dyn_array_for(i, sections) { - TB_ExportChunk* sec = tb_export_make_chunk(dst_arena, sections[i].total_size); - tb_helper_write_section(m, 0, §ions[i], sec->data, 0); - tb_export_append_chunk(&buffer, sec); - - COFF_UnwindInfo* unwind = sections[i].unwind; - if (unwind != NULL) { - tb_export_append_chunk(&buffer, unwind->pdata_chunk); - tb_export_append_chunk(&buffer, unwind->xdata_chunk); - - assert(unwind->pdata_header.pointer_to_reloc == buffer.total); - tb_export_append_chunk(&buffer, unwind->pdata_relocs); - } - } - - FOREACH_N(i, 0, debug_sections.length) { - TB_ExportChunk* sec = tb_export_make_chunk(dst_arena, debug_sections.data[i].raw_data.length); - memcpy(sec->data, debug_sections.data[i].raw_data.data, debug_sections.data[i].raw_data.length); - tb_export_append_chunk(&buffer, sec); - } - - // write relocations - dyn_array_for(i, sections) { - size_t reloc_count = sections[i].reloc_count; - DynArray(TB_FunctionOutput*) funcs = sections[i].funcs; - DynArray(TB_Global*) globals = sections[i].globals; - - TB_ExportChunk* relocations = tb_export_make_chunk(dst_arena, reloc_count * sizeof(COFF_ImageReloc)); - COFF_ImageReloc* relocs = (COFF_ImageReloc*) relocations->data; - - dyn_array_for(j, funcs) { - TB_FunctionOutput* func_out = funcs[j]; - size_t source_offset = func_out->code_pos; - - for (TB_SymbolPatch* p = func_out->first_patch; p; p = p->next) { - if (p->internal) continue; - - size_t actual_pos = source_offset + p->pos; - - TB_Symbol* target = p->target; - if (target->tag == TB_SYMBOL_EXTERNAL) { - TB_Symbol* resolved = atomic_load_explicit(&((TB_External*) target)->resolved, memory_order_relaxed); - if (resolved) target = resolved; - } - - size_t symbol_id = target->symbol_id; - assert(symbol_id != 0); - - if (target->tag == TB_SYMBOL_FUNCTION || target->tag == TB_SYMBOL_EXTERNAL) { - *relocs++ = (COFF_ImageReloc){ - .Type = IMAGE_REL_AMD64_REL32, - .SymbolTableIndex = symbol_id, - .VirtualAddress = actual_pos - }; - } else if (target->tag == TB_SYMBOL_GLOBAL) { - TB_Global* target_global = (TB_Global*) target; - bool is_tls = sections[target_global->parent].flags & TB_MODULE_SECTION_TLS; - *relocs++ = (COFF_ImageReloc){ - .Type = is_tls ? IMAGE_REL_AMD64_SECREL : IMAGE_REL_AMD64_REL32, - .SymbolTableIndex = symbol_id, - .VirtualAddress = actual_pos - }; - } else { - tb_todo(); - } - } - } - - dyn_array_for(j, globals) { - TB_Global* g = globals[j]; - - FOREACH_N(k, 0, g->obj_count) { - size_t actual_pos = g->pos + g->objects[k].offset; - - if (g->objects[k].type == TB_INIT_OBJ_RELOC) { - const TB_Symbol* s = g->objects[k].reloc; - - *relocs++ = (COFF_ImageReloc){ - .Type = IMAGE_REL_AMD64_ADDR64, - .SymbolTableIndex = s->symbol_id, - .VirtualAddress = actual_pos - }; - } - } - } - - assert((relocs - (COFF_ImageReloc*) relocations->data) == reloc_count); - tb_export_append_chunk(&buffer, relocations); - - assert(relocations->pos == sections[i].reloc_pos); - } - - FOREACH_N(i, 0, debug_sections.length) { - TB_ExportChunk* relocations = tb_export_make_chunk(dst_arena, debug_sections.data[i].relocation_count * sizeof(COFF_ImageReloc)); - COFF_ImageReloc* relocs = (COFF_ImageReloc*) relocations->data; - - FOREACH_N(j, 0, debug_sections.data[i].relocation_count) { - TB_ObjectReloc* in_reloc = &debug_sections.data[i].relocations[j]; - - int type = 0; - switch (in_reloc->type) { - case TB_OBJECT_RELOC_SECREL: type = IMAGE_REL_AMD64_SECREL; break; - case TB_OBJECT_RELOC_SECTION: type = IMAGE_REL_AMD64_SECTION; break; - case TB_OBJECT_RELOC_ADDR32NB:type = IMAGE_REL_AMD64_ADDR32NB; break; - default: tb_todo(); - } - - *relocs++ = (COFF_ImageReloc){ - .Type = type, - .SymbolTableIndex = in_reloc->symbol_index, - .VirtualAddress = in_reloc->virtual_address - }; - } - - tb_export_append_chunk(&buffer, relocations); - assert(relocations->pos == (uintptr_t) debug_sections.data[i].user_data); - } - - // write symbols - uint32_t string_table_mark = 4; - size_t string_table_length = 0; - CUIK_TIMED_BLOCK("write symbols") { - TB_ExportChunk* symtab = tb_export_make_chunk(dst_arena, header.symbol_count * sizeof(COFF_Symbol)); - - size_t write_pos = 0; - uint8_t* output = symtab->data; - - size_t symbol_count = 1; - dyn_array_for(i, sections) { - COFF_Symbol s = { - .section_number = symbol_count, - .storage_class = IMAGE_SYM_CLASS_STATIC, - .aux_symbols_count = 1 - }; - strncpy((char*) s.short_name, sections[i].name, 8); - WRITE(&s, sizeof(s)); - - COFF_AuxSectionSymbol aux = { - .length = sections[i].total_size, - .reloc_count = sections[i].reloc_count, - .number = symbol_count, - .selection = sections[i].comdat.type ? 2 /* pick any */ : 0, - }; - WRITE(&aux, sizeof(aux)); - symbol_count++; - } - - FOREACH_N(i, 0, debug_sections.length) { - COFF_Symbol s = { - .section_number = symbol_count, - .storage_class = IMAGE_SYM_CLASS_STATIC, - .aux_symbols_count = 1 - }; - - int l = debug_sections.data[i].name.length; - if (l > 8) l = 8; - strncpy((char*) s.short_name, (const char*) debug_sections.data[i].name.data, l); - - WRITE(&s, sizeof(s)); - - COFF_AuxSectionSymbol aux = { - .length = debug_sections.data[i].raw_data.length, - .reloc_count = debug_sections.data[i].relocation_count, - .number = symbol_count, - }; - WRITE(&aux, sizeof(aux)); - symbol_count++; - } - - dyn_array_for(i, sections) { - int section_num = sections[i].section_num; - COFF_UnwindInfo* u = sections[i].unwind; - if (u != NULL) { - COFF_Symbol s[2] = { - { // pdata - .short_name = ".pdata", .section_number = symbol_count, - .storage_class = IMAGE_SYM_CLASS_STATIC, .aux_symbols_count = 1 - }, - { // xdata - .short_name = ".xdata", .section_number = symbol_count + 1, - .storage_class = IMAGE_SYM_CLASS_STATIC, .aux_symbols_count = 1 - }, - }; - - COFF_AuxSectionSymbol aux[2] = { - { .length = u->pdata_chunk->size, .reloc_count = u->patch_count, .number = symbol_count }, // pdata - { .length = u->xdata_chunk->size, .number = symbol_count + 1 }, // .xdata - }; - - if (i > 0) { - // COMDAT functions go here - // associative with their matching text section - aux[0].selection = 5; - aux[0].number = section_num; - } - - WRITE(&s[0], sizeof(COFF_Symbol)); - WRITE(&aux[0], sizeof(COFF_AuxSectionSymbol)); - WRITE(&s[1], sizeof(COFF_Symbol)); - WRITE(&aux[1], sizeof(COFF_AuxSectionSymbol)); - - symbol_count += 2; - } - } - - dyn_array_for(i, sections) { - int section_num = sections[i].section_num; - DynArray(TB_FunctionOutput*) funcs = sections[i].funcs; - DynArray(TB_Global*) globals = sections[i].globals; - - dyn_array_for(j, funcs) { - TB_FunctionOutput* out_f = funcs[j]; - - bool is_extern = out_f->linkage == TB_LINKAGE_PUBLIC; - COFF_Symbol sym = { - .value = out_f->code_pos, - .section_number = section_num, - .storage_class = is_extern ? IMAGE_SYM_CLASS_EXTERNAL : IMAGE_SYM_CLASS_STATIC - }; - - const char* name = out_f->parent->super.name; - size_t name_len = strlen(name); - assert(name_len < UINT16_MAX); - if (name_len >= 8) { - sym.long_name[0] = 0; // this value is 0 for long names - sym.long_name[1] = string_table_mark; - - string_table[string_table_length++] = (char*)name; - string_table_mark += name_len + 1; - } else { - memcpy(sym.short_name, name, name_len + 1); - } - - WRITE(&sym, sizeof(sym)); - } - - dyn_array_for(j, globals) { - TB_Global* g = globals[j]; - bool is_extern = g->linkage == TB_LINKAGE_PUBLIC; - - assert(section_num == g->parent + 1); - COFF_Symbol sym = { - .value = g->pos, - .section_number = section_num, - .storage_class = is_extern ? IMAGE_SYM_CLASS_EXTERNAL : IMAGE_SYM_CLASS_STATIC - }; - - if (g->super.name[0] != 0) { - size_t name_len = strlen(g->super.name); - assert(name_len < UINT16_MAX); - - if (name_len >= 8) { - sym.long_name[0] = 0; // this value is 0 for long names - sym.long_name[1] = string_table_mark; - - string_table[string_table_length++] = g->super.name; - string_table_mark += name_len + 1; - } else { - memcpy(sym.short_name, g->super.name, name_len + 1); - } - } else { - snprintf((char*) sym.short_name, 8, "$%06zx", g->super.symbol_id); - } - - WRITE(&sym, sizeof(sym)); - } - } - - FOREACH_N(i, 0, exports.count) { - TB_External* ext = exports.data[i]; - COFF_Symbol sym = { - .value = 0, - .section_number = 0, - .storage_class = IMAGE_SYM_CLASS_EXTERNAL - }; - - size_t name_len = strlen(ext->super.name); - assert(name_len < UINT16_MAX); - - if (name_len >= 8) { - sym.long_name[0] = 0; // this value is 0 for long names - sym.long_name[1] = string_table_mark; - - string_table[string_table_length++] = ext->super.name; - string_table_mark += name_len + 1; - } else { - memcpy(sym.short_name, ext->super.name, name_len + 1); - } - - WRITE(&sym, sizeof(sym)); - } - - assert(write_pos == symtab->size); - tb_export_append_chunk(&buffer, symtab); - } - - // write string table - { - // assert(write_pos == strtbl.pos); - TB_ExportChunk* chunk = tb_export_make_chunk(dst_arena, strtbl.size); - - memcpy(chunk->data, &string_table_mark, sizeof(uint32_t)); - - size_t j = 4; - FOREACH_N(i, 0, string_table_length) { - const char* s = string_table[i]; - size_t l = strlen(s) + 1; - - memcpy(&chunk->data[j], s, l), j += l; - } - - tb_export_append_chunk(&buffer, chunk); - } - } - - tb_arena_clear(arena); - return buffer; -} diff --git a/vendor/tb/src/objects/coff.h b/vendor/tb/src/objects/coff.h deleted file mode 100644 index 85d52468..00000000 --- a/vendor/tb/src/objects/coff.h +++ /dev/null @@ -1,380 +0,0 @@ -// https://github.com/dotnet/runtime/blob/main/docs/design/specs/PE-COFF.md -#pragma once -#include "../tb_internal.h" - -/*#if TB_HOST_ARCH == TB_HOST_X86_64 -#include -#endif*/ - -// IMAGE_SCN_CNT_CODE | IMAGE_SCN_MEM_EXECUTE | IMAGE_SCN_MEM_READ | IMAGE_SCN_ALIGN_16BYTES -#define COFF_CHARACTERISTICS_TEXT 0x60500020u -// IMAGE_SCN_CNT_INITIALIZED_DATA | IMAGE_SCN_MEM_WRITE | IMAGE_SCN_MEM_READ -#define COFF_CHARACTERISTICS_DATA 0xC0000040u -// IMAGE_SCN_CNT_INITIALIZED_DATA | IMAGE_SCN_MEM_READ -#define COFF_CHARACTERISTICS_RODATA 0x40000040u -// IMAGE_SCN_CNT_UNINITIALIZED_DATA | IMAGE_SCN_MEM_WRITE | IMAGE_SCN_MEM_READ | -// IMAGE_SCN_ALIGN_16BYTES -#define COFF_CHARACTERISTICS_BSS 0xC0500080u -// IMAGE_SCN_CNT_INITIALIZED_DATA | IMAGE_SCN_ALIGN_8BYTES | IMAGE_SCN_MEM_READ | -// IMAGE_SCN_MEM_DISCARDABLE -#define COFF_CHARACTERISTICS_DEBUG 0x40100040u - -#define MD5_HASHBYTES 16 - -typedef struct { - uint16_t sig1; - uint16_t sig2; - uint16_t version; - uint16_t machine; - uint32_t timestamp; - uint32_t size_of_data; - uint16_t ordinal_hint; - - uint16_t type : 2; - uint16_t name_type : 3; - uint16_t reserved : 11; -} COFF_ImportHeader; - -typedef struct { - uint32_t import_lookup_table; // RVA - uint32_t timestamp; - uint32_t forwarder_chain; - uint32_t name; - uint32_t import_address_table; // RVA; Thunk table -} COFF_ImportDirectory; - -// NOTE: Symbols, relocations, and line numbers are 2 byte packed -#pragma pack(push, 2) -typedef struct COFF_AuxSectionSymbol { - uint32_t length; // section length - uint16_t reloc_count; // number of relocation entries - uint16_t lineno_count; // number of line numbers - uint32_t checksum; // checksum for communal - int16_t number; // section number to associate with - uint8_t selection; // communal selection type - uint8_t reserved; - int16_t high_bits; // high bits of the section number -} COFF_AuxSectionSymbol; -static_assert(sizeof(COFF_AuxSectionSymbol) == 18, "COFF Aux Section Symbol size != 18 bytes"); - -typedef union COFF_SymbolUnion { - COFF_Symbol s; - COFF_AuxSectionSymbol a; -} COFF_SymbolUnion; -#pragma pack(pop) - -#pragma pack(push, 1) -typedef struct { - uint32_t virtual_address; - uint32_t size; -} PE_ImageDataDirectory; - -typedef struct { - uint64_t raw_data_start; - uint64_t raw_data_end; - uint64_t index_address; - uint64_t callback_address; - uint32_t zero_fill_size; - uint32_t flags; -} PE_TLSDirectory; - -#define IMAGE_NUMBEROF_DIRECTORY_ENTRIES 16 -typedef struct { - uint16_t magic; - uint8_t major_linker_version; - uint8_t minor_linker_version; - uint32_t size_of_code; - uint32_t size_of_initialized_data; - uint32_t size_of_uninitialized_data; - uint32_t entrypoint; - uint32_t base_of_code; - uint64_t image_base; - uint32_t section_alignment; - uint32_t file_alignment; - uint16_t major_os_ver; - uint16_t minor_os_ver; - uint16_t major_image_ver; - uint16_t minor_image_ver; - uint16_t major_subsystem_ver; - uint16_t minor_subsystem_ver; - uint32_t win32_version_value; - uint32_t size_of_image; - uint32_t size_of_headers; - uint32_t checksum; - uint16_t subsystem; - uint16_t dll_characteristics; - uint64_t size_of_stack_reserve; - uint64_t size_of_stack_commit; - uint64_t size_of_heap_reserve; - uint64_t size_of_heap_commit; - uint32_t loader_flags; - uint32_t rva_size_count; - PE_ImageDataDirectory data_directories[IMAGE_NUMBEROF_DIRECTORY_ENTRIES]; -} PE_OptionalHeader64; - -typedef struct { // size 40 bytes - char name[8]; - uint32_t virtual_size; - uint32_t virtual_address; - uint32_t size_of_raw_data; - uint32_t pointer_to_raw_data; - uint32_t pointer_to_relocs; - uint32_t pointer_to_linenos; - uint16_t relocation_count; - uint16_t linenos_count; - uint32_t characteristics; -} PE_SectionHeader; - -typedef struct { - unsigned long ptrtype :5; // ordinal specifying pointer type (CV_ptrtype_e) - unsigned long ptrmode :3; // ordinal specifying pointer mode (CV_ptrmode_e) - unsigned long isflat32 :1; // true if 0:32 pointer - unsigned long isvolatile :1; // TRUE if volatile pointer - unsigned long isconst :1; // TRUE if const pointer - unsigned long isunaligned :1; // TRUE if unaligned pointer - unsigned long isrestrict :1; // TRUE if restricted pointer (allow agressive opts) - unsigned long size :6; // size of pointer (in bytes) - unsigned long ismocom :1; // TRUE if it is a MoCOM pointer (^ or %) - unsigned long islref :1; // TRUE if it is this pointer of member function with & ref-qualifier - unsigned long isrref :1; // TRUE if it is this pointer of member function with && ref-qualifier - unsigned long unused :10;// pad out to 32-bits for following cv_typ_t's -} CV_LFPointerAttribs; - -typedef struct { - uint16_t len; - uint16_t leaf; // LF_POINTER - uint32_t utype; // type index of the underlying type - CV_LFPointerAttribs attr; - union { - struct { - uint32_t pmclass; // index of containing class for pointer to member - uint16_t pmenum; // enumeration specifying pm format (CV_pmtype_e) - } pm; - } pbase; -} CV_LFPointer; - -typedef struct { - uint16_t packed :1; // true if structure is packed - uint16_t ctor :1; // true if constructors or destructors present - uint16_t ovlops :1; // true if overloaded operators present - uint16_t isnested :1; // true if this is a nested class - uint16_t cnested :1; // true if this class contains nested types - uint16_t opassign :1; // true if overloaded assignment (=) - uint16_t opcast :1; // true if casting methods - uint16_t fwdref :1; // true if forward reference (incomplete defn) - uint16_t scoped :1; // scoped definition - uint16_t hasuniquename :1; // true if there is a decorated name following the regular name - uint16_t sealed :1; // true if class cannot be used as a base class - uint16_t hfa :2; // CV_HFA_e - uint16_t intrinsic :1; // true if class is an intrinsic type (e.g. __m128d) - uint16_t mocom :2; // CV_MOCOM_UDT_e -} CV_Prop; - -typedef struct { - uint16_t access :2; // access protection CV_access_t - uint16_t mprop :3; // method properties CV_methodprop_t - uint16_t pseudo :1; // compiler generated fcn and does not exist - uint16_t noinherit :1; // true if class cannot be inherited - uint16_t noconstruct :1; // true if class cannot be constructed - uint16_t compgenx :1; // compiler generated fcn and does exist - uint16_t sealed :1; // true if method cannot be overridden - uint16_t unused :6; // unused -} CV_FieldAttrib; - -typedef struct { - uint16_t len; - uint16_t leaf; // LF_FIELDLIST -} CV_LFFieldList; - -typedef struct { - uint16_t len; - uint16_t leaf; // LF_ALIAS - uint32_t utype; - char name[]; -} CV_LFAlias; - -typedef struct { - uint16_t leaf; // LF_LONG - int32_t val; // signed 32-bit value -} CV_LFLong; - -typedef struct { - uint16_t leaf; // LF_QUAD - int64_t val; // signed 64-bit value -} CV_LFQuad; - -typedef struct { - uint16_t leaf; // LF_VARSTRING - uint8_t val[]; -} CV_LFVarString; - -typedef struct { - uint16_t leaf; // LF_MEMBER - CV_FieldAttrib attr; // attribute mask - uint32_t index; // index of type record for field - // variable length offset of field followed - // by length prefixed name of field - uint8_t offset[]; -} CV_LFMember; - -typedef struct CV_LFStruct { - uint16_t len; - uint16_t leaf; // LF_CLASS, LF_STRUCT, LF_INTERFACE - uint16_t count; // count of number of elements in class - CV_Prop property;// property attribute field (prop_t) - uint32_t field; // type index of LF_FIELD descriptor list - uint32_t derived; // type index of derived from list if not zero - uint32_t vshape; // type index of vshape table for this class - uint8_t data[]; // data describing length of structure in bytes and name -} CV_LFStruct; - -typedef enum { - CV_LOCAL_IS_PARAM = 1, // variable is a parameter - CV_LOCAL_IS_ADDR_TAKEN = 2, // address is taken - CV_LOCAL_IS_COMPILER_GEND = 4, // variable is compiler generated - CV_LOCAL_IS_AGGREGATE = 8, // the symbol is splitted in temporaries, which are treated by compiler as independent entities - CV_LOCAL_IS_AGGREGATED = 16, // Counterpart of fIsAggregate - tells that it is a part of a fIsAggregate symbol - CV_LOCAL_IS_ALIASED = 32, // variable has multiple simultaneous lifetimes - CV_LOCAL_IS_ALIAS = 64, // represents one of the multiple simultaneous lifetimes - CV_LOCAL_IS_RETURN_VALUE = 128, // represents a function return value - CV_LOCAL_IS_OPTIMIZED_OUT = 256, // variable has no lifetimes - CV_LOCAL_IS_ENREG_GLOBAL = 512, // variable is an enregistered global - CV_LOCAL_IS_ENREG_STATIC = 1024,// variable is an enregistered static -} CV_LocalVarFlags; - -// CV_Local is followed by CV_DefRange memory -typedef struct { - uint16_t reclen; // Record length - uint16_t rectyp; // S_LOCAL - uint32_t typind; // type index - uint16_t flags; // local var flags (CV_LocalVarFlags) - uint8_t name[]; // Name of this symbol, a null terminated array of UTF8 characters. -} CV_Local; - -typedef struct { - uint32_t offset_start; - uint16_t isect_start; - uint16_t cb_range; -} CV_AddressRange; - -// Represents the holes in overall address range, all address is pre-bbt. -// it is for compress and reduce the amount of relocations need. -typedef struct { - uint16_t gap_start_offset; // relative offset from the beginning of the live range. - uint16_t cb_range; // length of this gap. -} CV_AddressGap; - -// A live range of sub field of variable -typedef struct { - uint16_t reclen; // Record length - uint16_t rectyp; // S_DEFRANGE - uint32_t program; // DIA program to evaluate the value of the symbol - CV_AddressRange range; // Range of addresses where this program is valid - CV_AddressGap gaps[]; // The value is not available in following gaps -} CV_DefRange; - -typedef struct { - uint16_t reclen; // Record length - uint16_t rectyp; // S_DEFRANGE_FRAMEPOINTER_REL - int32_t local; - CV_AddressRange range; // Range of addresses where this program is valid - CV_AddressGap gaps[]; // The value is not available in following gaps -} CV_DefRangeFrameRel; - -typedef struct { - uint16_t reclen; // Record length - uint16_t rectyp; // S_REGREL32 - uint32_t off; // offset of symbol - uint32_t typind; // Type index or metadata token - uint16_t reg; // register index for symbol - uint8_t name[]; // Length-prefixed name -} CV_RegRel32; -#pragma pack(pop) - -// represents a CodeView type entry, they start with 16bits for length field -typedef struct CV_TypeEntry { - uint32_t key; // points to somewhere in the debug$T section, 0 is assumed to mean nothing - uint16_t value; // type index -} CV_TypeEntry; - -enum { - COFF_MACHINE_AMD64 = 0x8664, // AMD64 (K8) - COFF_MACHINE_ARM64 = 0xAA64, // ARM64 Little-Endian -}; - -enum { - S_LDATA32 = 0x110c, // Module-local symbol - S_GDATA32 = 0x110d, // Global data symbol - S_LPROC32_ID = 0x1146, - S_GPROC32_ID = 0x1147, - S_INLINESITE = 0x114d, // inlined function callsite. - S_INLINESITE_END = 0x114e, - S_PROC_ID_END = 0x114f, - S_FRAMEPROC = 0x1012, // extra frame and proc information - S_REGREL32 = 0x1111, // register relative address - S_LOCAL = 0x113e, // defines a local symbol in optimized code - S_DEFRANGE = 0x113f, // defines a single range of addresses in which symbol can be evaluated - S_DEFRANGE_FRAMEPOINTER_REL = 0x1142, // range for stack symbol. -}; - -// types -enum { - T_VOID = 0x0003, // void - T_BOOL08 = 0x0030, // 8 bit boolean - T_CHAR = 0x0010, // 8 bit signed - T_UCHAR = 0x0020, // 8 bit unsigned - T_INT1 = 0x0068, // 8 bit signed int - T_UINT1 = 0x0069, // 8 bit unsigned int - T_INT2 = 0x0072, // 16 bit signed int - T_UINT2 = 0x0073, // 16 bit unsigned int - T_INT4 = 0x0074, // 32 bit signed int - T_UINT4 = 0x0075, // 32 bit unsigned int - T_INT8 = 0x0076, // 64 bit signed int - T_UINT8 = 0x0077, // 64 bit unsigned int - T_REAL32 = 0x0040, // 32 bit real - T_REAL64 = 0x0041, // 64 bit real -}; - -enum { - LF_NUMERIC = 0x8000, - LF_CHAR = 0x8000, - LF_SHORT = 0x8001, - LF_USHORT = 0x8002, - LF_LONG = 0x8003, - LF_ULONG = 0x8004, - LF_REAL32 = 0x8005, - LF_REAL64 = 0x8006, - LF_REAL80 = 0x8007, - LF_REAL128 = 0x8008, - LF_QUADWORD = 0x8009, - LF_UQUADWORD = 0x800a, - LF_REAL48 = 0x800b, - LF_COMPLEX32 = 0x800c, - LF_COMPLEX64 = 0x800d, - LF_COMPLEX80 = 0x800e, - LF_COMPLEX128 = 0x800f, - LF_VARSTRING = 0x8010, - - LF_POINTER = 0x1002, - LF_PROCEDURE = 0x1008, - LF_ARGLIST = 0x1201, - LF_FIELDLIST = 0x1203, - LF_ARRAY = 0x1503, - LF_CLASS = 0x1504, - LF_STRUCTURE = 0x1505, - LF_UNION = 0x1506, - LF_ENUM = 0x1507, - LF_ALIAS = 0x150a, - LF_MEMBER = 0x150d, - LF_FUNC_ID = 0x1601, - - LF_STRING = 0x0082, - - // the idea is that if you need to pad a - // type you fill in the remaining space with a - // sequence of LF_PADs like this - // - // Your record's bytes: - // DATA LF_PAD2 LF_PAD1 LF_PAD0 - LF_PAD0 = 0x00f0, -}; diff --git a/vendor/tb/src/objects/coff_parse.c b/vendor/tb/src/objects/coff_parse.c deleted file mode 100644 index beae09b4..00000000 --- a/vendor/tb/src/objects/coff_parse.c +++ /dev/null @@ -1,53 +0,0 @@ -#include "coff.h" - -#if 0 -#define IMAGE_SYM_CLASS_EXTERNAL 0x0002 -#define IMAGE_SYM_CLASS_STATIC 0x0003 -#define IMAGE_SYM_CLASS_LABEL 0x0006 -#define IMAGE_SYM_CLASS_FILE 0x0067 - -// let's ignore error handling for now :p -// buffered reading i guess? -TB_ObjectFile* tb_object_parse_coff(const TB_Slice file) { - COFF_FileHeader* header = (COFF_FileHeader*) &file.data[0]; - - TB_ObjectFile* obj_file = tb_platform_heap_alloc(sizeof(TB_ObjectFile) + (header->section_count * sizeof(TB_ObjectSection))); - - // not using calloc since i only really wanna clear the header - memset(obj_file, 0, sizeof(TB_ObjectFile)); - obj_file->type = TB_OBJECT_FILE_COFF; - - switch (header->machine) { - case COFF_MACHINE_AMD64: obj_file->arch = TB_ARCH_X86_64; break; - case COFF_MACHINE_ARM64: obj_file->arch = TB_ARCH_AARCH64; break; - default: obj_file->arch = TB_ARCH_UNKNOWN; break; - } - - size_t string_table_pos = header->symbol_table + (header->symbol_count * sizeof(COFF_Symbol)); - - // Read string table - TB_Slice string_table = { - .length = file.length - string_table_pos, - .data = &file.data[string_table_pos] - }; - - obj_file->section_count = header->section_count; - FOREACH_N(i, 0, header->section_count) { - } - - obj_file->symbols = tb_platform_heap_alloc(header->symbol_count * sizeof(TB_ObjectSymbol)); - obj_file->symbol_count = 0; - - size_t sym_id = 0; - while (sym_id < header->symbol_count) { - size_t symbol_offset = header->symbol_table + (sym_id * sizeof(COFF_Symbol)); - COFF_Symbol* sym = (COFF_Symbol*) &file.data[symbol_offset]; - - } - - // trim the symbol table - obj_file->symbols = tb_platform_heap_realloc(obj_file->symbols, obj_file->symbol_count * sizeof(TB_ObjectSymbol)); - - return obj_file; -} -#endif diff --git a/vendor/tb/src/objects/elf64.c b/vendor/tb/src/objects/elf64.c deleted file mode 100644 index d6cc04e1..00000000 --- a/vendor/tb/src/objects/elf64.c +++ /dev/null @@ -1,289 +0,0 @@ -#include "../tb_internal.h" -#include -#include - -static bool is_nonlocal(const TB_Symbol* s) { - if (s->tag == TB_SYMBOL_GLOBAL) { - return ((TB_Global*) s)->linkage == TB_LINKAGE_PUBLIC; - } else if (s->tag == TB_SYMBOL_FUNCTION) { - return ((TB_Function*) s)->linkage == TB_LINKAGE_PUBLIC; - } else { - return true; - } -} - -static int put_symbol(TB_Emitter* stab, uint32_t name, uint8_t sym_info, uint16_t section_index, uint64_t value, uint64_t size) { - // Emit symbol - TB_Elf64_Sym sym = { - .name = name, - .info = sym_info, - .shndx = section_index, - .value = value, - .size = size - }; - tb_outs(stab, sizeof(sym), (uint8_t*)&sym); - return (stab->count / sizeof(TB_Elf64_Sym)) - 1; -} - -static void put_section_symbols(DynArray(TB_ModuleSection) sections, TB_Emitter* strtbl, TB_Emitter* stab, int t) { - dyn_array_for(i, sections) { - int sec_num = sections[i].section_num; - DynArray(TB_FunctionOutput*) funcs = sections[i].funcs; - DynArray(TB_Global*) globals = sections[i].globals; - - dyn_array_for(i, funcs) { - TB_FunctionOutput* out_f = funcs[i]; - const char* name_str = out_f->parent->super.name; - - uint32_t name = name_str ? tb_outstr_nul_UNSAFE(strtbl, name_str) : 0; - out_f->parent->super.symbol_id = put_symbol(stab, name, TB_ELF64_ST_INFO(t, TB_ELF64_STT_FUNC), sec_num, out_f->code_pos, out_f->code_size); - } - - int acceptable = t == TB_ELF64_STB_GLOBAL ? TB_LINKAGE_PUBLIC : TB_LINKAGE_PRIVATE; - dyn_array_for(i, globals) { - TB_Global* g = globals[i]; - if (g->linkage != acceptable) { - continue; - } - - uint32_t name = 0; - if (g->super.name) { - name = tb_outstr_nul_UNSAFE(strtbl, g->super.name); - } else { - char buf[8]; - snprintf(buf, 8, "$%d_%td", sec_num, i); - name = tb_outstr_nul_UNSAFE(strtbl, buf); - } - - g->super.symbol_id = put_symbol(stab, name, TB_ELF64_ST_INFO(t, TB_ELF64_STT_OBJECT), sec_num, g->pos, 0); - } - } -} - -#define WRITE(data, size) (memcpy(&output[write_pos], data, size), write_pos += (size)) -TB_ExportBuffer tb_elf64obj_write_output(TB_Module* m, TB_Arena* dst_arena, const IDebugFormat* dbg) { - ExportList exports; - CUIK_TIMED_BLOCK("layout section") { - exports = tb_module_layout_sections(m); - } - - const ICodeGen* restrict code_gen = tb__find_code_generator(m); - - uint16_t machine = 0; - switch (m->target_arch) { - case TB_ARCH_X86_64: machine = TB_EM_X86_64; break; - case TB_ARCH_AARCH64: machine = TB_EM_AARCH64; break; - case TB_ARCH_MIPS32: machine = TB_EM_MIPS; break; - case TB_ARCH_MIPS64: machine = TB_EM_MIPS; break; - default: tb_todo(); - } - - TB_Emitter strtbl = { 0 }; - tb_out_reserve(&strtbl, 1024); - tb_out1b(&strtbl, 0); // null string in the table - - TB_Elf64_Ehdr header = { - .ident = { - [TB_EI_MAG0] = 0x7F, // magic number - [TB_EI_MAG1] = 'E', - [TB_EI_MAG2] = 'L', - [TB_EI_MAG3] = 'F', - [TB_EI_CLASS] = 2, // 64bit ELF file - [TB_EI_DATA] = 1, // little-endian - [TB_EI_VERSION] = 1, // 1.0 - [TB_EI_OSABI] = 0, - [TB_EI_ABIVERSION] = 0 - }, - .type = TB_ET_REL, // relocatable - .version = 1, - .machine = machine, - .entry = 0, - - // section headers go at the end of the file - // and are filed in later. - .shoff = 0, - .flags = 0, - - .ehsize = sizeof(TB_Elf64_Ehdr), - - .shentsize = sizeof(TB_Elf64_Shdr), - .shstrndx = 1, - }; - - // accumulate all sections - DynArray(TB_ModuleSection) sections = m->sections; - - int dbg_section_count = (dbg ? dbg->number_of_debug_sections(m) : 0); - int section_count = 2 + dyn_array_length(sections) + dbg_section_count; - - size_t output_size = sizeof(TB_Elf64_Ehdr); - dyn_array_for(i, sections) { - sections[i].section_num = 3 + i; - sections[i].raw_data_pos = output_size; - output_size += sections[i].total_size; - } - - // calculate relocation layout - // each section with relocations needs a matching .rel section - output_size = tb__layout_relocations(m, sections, code_gen, output_size, sizeof(TB_Elf64_Rela)); - dyn_array_for(i, sections) { - if (sections[i].reloc_count > 0) { - section_count += 1; - tb_outs(&strtbl, 5, ".rela"); - } - - sections[i].name_pos = tb_outstr_nul_UNSAFE(&strtbl, sections[i].name); - } - - // calculate symbol IDs - TB_Emitter local_symtab = { 0 }, global_symtab = { 0 }; - tb_out_zero(&local_symtab, sizeof(TB_Elf64_Sym)); - dyn_array_for(i, sections) { - put_symbol(&local_symtab, sections[i].name_pos, TB_ELF64_ST_INFO(TB_ELF64_STB_LOCAL, TB_ELF64_STT_SECTION), 1 + i, 0, 0); - } - - // .rela sections - dyn_array_for(i, sections) if (sections[i].reloc_count) { - put_symbol(&local_symtab, sections[i].name_pos - 5, TB_ELF64_ST_INFO(TB_ELF64_STB_LOCAL, TB_ELF64_STT_SECTION), 1 + i, 0, 0); - } - - assert(dbg_section_count == 0); - - put_section_symbols(sections, &strtbl, &local_symtab, TB_ELF64_STB_LOCAL); - put_section_symbols(sections, &strtbl, &global_symtab, TB_ELF64_STB_GLOBAL); - - FOREACH_N(i, 0, exports.count) { - TB_External* ext = exports.data[i]; - uint32_t name = tb_outstr_nul_UNSAFE(&strtbl, ext->super.name); - ext->super.symbol_id = global_symtab.count / sizeof(TB_Elf64_Sym); - - put_symbol(&global_symtab, name, TB_ELF64_ST_INFO(TB_ELF64_STB_GLOBAL, 0), 0, 0, 0); - } - - uint32_t symtab_name = tb_outstr_nul_UNSAFE(&strtbl, ".symtab"); - TB_Elf64_Shdr strtab = { - .name = tb_outstr_nul_UNSAFE(&strtbl, ".strtab"), - .type = TB_SHT_STRTAB, - .flags = 0, - .addralign = 1, - .size = strtbl.count, - .offset = output_size, - }; - output_size += strtbl.count; - - TB_Elf64_Shdr symtab = { - .name = symtab_name, - .type = TB_SHT_SYMTAB, - .flags = 0, .addralign = 1, - .link = 1, .info = local_symtab.count / sizeof(TB_Elf64_Sym), /* first non-local */ - .size = local_symtab.count + global_symtab.count, - .entsize = sizeof(TB_Elf64_Sym), - .offset = output_size, - }; - output_size += local_symtab.count; - output_size += global_symtab.count; - - header.shoff = output_size; - header.shnum = section_count + 1; - // sections plus the NULL section at the start - output_size += (1 + section_count) * sizeof(TB_Elf64_Shdr); - - //////////////////////////////// - // write output - //////////////////////////////// - size_t write_pos = 0; - TB_ExportChunk* chunk = tb_export_make_chunk(dst_arena, output_size); - uint8_t* restrict output = chunk->data; - - WRITE(&header, sizeof(header)); - - // write section content - dyn_array_for(i, sections) { - write_pos = tb_helper_write_section(m, write_pos, §ions[i], output, sections[i].raw_data_pos); - } - - // write relocation arrays - size_t local_sym_count = local_symtab.count / sizeof(TB_Elf64_Sym); - dyn_array_for(i, sections) if (sections[i].reloc_count > 0) { - assert(sections[i].reloc_pos == write_pos); - TB_Elf64_Rela* rels = (TB_Elf64_Rela*) &output[write_pos]; - DynArray(TB_FunctionOutput*) funcs = sections[i].funcs; - DynArray(TB_Global*) globals = sections[i].globals; - - dyn_array_for(j, funcs) { - TB_FunctionOutput* func_out = funcs[j]; - - size_t source_offset = func_out->code_pos; - for (TB_SymbolPatch* p = func_out->first_patch; p; p = p->next) { - if (p->internal) continue; - - size_t actual_pos = source_offset + p->pos; - size_t symbol_id = p->target->symbol_id; - if (is_nonlocal(p->target)) { - symbol_id += local_sym_count; - } - assert(symbol_id != 0); - - TB_ELF_RelocType type = p->target->tag == TB_SYMBOL_GLOBAL ? TB_ELF_X86_64_PC32 : TB_ELF_X86_64_PLT32; - *rels++ = (TB_Elf64_Rela){ - .offset = actual_pos, - // check when we should prefer R_X86_64_GOTPCREL - .info = TB_ELF64_R_INFO(symbol_id, type), - .addend = -4 - }; - } - } - - write_pos += sections[i].reloc_count * sizeof(TB_Elf64_Rela); - } - - assert(write_pos == strtab.offset); - WRITE(strtbl.data, strtbl.count); - - assert(write_pos == symtab.offset); - WRITE(local_symtab.data, local_symtab.count); - WRITE(global_symtab.data, global_symtab.count); - - // write section header - memset(&output[write_pos], 0, sizeof(TB_Elf64_Shdr)), write_pos += sizeof(TB_Elf64_Shdr); - WRITE(&strtab, sizeof(strtab)); - WRITE(&symtab, sizeof(symtab)); - dyn_array_for(i, sections) { - TB_Elf64_Shdr sec = { - .name = sections[i].name_pos, - .type = TB_SHT_PROGBITS, - .flags = TB_SHF_ALLOC, - .addralign = 16, - .size = sections[i].total_size, - .offset = sections[i].raw_data_pos, - }; - - if (sections[i].flags & TB_MODULE_SECTION_WRITE) { - sec.flags |= TB_SHF_WRITE; - } - - if (sections[i].flags & TB_MODULE_SECTION_EXEC) { - sec.flags |= TB_SHF_EXECINSTR; - } - - WRITE(&sec, sizeof(sec)); - } - - dyn_array_for(i, sections) if (sections[i].reloc_count) { - TB_Elf64_Shdr sec = { - .name = sections[i].name_pos - 5, - .type = TB_SHT_RELA, - .flags = TB_SHF_INFO_LINK, - .addralign = 16, - .info = 3 + i, - .link = 2, - .size = sections[i].reloc_count * sizeof(TB_Elf64_Rela), - .offset = sections[i].reloc_pos, - .entsize = sizeof(TB_Elf64_Rela) - }; - WRITE(&sec, sizeof(sec)); - } - - assert(write_pos == output_size); - return (TB_ExportBuffer){ .total = output_size, .head = chunk, .tail = chunk }; -} diff --git a/vendor/tb/src/objects/lib_parse.h b/vendor/tb/src/objects/lib_parse.h deleted file mode 100644 index 89ac0036..00000000 --- a/vendor/tb/src/objects/lib_parse.h +++ /dev/null @@ -1,143 +0,0 @@ -#include "coff.h" - -typedef struct { - char name[16]; - char date[12]; - - // Microsoft tools don't actually do anything with this - char user_id[6]; - char group_id[6]; - - char mode[8]; - char size[10]; - - uint8_t newline[2]; - uint8_t contents[]; -} COFF_ArchiveMemberHeader; - -// section related crap likes to be sorted in lexical order :p -static int compare_sections(const void* a, const void* b) { - const TB_ObjectSection* sec_a = (const TB_ObjectSection*)a; - const TB_ObjectSection* sec_b = (const TB_ObjectSection*)b; - - size_t shortest_len = sec_a->name.length < sec_b->name.length ? sec_a->name.length : sec_b->name.length; - return memcmp(sec_a->name.data, sec_b->name.data, shortest_len); -} - -static uint16_t read16be(uint8_t* ptr) { - return (ptr[0] << 8u) | (ptr[1]); -} - -static uint32_t read32be(uint8_t* ptr) { - return (ptr[0] << 24u) | (ptr[1] << 16u) | (ptr[2] << 8u) | (ptr[3]); -} - -bool tb_archive_parse(TB_Slice file, TB_ArchiveFileParser* restrict out_parser) { - *out_parser = (TB_ArchiveFileParser){ file }; - - if (memcmp(&file.data[0], "!\n", 8) != 0) { - // TODO(NeGate): maybe we should make a custom error stream... - fprintf(stderr, "TB archive parser: invalid .lib header!\n"); - return false; - } - - size_t file_offset = 8; - COFF_ArchiveMemberHeader* first = (COFF_ArchiveMemberHeader*) &file.data[file_offset]; - if (memcmp(first->name, (char[16]) { "/ " }, 16) != 0) { - fprintf(stderr, "TB archive parser: first archive member name is invalid\n"); - return false; - } - size_t first_content_length = tb__parse_decimal_int(sizeof(first->size), first->size); - - file_offset += sizeof(COFF_ArchiveMemberHeader) + first_content_length; - file_offset = (file_offset + 1u) & ~1u; - - // Process second member - COFF_ArchiveMemberHeader* second = (COFF_ArchiveMemberHeader*) &file.data[file_offset]; - if (memcmp(second->name, (char[16]) { "/ " }, 16) != 0) { - fprintf(stderr, "TB archive parser: second archive member name is invalid\n"); - return false; - } - size_t second_content_length = tb__parse_decimal_int(sizeof(second->size), second->size); - - // Extract number of symbols - if (second_content_length >= 8) { - memcpy(&out_parser->member_count, &second->contents[0], sizeof(uint32_t)); - out_parser->members = (uint32_t*) &second->contents[4]; - - memcpy(&out_parser->symbol_count, &second->contents[4 + out_parser->member_count*sizeof(uint32_t)], sizeof(uint32_t)); - out_parser->symbols = (uint16_t*) &second->contents[8 + out_parser->member_count*sizeof(uint32_t)]; - } - - // Advance - file_offset += sizeof(COFF_ArchiveMemberHeader) + second_content_length; - file_offset = (file_offset + 1u) & ~1u; - - // Process long name member - COFF_ArchiveMemberHeader* longnames = (COFF_ArchiveMemberHeader*) &file.data[file_offset]; - if (memcmp(longnames->name, (char[16]) { "// " }, 16) == 0) { - size_t longname_content_length = tb__parse_decimal_int(sizeof(second->size), second->size); - out_parser->strtbl = (TB_Slice){ longnames->contents, longname_content_length }; - - // Advance - file_offset += sizeof(COFF_ArchiveMemberHeader) + longname_content_length; - file_offset = (file_offset + 1u) & ~1u; - } - - out_parser->pos = file_offset; - return true; -} - -size_t tb_archive_parse_entries(TB_ArchiveFileParser* restrict parser, size_t start, size_t count, TB_ArchiveEntry* out_entry) { - TB_Slice file = parser->file; - TB_Slice strtbl = parser->strtbl; - size_t entry_count = 0; - - FOREACH_N(i, start, count) { - COFF_ArchiveMemberHeader* restrict sym = (COFF_ArchiveMemberHeader*) &file.data[parser->members[i]]; - size_t len = tb__parse_decimal_int(sizeof(sym->size), sym->size); - - TB_Slice sym_name = { (uint8_t*) sym->name, strchr(sym->name, ' ') - sym->name }; - if (sym_name.data[0] == '/') { - // name is actually just an index into the long names table - size_t num = tb__parse_decimal_int(sym_name.length - 1, (char*)sym_name.data + 1); - sym_name = (TB_Slice){ &strtbl.data[num], strlen((const char*) &strtbl.data[num]) }; - } - - uint32_t short_form_header = *(uint32_t*)sym->contents; - if (short_form_header == 0xFFFF0000) { - COFF_ImportHeader* import = (COFF_ImportHeader*) sym->contents; - - const char* imported_symbol = (const char*) &sym->contents[sizeof(COFF_ImportHeader)]; - const char* dll_path = (const char*) &sym->contents[sizeof(COFF_ImportHeader) + strlen(imported_symbol) + 1]; - - TB_Slice import_name = { 0 }; - if (import->name_type == 3) { - // for odd reasons windows has some symbols with @ and underscores (C++ amirite) - // and we have to strip them out. - const char* leading = imported_symbol; - const char* at = strchr(imported_symbol, '@'); - if (at == NULL) at = imported_symbol + strlen(imported_symbol); - - for (const char* s = imported_symbol; s != at; s++) { - if (*s == '_') leading = s+1; - } - - import_name.length = at - leading; - import_name.data = (const uint8_t*) leading; - } else { - import_name.length = strlen(imported_symbol); - import_name.data = (const uint8_t*) imported_symbol; - } - - // printf("%s : %s : %d\n", dll_path, imported_symbol, import->name_type); - out_entry[entry_count++] = (TB_ArchiveEntry){ { (const uint8_t*) dll_path, strlen(dll_path) }, .import_name = import_name, .ordinal = import->ordinal_hint }; - } else { - out_entry[entry_count++] = (TB_ArchiveEntry){ sym_name, .content = { sym->contents, len } }; - } - - skip:; - } - - return entry_count; -} diff --git a/vendor/tb/src/objects/macho.c b/vendor/tb/src/objects/macho.c deleted file mode 100644 index 01f20154..00000000 --- a/vendor/tb/src/objects/macho.c +++ /dev/null @@ -1,117 +0,0 @@ -#include "macho.h" - -#define WRITE(data, size) (memcpy(&output[write_pos], data, size), write_pos += (size)) -TB_ExportBuffer tb_macho_write_output(TB_Module* m, TB_Arena* dst_arena, const IDebugFormat* dbg) { - const ICodeGen* code_gen = tb__find_code_generator(m); - - //TB_TemporaryStorage* tls = tb_tls_allocate(); - TB_Emitter string_table = { 0 }; - - CUIK_TIMED_BLOCK("layout section") { - tb_module_layout_sections(m); - } - - // segments - size_t text_size = 0; - MO_Section64 sections[] = { - { - .sectname = { "__text" }, - .segname = { "__TEXT" }, - .align = 2, - .size = text_size, - .flags = S_ATTR_PURE_INSTRUCTIONS | S_ATTR_SOME_INSTRUCTIONS - } - }; - enum { NUMBER_OF_SECTIONS = COUNTOF(sections) }; - - size_t output_size = sizeof(MO_Header64) - + sizeof(MO_SymtabCmd) - + sizeof(MO_SegmentCmd64) - + (sizeof(MO_Section64) * NUMBER_OF_SECTIONS); - - MO_SegmentCmd64 segment_cmd = { - .header = { - .cmd = LC_SEGMENT_64, - .cmdsize = sizeof(MO_SegmentCmd64) + sizeof(MO_Section64)*NUMBER_OF_SECTIONS, - }, - .segname = { "__TEXT" }, - .nsects = NUMBER_OF_SECTIONS, - .vmsize = text_size, - .fileoff = output_size, - .filesize = text_size - }; - - // layout section data - sections[0].offset = output_size; - output_size += text_size; - - // generate symbol table - MO_SymtabCmd symtab_cmd = { - .header = { - .cmd = LC_SYMTAB, - .cmdsize = sizeof(MO_SymtabCmd) - } - }; - - // count symbols - { - /*symtab_cmd.nsyms += m->compiled_function_count; - - FOREACH_N(i, 0, m->max_threads) { - size_t external_len = pool_popcount(m->thread_info[i].externals); - symtab_cmd.nsyms += external_len ? external_len - 1 : 0; - } - - FOREACH_N(i, 0, m->max_threads) { - symtab_cmd.nsyms += pool_popcount(m->thread_info[i].globals); - }*/ - } - - MO_Header64 header = { - .magic = MH_MAGIC_64, - .filetype = MH_OBJECT, - .ncmds = 2, - .sizeofcmds = symtab_cmd.header.cmdsize + segment_cmd.header.cmdsize, - .flags = 0x2000 - }; - - // fill in CPU type and subtype based on target - switch (m->target_arch) { - case TB_ARCH_X86_64: header.cputype = CPU_TYPE_X86_64; header.cpusubtype = 3; break; - case TB_ARCH_AARCH64: header.cputype = CPU_TYPE_AARCH64; header.cpusubtype = 0; break; - default: tb_todo(); - } - - //size_t load_cmds_start = sizeof(MO_Header64); - // fprintf(stderr, "TB warning: Mach-O output isn't ready yet :p sorry\n"); - - // Allocate memory now - size_t write_pos = 0; - TB_ExportChunk* chunk = tb_export_make_chunk(dst_arena, output_size); - uint8_t* restrict output = chunk->data; - - // General layout is: - // HEADER - // LOAD COMMAND - // LOAD COMMAND - // ... - // SEGMENT - // SEGMENT - // ... - WRITE(&header, sizeof(header)); - WRITE(&symtab_cmd, sizeof(symtab_cmd)); - WRITE(&segment_cmd, sizeof(segment_cmd)); - WRITE(§ions, sizeof(MO_Section64) * NUMBER_OF_SECTIONS); - - // write_pos = tb_helper_write_section(m, write_pos, &m->text, output, sections[0].offset); - - // emit section contents - FOREACH_N(i, 0, NUMBER_OF_SECTIONS) { - - } - - // fwrite(string_table.data, string_table.count, 1, f); - - tb_platform_heap_free(string_table.data); - return (TB_ExportBuffer){ .total = output_size, .head = chunk, .tail = chunk }; -} diff --git a/vendor/tb/src/objects/macho.h b/vendor/tb/src/objects/macho.h deleted file mode 100644 index 08015689..00000000 --- a/vendor/tb/src/objects/macho.h +++ /dev/null @@ -1,97 +0,0 @@ -// imma shorten Mach-O into MO -#include "../tb_internal.h" - -typedef struct { - uint32_t magic; - uint32_t cputype; - uint32_t cpusubtype; - uint32_t filetype; - uint32_t ncmds; - uint32_t sizeofcmds; - uint32_t flags; - uint32_t reserved; -} MO_Header64; - -typedef struct { - uint32_t cmd; - uint32_t cmdsize; -} MO_LoadCmd; - -typedef struct { - MO_LoadCmd header; - uint32_t symoff; - uint32_t nsyms; - uint32_t stroff; - uint32_t strsize; -} MO_SymtabCmd; - -typedef struct { - MO_LoadCmd header; - char segname[16]; - uint64_t vmaddr; - uint64_t vmsize; - uint64_t fileoff; - uint64_t filesize; - uint32_t maxprot; - uint32_t initprot; - uint32_t nsects; - uint32_t flags; -} MO_SegmentCmd64; - -typedef struct { - char sectname[16]; - char segname[16]; - uint64_t addr; - uint64_t size; - uint32_t offset; // file offset - uint32_t align; - uint32_t reloff; - uint32_t nreloc; - uint32_t flags; - uint32_t reserved1; - uint32_t reserved2; - uint32_t reserved3; -} MO_Section64; - -// extracted from the header -#define MH_MAGIC_64 0xFEEDFACF -#define MH_CIGAM_64 0xCFFAEDFE - -#define CPU_TYPE_X86_64 0x1000007 -#define CPU_TYPE_AARCH64 0x100000C - -#define MH_OBJECT 1 -#define MH_EXECUTE 2 -#define MH_FVMLIB 3 -#define MH_CORE 4 -#define MH_PRELOAD 5 -#define MH_DYLIB 6 -#define MH_DYLINKER 7 -#define MH_BUNDLE 8 - -/* Constants for the cmd field of all load commands, the type */ -#define LC_SEGMENT 0x1 -#define LC_SYMTAB 0x2 -#define LC_SYMSEG 0x3 -#define LC_THREAD 0x4 -#define LC_UNIXTHREAD 0x5 -#define LC_LOADFVMLIB 0x6 -#define LC_IDFVMLIB 0x7 -#define LC_IDENT 0x8 -#define LC_FVMFILE 0x9 -#define LC_PREPAGE 0xa -#define LC_DYSYMTAB 0xb -#define LC_LOAD_DYLIB 0xc -#define LC_ID_DYLIB 0xd -#define LC_LOAD_DYLINKER 0xe -#define LC_ID_DYLINKER 0xf -#define LC_PREBOUND_DYLIB 0x10 -#define LC_SEGMENT_64 0x19 - -#define N_EXT 0x01 -#define N_SECT 0x0E - -#define NO_SECT 0 - -#define S_ATTR_PURE_INSTRUCTIONS 0x80000000 -#define S_ATTR_SOME_INSTRUCTIONS 0x00000400 diff --git a/vendor/tb/src/objects/wasm_obj.c b/vendor/tb/src/objects/wasm_obj.c deleted file mode 100644 index 00a95e00..00000000 --- a/vendor/tb/src/objects/wasm_obj.c +++ /dev/null @@ -1,157 +0,0 @@ -#include "coff.h" - -// uleb128 encode -static void emit_uint(TB_Emitter* emit, uint64_t x) { - do { - uint32_t lo = x & 0x7F; - x >>= 7; - if (x) { - lo |= 0x80; - } - tb_out1b(emit, lo); - } while (x); -} - -static uint8_t* emit_uint2(uint8_t* p, uint64_t x) { - do { - uint32_t lo = x & 0x7F; - x >>= 7; - if (x) { - lo |= 0x80; - } - *p++ = lo; - } while (x); - return p; -} - -static int len_uint(uint64_t x) { - int c = 0; - do { - uint32_t lo = x & 0x7F; - x >>= 7; - if (x) { - lo |= 0x80; - } - c += 1; - } while (x); - return c; -} - -static uint8_t get_wasm_type(TB_DataType dt) { - switch (dt.type) { - case TB_INT: { - if (dt.data <= 8) return 0x7F; - if (dt.data <= 16) return 0x7F; - if (dt.data <= 32) return 0x7F; - if (dt.data <= 64) return 0x7E; - break; - } - case TB_FLOAT: { - if (dt.data == TB_FLT_32) return 0x7D; - if (dt.data == TB_FLT_64) return 0x7C; - break; - } - case TB_PTR: return 0x7F; - } - - assert(0 && "TODO"); - return 0; -} - -static TB_ExportChunk* emitter_to_chunk(TB_Arena* dst_arena, TB_Emitter* e) { - TB_ExportChunk* stuff = tb_export_make_chunk(dst_arena, e->count); - memcpy(stuff->data, e->data, e->count); - tb_platform_heap_free(e->data); - return stuff; -} - -TB_ExportBuffer tb_wasm_write_output(TB_Module* m, TB_Arena* dst_arena, const IDebugFormat* dbg) { - TB_Arena* arena = get_temporary_arena(m); - - ExportList exports; - CUIK_TIMED_BLOCK("layout section") { - exports = tb_module_layout_sections(m); - } - - // Construct type section - size_t func_type_count = 0; - TB_Emitter func_types = { 0 }; - - DynArray(TB_ModuleSection) sections = m->sections; - dyn_array_for(i, sections) { - DynArray(TB_FunctionOutput*) funcs = sections[i].funcs; - dyn_array_for(j, funcs) { - funcs[j]->wasm_type = func_type_count++; - TB_FunctionPrototype* proto = funcs[j]->parent->prototype; - - tb_out1b(&func_types, 0x60); - emit_uint(&func_types, proto->param_count); - FOREACH_N(k, 0, proto->param_count) { - tb_out1b(&func_types, get_wasm_type(proto->params[k].dt)); - } - emit_uint(&func_types, proto->return_count); - FOREACH_N(k, 0, proto->return_count) { - tb_out1b(&func_types, get_wasm_type(proto->params[proto->param_count + k].dt)); - } - } - } - - size_t func_section_size = 0; - size_t func_count = 0; - dyn_array_for(i, sections) { - DynArray(TB_FunctionOutput*) funcs = sections[i].funcs; - dyn_array_for(j, funcs) { - func_section_size += len_uint(funcs[j]->wasm_type); - func_count += 1; - } - } - func_section_size += len_uint(func_count); - - // Module header - TB_Emitter emit = { 0 }; - tb_out4b(&emit, 0x6D736100); // MAGIC \0asm - tb_out4b(&emit, 0x1); // VERSION 1 - - // Type section - tb_out1b(&emit, 0x01); - emit_uint(&emit, func_types.count + len_uint(func_type_count)); - emit_uint(&emit, func_type_count); - - tb_out_reserve(&emit, func_types.count + len_uint(func_type_count)); - tb_outs_UNSAFE(&emit, func_types.count, (const uint8_t*) func_types.data); - - // Function section - tb_out1b(&emit, 0x03); - emit_uint(&emit, func_section_size); - emit_uint(&emit, func_count); - dyn_array_for(i, sections) { - DynArray(TB_FunctionOutput*) funcs = sections[i].funcs; - dyn_array_for(j, funcs) { - emit_uint(&emit, funcs[j]->wasm_type); - } - } - - TB_ExportBuffer buffer = { 0 }; - tb_export_append_chunk(&buffer, emitter_to_chunk(dst_arena, &emit)); - - #if 1 - dyn_array_for(i, sections) { - if ((sections[i].flags & TB_MODULE_SECTION_EXEC) == 0) { - continue; - } - - size_t body_size = len_uint(dyn_array_length(sections[i].funcs)) + sections[i].total_size; - TB_ExportChunk* sec = tb_export_make_chunk(dst_arena, 1 + body_size + len_uint(body_size)); - - uint8_t* p = sec->data; - *p++ = 0x0A; - p = emit_uint2(p, body_size); - p = emit_uint2(p, dyn_array_length(sections[i].funcs)); - tb_helper_write_section(m, 0, §ions[i], p, 0); - - tb_export_append_chunk(&buffer, sec); - } - #endif - return buffer; -} - diff --git a/vendor/tb/src/opt/branches.h b/vendor/tb/src/opt/branches.h deleted file mode 100644 index 2c8ebcf9..00000000 --- a/vendor/tb/src/opt/branches.h +++ /dev/null @@ -1,550 +0,0 @@ - -static TB_Node* ideal_region(TB_Passes* restrict p, TB_Function* f, TB_Node* n) { - TB_NodeRegion* r = TB_NODE_GET_EXTRA(n); - - // if a region is dead, start a violent death chain - if (n->input_count == 0) { - return NULL; - } else if (n->input_count == 1) { - // single entry regions are useless... - // check for any phi nodes, because we're single entry they're all degens - User* use = n->users; - while (use != NULL) { - User* next = use->next; - if (use->n->type == TB_PHI) { - assert(use->n->input_count == 2); - subsume_node(f, use->n, use->n->inputs[1]); - } - use = next; - } - - // we might want this as an identity - return n->inputs[0]; - } else { - bool changes = false; - - size_t i = 0, extra_edges = 0; - while (i < n->input_count) { - Lattice* ty = lattice_universe_get(p, n->inputs[i]); - if (n->inputs[i]->type == TB_DEAD || ty == &XCTRL_IN_THE_SKY) { - remove_input(f, n, i); - - // update PHIs - FOR_USERS(use, n) { - if (use->n->type == TB_PHI && use->slot == 0) { - remove_input(f, use->n, i + 1); - } - } - } else if (cfg_is_region(n->inputs[i])) { - #if 1 - // pure regions can be collapsed into direct edges - if (n->inputs[i]->users->next == NULL && n->inputs[i]->input_count > 0) { - assert(n->inputs[i]->users->n == n); - changes = true; - - TB_Node* pred = n->inputs[i]; - { - size_t old_count = n->input_count; - size_t new_count = old_count + (pred->input_count - 1); - - // convert pred-of-pred into direct pred - set_input(f, n, pred->inputs[0], i); - - // append rest to the end (order really doesn't matter) - // - // NOTE(NeGate): we might waste quite a bit of space because of the arena - // alloc and realloc - TB_Node** new_inputs = alloc_from_node_arena(f, new_count * sizeof(TB_Node*)); - memcpy(new_inputs, n->inputs, old_count * sizeof(TB_Node*)); - n->inputs = new_inputs; - n->input_count = new_count; - n->input_cap = new_count; - - FOREACH_N(j, 0, pred->input_count - 1) { - new_inputs[old_count + j] = pred->inputs[j + 1]; - add_user(f, n, pred->inputs[j + 1], old_count + j, NULL); - } - } - - // update PHIs - FOR_USERS(use, n) { - if (use->n->type == TB_PHI && use->slot == 0) { - // we don't replace the initial, just the rest - TB_Node* phi = use->n; - TB_Node* phi_val = phi->inputs[i + 1]; - - // append more phi vals... lovely allocs - size_t phi_ins = phi->input_count; - size_t new_phi_ins = phi_ins + (pred->input_count - 1); - - TB_Node** new_inputs = alloc_from_node_arena(f, new_phi_ins * sizeof(TB_Node*)); - memcpy(new_inputs, phi->inputs, phi_ins * sizeof(TB_Node*)); - phi->inputs = new_inputs; - phi->input_count = new_phi_ins; - phi->input_cap = new_phi_ins; - - FOREACH_N(j, 0, pred->input_count - 1) { - new_inputs[phi_ins + j] = phi_val; - add_user(f, phi, phi_val, phi_ins + j, NULL); - } - } - } - - extra_edges += 1; - continue; - } - #endif - } - - i += 1; - } - - if (changes) { - return n->input_count == 1 ? n->inputs[0] : n; - } - } - - return NULL; -} - -static TB_Node* ideal_phi(TB_Passes* restrict opt, TB_Function* f, TB_Node* n) { - // degenerate PHI, poison it - if (n->input_count == 1) { - log_warn("%s: ir: generated poison due to PHI with no edges", f->super.name); - return make_poison(f, n->dt); - } - - // if branch, both paths are empty => select(cond, t, f) - // - // TODO(NeGate): we can make this diamond trick work for bigger - // branches, we should support a lookup instruction similar to - // "switch" logic for data. - TB_DataType dt = n->dt; - TB_Node* region = n->inputs[0]; - if (n->dt.type != TB_MEMORY) { - if (region->input_count == 2) { - // for now we'll leave multi-phi scenarios alone, we need - // to come up with a cost-model around this stuff. - FOR_USERS(use, region) { - if (use->n->type == TB_PHI) { - if (use->n != n) return NULL; - } - } - - // guarentee paths are effectless (there's only one data phi and no control nodes) - // - // If - // / \ - // CProjT CProjF Region[0][0] == Region[1][0] - // \ / - // Region - // - TB_Node* left = region->inputs[0]; - TB_Node* right = region->inputs[1]; - if (left->type == TB_PROJ && right->type == TB_PROJ && - left->inputs[0]->type == TB_BRANCH && left->inputs[0] == right->inputs[0]) { - TB_Node* branch = left->inputs[0]; - TB_NodeBranch* header_br = TB_NODE_GET_EXTRA(branch); - - if (header_br->succ_count == 2) { - assert(branch->input_count == 2); - - TB_Node *values[2]; - FOR_USERS(u, branch) { - TB_Node* proj = u->n; - if (proj->type == TB_PROJ) { - int index = TB_NODE_GET_EXTRA_T(proj, TB_NodeProj)->index; - // the projection needs to exclusively refer to the region, - // if not we can't elide those effects here. - if (proj->users->next != NULL || proj->users->n != region) { - return NULL; - } - - int phi_i = proj->users->slot; - assert(phi_i + 1 < n->input_count); - values[index] = n->inputs[1 + phi_i]; - } - } - - uint64_t falsey = TB_NODE_GET_EXTRA_T(branch, TB_NodeBranch)->keys[0].key; - TB_Node* cond = branch->inputs[1]; - - // TODO(NeGate): handle non-zero falseys - if (falsey == 0) { - // header -> merge - { - TB_Node* parent = branch->inputs[0]; - tb_pass_kill_node(f, branch); - tb_pass_kill_node(f, left); - tb_pass_kill_node(f, right); - - // attach the header and merge to each other - tb_pass_mark(opt, parent); - tb_pass_mark_users(opt, region); - subsume_node(f, region, parent); - } - - TB_Node* selector = tb_alloc_node(f, TB_SELECT, dt, 4, 0); - set_input(f, selector, cond, 1); - set_input(f, selector, values[0], 2); - set_input(f, selector, values[1], 3); - return selector; - } - } - } - } - - if (region->input_count > 2 && n->dt.type == TB_INT) { - if (region->inputs[0]->type != TB_PROJ || region->inputs[0]->inputs[0]->type != TB_BRANCH) { - return NULL; - } - - // try to make a multi-way lookup: - // - // Branch - // / | \ - // ... ... ... each of these is a CProj - // \ | / - // Region - // \ - // \ ... ... each of these is a trivial value (int consts only for now) - // \ | / - // Phi - TB_Node* parent = region->inputs[0]->inputs[0]; - TB_NodeBranch* br = TB_NODE_GET_EXTRA(parent); - if (parent->type != TB_BRANCH || br->succ_count != n->input_count - 1) { - return NULL; - } - - // verify we have a really clean looking diamond shape - FOREACH_N(i, 0, n->input_count - 1) { - if (region->inputs[i]->type != TB_PROJ || region->inputs[i]->inputs[0] != parent) return NULL; - if (n->inputs[1 + i]->type != TB_INTEGER_CONST) return NULL; - } - - // convert to lookup node - TB_Node* lookup = tb_alloc_node(f, TB_LOOKUP, n->dt, 2, sizeof(TB_NodeLookup) + (br->succ_count * sizeof(TB_LookupEntry))); - set_input(f, lookup, parent->inputs[1], 1); - - TB_NodeLookup* l = TB_NODE_GET_EXTRA(lookup); - l->entry_count = br->succ_count; - FOREACH_N(i, 0, n->input_count - 1) { - TB_Node* k = region->inputs[i]; - int index = TB_NODE_GET_EXTRA_T(k, TB_NodeProj)->index; - assert(index < br->succ_count); - - if (index == 0) { - l->entries[index].key = 0; // default value, doesn't matter - } else { - l->entries[index].key = br->keys[index - 1].key; - } - - TB_NodeInt* v = TB_NODE_GET_EXTRA(n->inputs[1 + i]); - l->entries[index].val = v->value; - } - - // kill branch, we don't really need it anymore - TB_Node* before = parent->inputs[0]; - tb_pass_kill_node(f, parent); - subsume_node(f, region, before); - - return lookup; - } - } - - return NULL; -} - -static TB_Node* ideal_branch(TB_Passes* restrict opt, TB_Function* f, TB_Node* n) { - TB_NodeBranch* br = TB_NODE_GET_EXTRA(n); - - if (br->succ_count == 2) { - if (n->input_count == 2 && br->keys[0].key == 0) { - TB_Node* cmp_node = n->inputs[1]; - TB_NodeTypeEnum cmp_type = cmp_node->type; - - // empty BB, just does if branch but the condition is effect-less - // if (a && b) A else B => if (a ? b : 0) A else B - // - // TODO(NeGate): implement form which works on an arbitrary falsey - if (n->inputs[0]->type == TB_PROJ && n->inputs[0]->inputs[0]->type == TB_BRANCH && is_empty_bb(opt, n)) { - uint64_t falsey = br->keys[0].key; - TB_Node* pred_branch = n->inputs[0]->inputs[0]; - - int index = TB_NODE_GET_EXTRA_T(n->inputs[0], TB_NodeProj)->index; - - // needs one pred - uint64_t pred_falsey; - if (is_if_branch(pred_branch, &pred_falsey) && pred_falsey == 0) { - TB_NodeBranch* pred_br_info = TB_NODE_GET_EXTRA(pred_branch); - - // check our parent's aux path - User* other_proj = proj_with_index(pred_branch, 1 - index); - TB_Node* shared_edge = cfg_next_bb_after_cproj(other_proj->n); - - // check our aux path - User* other_proj2 = proj_with_index(n, 1 - index); - TB_Node* shared_edge2 = cfg_next_bb_after_cproj(other_proj2->n); - - // if they're the same then we've got a shortcircuit eval setup - if (shared_edge == shared_edge2) { - assert(cfg_is_region(shared_edge)); - int shared_i = other_proj->n->users->slot; - int shared_i2 = other_proj2->n->users->slot; - - bool match = true; - FOR_USERS(phis, shared_edge) if (phis->n->type == TB_PHI) { - if (phis->n->inputs[1+shared_i] != phis->n->inputs[1+shared_i2]) { - match = false; - break; - } - } - - if (match) { - // remove pred from shared edge - remove_input(f, shared_edge, shared_i); - FOR_USERS(use, shared_edge) { - if (use->n->type == TB_PHI && use->slot == 0) { - remove_input(f, use->n, shared_i + 1); - tb_pass_mark(opt, use->n); - } - } - - TB_Node* before = pred_branch->inputs[0]; - TB_Node* cmp = pred_branch->inputs[1]; - - // remove first branch - tb_pass_kill_node(f, pred_branch); - set_input(f, n, before, 0); - - // we wanna normalize into a comparison (not a boolean -> boolean) - if (!(cmp->dt.type == TB_INT && cmp->dt.data == 1)) { - assert(cmp->dt.type != TB_FLOAT && "TODO"); - TB_Node* imm = make_int_node(f, opt, cmp->dt, pred_falsey); - - TB_Node* new_node = tb_alloc_node(f, TB_CMP_NE, TB_TYPE_BOOL, 3, sizeof(TB_NodeCompare)); - set_input(f, new_node, cmp, 1); - set_input(f, new_node, imm, 2); - TB_NODE_SET_EXTRA(new_node, TB_NodeCompare, .cmp_dt = cmp->dt); - - tb_pass_mark(opt, new_node); - cmp = new_node; - } - - // construct branchless merge - TB_Node* false_node = make_int_node(f, opt, n->inputs[1]->dt, 0); - - // a ? b : 0 - TB_Node* selector = tb_alloc_node(f, TB_SELECT, n->inputs[1]->dt, 4, 0); - set_input(f, selector, cmp, 1); - set_input(f, selector, n->inputs[1], 2); - set_input(f, selector, false_node, 3); - - set_input(f, n, selector, 1); - tb_pass_mark(opt, selector); - return n; - } - } - } - } - - // br ((y <= x)) => br (x < y) flipped conditions - if (cmp_type == TB_CMP_SLE || cmp_type == TB_CMP_ULE) { - TB_Node* new_cmp = tb_alloc_node(f, cmp_type == TB_CMP_SLE ? TB_CMP_SLT : TB_CMP_ULT, TB_TYPE_BOOL, 3, sizeof(TB_NodeCompare)); - set_input(f, new_cmp, cmp_node->inputs[2], 1); - set_input(f, new_cmp, cmp_node->inputs[1], 2); - TB_NODE_SET_EXTRA(new_cmp, TB_NodeCompare, .cmp_dt = TB_NODE_GET_EXTRA_T(cmp_node, TB_NodeCompare)->cmp_dt); - - // flip - FOR_USERS(u, n) { - TB_NodeProj* p = TB_NODE_GET_EXTRA(u->n); - p->index = !p->index; - } - - set_input(f, n, new_cmp, 1); - tb_pass_mark(opt, new_cmp); - return n; - } - - // br ((x != y) != 0) => br (x != y) - if ((cmp_type == TB_CMP_NE || cmp_type == TB_CMP_EQ) && cmp_node->inputs[2]->type == TB_INTEGER_CONST) { - uint64_t imm = TB_NODE_GET_EXTRA_T(cmp_node->inputs[2], TB_NodeInt)->value; - set_input(f, n, cmp_node->inputs[1], 1); - br->keys[0].key = imm; - - // flip successors - if (cmp_type == TB_CMP_EQ) { - br->keys[0].taken = br->total_hits - br->keys[0].taken; - FOR_USERS(u, n) { - TB_NodeProj* p = TB_NODE_GET_EXTRA(u->n); - p->index = !p->index; - } - } - - return n; - } - } - } - - return NULL; -} - -static Lattice* value_call(TB_Passes* restrict opt, TB_Node* n) { - TB_NodeCall* c = TB_NODE_GET_EXTRA(n); - - size_t size = sizeof(Lattice) + c->proj_count*sizeof(Lattice*); - Lattice* l = tb_arena_alloc(tmp_arena, size); - *l = (Lattice){ LATTICE_TUPLE, ._tuple = { c->proj_count } }; - - FOREACH_N(i, 1, c->proj_count) { - l->elems[i] = &BOT_IN_THE_SKY; - } - - FOR_USERS(u, n) { - if (u->n->type == TB_PROJ) { - int index = TB_NODE_GET_EXTRA_T(u->n, TB_NodeProj)->index; - if (index > 0) { - l->elems[index] = lattice_from_dt(opt, u->n->dt); - } - } - } - - // control just flows through - l->elems[0] = lattice_universe_get(opt, n->inputs[0]); - - Lattice* k = nl_hashset_put2(&opt->type_interner, l, lattice_hash, lattice_cmp); - if (k) { - tb_arena_free(tmp_arena, l, size); - return k; - } else { - return l; - } -} - -static Lattice* value_branch(TB_Passes* restrict opt, TB_Node* n) { - Lattice* before = lattice_universe_get(opt, n->inputs[0]); - if (before == &TOP_IN_THE_SKY) { - return &TOP_IN_THE_SKY; - } - - TB_NodeBranch* br = TB_NODE_GET_EXTRA(n); - - // constant fold branch - assert(n->input_count == 2); - Lattice* key = lattice_universe_get(opt, n->inputs[1]); - if (key == &TOP_IN_THE_SKY) { - return &TOP_IN_THE_SKY; - } - - ptrdiff_t taken = -1; - if (key->tag == LATTICE_INT && key->_int.min == key->_int.max) { - int64_t key_const = key->_int.max; - taken = 0; - - FOREACH_N(i, 0, br->succ_count - 1) { - int64_t case_key = br->keys[i].key; - if (key_const == case_key) { - taken = i + 1; - break; - } - } - } else if (br->succ_count == 2) { - TB_BranchKey* primary_keys = br->keys; - - // check for redundant conditions in the doms. - FOR_USERS(u, n->inputs[1]) { - if (u->n->type != TB_BRANCH || u->slot != 1 || u->n == n) { - continue; - } - - TB_Node* end = u->n; - TB_BranchKey* keys = TB_NODE_GET_EXTRA_T(end, TB_NodeBranch)->keys; - if (TB_NODE_GET_EXTRA_T(end, TB_NodeBranch)->succ_count != 2 || keys[0].key != primary_keys[0].key) { - continue; - } - - FOR_USERS(succ_user, end) { - assert(succ_user->n->type == TB_PROJ); - int index = TB_NODE_GET_EXTRA_T(succ_user->n, TB_NodeProj)->index; - TB_Node* succ = cfg_next_bb_after_cproj(succ_user->n); - - // we must be dominating for this to work - if (!fast_dommy(succ, n)) { - continue; - } - - taken = index; - goto match; - } - } - } - - // construct tuple type - match:; - size_t size = sizeof(Lattice) + br->succ_count*sizeof(Lattice*); - Lattice* l = tb_arena_alloc(tmp_arena, size); - *l = (Lattice){ LATTICE_TUPLE, ._tuple = { br->succ_count } }; - FOREACH_N(i, 0, br->succ_count) { - l->elems[i] = taken < 0 || i == taken ? &CTRL_IN_THE_SKY : &XCTRL_IN_THE_SKY; - } - - Lattice* k = nl_hashset_put2(&opt->type_interner, l, lattice_hash, lattice_cmp); - if (k) { - tb_arena_free(tmp_arena, l, size); - return k; - } else { - return l; - } -} - -static TB_Node* identity_safepoint(TB_Passes* restrict p, TB_Function* f, TB_Node* n) { - if (n->inputs[0]->type == TB_SAFEPOINT_POLL) { - // (safepoint (safepoint X)) => (safepoint X) - return n->inputs[0]; - } else { - return n; - } -} - -static TB_Node* identity_region(TB_Passes* restrict p, TB_Function* f, TB_Node* n) { - // fold out diamond shaped patterns - TB_Node* same = n->inputs[0]; - if (same->type == TB_PROJ && same->inputs[0]->type == TB_BRANCH) { - same = same->inputs[0]; - - // if it has phis... quit - FOR_USERS(u, n) { - if (u->n->type == TB_PHI) { - return n; - } - } - - FOREACH_N(i, 1, n->input_count) { - if (n->inputs[i]->type != TB_PROJ || n->inputs[i]->inputs[0] != same) { - return n; - } - } - - TB_Node* before = same->inputs[0]; - tb_pass_kill_node(f, same); - return before; - } - - return n; -} - -static TB_Node* identity_phi(TB_Passes* restrict p, TB_Function* f, TB_Node* n) { - TB_Node* same = NULL; - FOREACH_N(i, 1, n->input_count) { - if (n->inputs[i] == n) continue; - if (same && same != n->inputs[i]) return n; - same = n->inputs[i]; - } - - assert(same); - if (p) { - tb_pass_mark_users(p, n->inputs[0]); - } - - return same; -} diff --git a/vendor/tb/src/opt/cfg.h b/vendor/tb/src/opt/cfg.h deleted file mode 100644 index d68e0c05..00000000 --- a/vendor/tb/src/opt/cfg.h +++ /dev/null @@ -1,288 +0,0 @@ - -typedef struct Block { - struct Block* parent; - TB_ArenaSavepoint sp; - TB_Node* bb; - TB_Node* end; - int succ_i; - TB_Node* succ[]; -} Block; - -void tb_free_cfg(TB_CFG* cfg) { - nl_map_for(i, cfg->node_to_block) { - nl_hashset_free(cfg->node_to_block[i].v.items); - } - nl_map_free(cfg->node_to_block); -} - -TB_CFG tb_compute_rpo(TB_Function* f, TB_Passes* p) { - return tb_compute_rpo2(f, &p->worklist); -} - -// walks until the terminator or other critical edge -static TB_Node* end_of_bb(TB_Node* n) { - while (!cfg_is_terminator(n)) { - TB_Node* next = cfg_next_control0(n); - if (next == NULL || cfg_is_region(next)) { - break; - } - n = next; - } - - return n; -} - -static Block* create_block(TB_Arena* arena, TB_Node* bb) { - TB_ArenaSavepoint sp = tb_arena_save(tmp_arena); - - TB_Node* end = end_of_bb(bb); - size_t succ_count = end->type == TB_BRANCH ? TB_NODE_GET_EXTRA_T(end, TB_NodeBranch)->succ_count : 1; - if (cfg_is_endpoint(end)) { - succ_count = 0; - } - - Block* top = tb_arena_alloc(arena, sizeof(Block) + succ_count*sizeof(TB_Node*)); - *top = (Block){ - .sp = sp, - .bb = bb, - .end = end, - .succ_i = succ_count, - }; - - if (end->type == TB_BRANCH) { - FOR_USERS(u, end) { - if (u->n->type == TB_PROJ) { - int index = TB_NODE_GET_EXTRA_T(u->n, TB_NodeProj)->index; - top->succ[index] = cfg_next_bb_after_cproj(u->n); - } - } - - TB_NodeBranch* br = TB_NODE_GET_EXTRA(end); - if (br->succ_count == 2) { - // make the hot path the fallthrough - float taken = (float)br->keys[0].taken / (float)br->total_hits; - if (taken < 0.5f) { - SWAP(TB_Node*, top->succ[0], top->succ[1]); - } - } - } else if (!cfg_is_endpoint(end)) { - top->succ[0] = cfg_next_user(end)->n; - } - - return top; -} - -TB_CFG tb_compute_rpo2(TB_Function* f, Worklist* ws) { - cuikperf_region_start("RPO", NULL); - assert(dyn_array_length(ws->items) == 0); - - TB_CFG cfg = { 0 }; - nl_map_create(cfg.node_to_block, (f->node_count / 16) + 4); - - // push initial block - Block* top = create_block(tmp_arena, f->params[0]); - worklist_test_n_set(ws, f->params[0]); - - while (top != NULL) { - cuikperf_region_start("rpo_iter", NULL); - if (top->succ_i > 0) { - // push next unvisited succ - TB_Node* succ = top->succ[--top->succ_i]; - if (!worklist_test_n_set(ws, succ)) { - Block* new_top = create_block(tmp_arena, succ); - new_top->parent = top; - top = new_top; - } - } else { - Block b = *top; - - TB_BasicBlock bb = { .start = b.bb, .end = b.end, .dom_depth = -1 }; - bb.freq = 1.0; - - dyn_array_put(ws->items, b.bb); - nl_map_put(cfg.node_to_block, b.bb, bb); - cfg.block_count += 1; - - tb_arena_restore(tmp_arena, top->sp); - top = b.parent; // off to wherever we left off - } - cuikperf_region_end(); - } - - // just reverse the items here... im too lazy to flip all my uses - CUIK_TIMED_BLOCK("reversing") { - size_t last = cfg.block_count - 1; - FOREACH_N(i, 0, cfg.block_count / 2) { - SWAP(TB_Node*, ws->items[i], ws->items[last - i]); - } - } - - CUIK_TIMED_BLOCK("dom depths") { - FOREACH_N(i, 0, cfg.block_count) { - TB_BasicBlock* bb = &nl_map_get_checked(cfg.node_to_block, ws->items[i]); - if (i == 0) { - bb->dom_depth = 0; - } - bb->id = i; - } - } - - cuikperf_region_end(); - return cfg; -} - -static int find_traversal_index(TB_CFG* cfg, TB_Node* n) { - return nl_map_get_checked(cfg->node_to_block, n).id; -} - -static int try_find_traversal_index(TB_CFG* cfg, TB_Node* n) { - ptrdiff_t search = nl_map_get(cfg->node_to_block, n); - return search >= 0 ? cfg->node_to_block[search].v.id : -1; -} - -static int resolve_dom_depth(TB_CFG* cfg, TB_Node* bb) { - if (dom_depth(cfg, bb) >= 0) { - return dom_depth(cfg, bb); - } - - int parent = resolve_dom_depth(cfg, idom(cfg, bb)); - - // it's one more than it's parent - nl_map_get_checked(cfg->node_to_block, bb).dom_depth = parent + 1; - return parent + 1; -} - -static TB_BasicBlock* get_pred_bb(TB_CFG* cfg, TB_Node* n, int i) { - n = get_pred(n, i); - return &nl_map_get_checked(cfg->node_to_block, n); -} - -TB_DominanceFrontiers* tb_get_dominance_frontiers(TB_Function* f, TB_Passes* restrict p, TB_CFG cfg, TB_Node** blocks) { - size_t stride = (cfg.block_count + 63) / 64; - size_t elems = stride * cfg.block_count; - size_t size = sizeof(TB_DominanceFrontiers) + sizeof(uint64_t)*elems; - - TB_DominanceFrontiers* df = tb_platform_heap_alloc(size); - memset(df, 0, size); - df->stride = stride; - - FOREACH_N(i, 0, cfg.block_count) { - TB_Node* bb = blocks[i]; - assert(find_traversal_index(&cfg, bb) == i); - - if (cfg_is_region(bb) && bb->input_count >= 2) { - FOREACH_N(k, 0, bb->input_count) { - TB_Node* runner = cfg_get_pred(&cfg, bb, k); - - while (!(runner->type == TB_PROJ && runner->inputs[0]->type == TB_ROOT) && runner != idom(&cfg, bb)) { - // add to frontier set - int id = nl_map_get_checked(cfg.node_to_block, runner).id; - tb_dommy_fronts_put(df, id, i); - - runner = idom(&cfg, runner); - } - } - } - } - - return df; -} - -TB_API void tb_free_dominance_frontiers(TB_DominanceFrontiers* df) { - tb_platform_heap_free(df); -} - -void tb_compute_dominators(TB_Function* f, TB_Passes* restrict p, TB_CFG cfg) { - tb_compute_dominators2(f, &p->worklist, cfg); -} - -// Cooper, Keith D., Harvey, Timothy J. and Kennedy, Ken. "A simple, fast dominance algorithm." (2006) -// https://repository.rice.edu/items/99a574c3-90fe-4a00-adf9-ce73a21df2ed -void tb_compute_dominators2(TB_Function* f, Worklist* ws, TB_CFG cfg) { - TB_Node** blocks = ws->items; - - TB_BasicBlock* entry = &nl_map_get_checked(cfg.node_to_block, blocks[0]); - entry->dom = entry; - - bool changed = true; - while (changed) { - changed = false; - - // for all nodes, b, in reverse postorder (except start node) - FOREACH_N(i, 1, cfg.block_count) { - TB_Node* b = blocks[i]; - TB_Node* new_idom = NULL; - - // pick first "processed" pred - size_t j = 0, pred_count = b->input_count; - for (; j < pred_count; j++) { - TB_Node* p = cfg_get_pred(&cfg, b, j); - if (idom(&cfg, p) != NULL) { - new_idom = p; - break; - } - } - - // for all other predecessors, p, of b - for (; j < pred_count; j++) { - TB_Node* p = cfg_get_pred(&cfg, b, j); - - // if doms[p] already calculated - TB_Node* idom_p = idom(&cfg, p); - if (idom_p != NULL) { - assert(p->input_count > 0); - - int a = try_find_traversal_index(&cfg, p); - if (a >= 0) { - int b = find_traversal_index(&cfg, new_idom); - while (a != b) { - // while (finger1 < finger2) - // finger1 = doms[finger1] - while (a > b) { - TB_Node* d = idom(&cfg, blocks[a]); - a = d ? find_traversal_index(&cfg, d) : 0; - } - - // while (finger2 < finger1) - // finger2 = doms[finger2] - while (b > a) { - TB_Node* d = idom(&cfg, blocks[b]); - b = d ? find_traversal_index(&cfg, d) : 0; - } - } - - new_idom = blocks[a]; - } - } - } - - assert(new_idom != NULL); - TB_BasicBlock* b_bb = &nl_map_get_checked(cfg.node_to_block, b); - if (b_bb->dom == NULL || b_bb->dom->start != new_idom) { - b_bb->dom = &nl_map_get_checked(cfg.node_to_block, new_idom); - changed = true; - } - } - } - - // generate depth values - CUIK_TIMED_BLOCK("generate dom tree") { - FOREACH_REVERSE_N(i, 1, cfg.block_count) { - resolve_dom_depth(&cfg, blocks[i]); - } - } -} - -bool tb_is_dominated_by(TB_CFG cfg, TB_Node* expected_dom, TB_Node* n) { - TB_BasicBlock* expected = &nl_map_get_checked(cfg.node_to_block, expected_dom); - TB_BasicBlock* bb = &nl_map_get_checked(cfg.node_to_block, n); - - while (bb != expected) { - if (bb->dom == bb) { - return false; - } - bb = bb->dom; - } - - return true; -} diff --git a/vendor/tb/src/opt/fold.h b/vendor/tb/src/opt/fold.h deleted file mode 100644 index 721c5c47..00000000 --- a/vendor/tb/src/opt/fold.h +++ /dev/null @@ -1,1026 +0,0 @@ - -static bool get_int_const(TB_Node* n, uint64_t* imm) { - if (n->type == TB_INTEGER_CONST) { - TB_NodeInt* i = TB_NODE_GET_EXTRA(n); - *imm = i->value; - return true; - } else { - return false; - } -} - -//////////////////////////////// -// Integer idealizations -//////////////////////////////// -static TB_Node* ideal_bitcast(TB_Passes* restrict opt, TB_Function* f, TB_Node* n) { - TB_Node* src = n->inputs[1]; - - if (src->type == TB_BITCAST) { - set_input(f, n, src->inputs[1], 1); - return n; - } - - // int -> smaller int means truncate - if (src->dt.type == TB_INT && n->dt.type == TB_INT && src->dt.data > n->dt.data) { - n->type = TB_TRUNCATE; - return n; - } else if (src->type == TB_INTEGER_CONST) { - return make_int_node(f, opt, n->dt, TB_NODE_GET_EXTRA_T(src, TB_NodeInt)->value); - } - - return NULL; -} - -// cmp.slt(a, 0) => is_sign(a) -static bool sign_check(TB_Node* n) { - uint64_t x; - return n->type == TB_CMP_SLT && get_int_const(n->inputs[2], &x) && x == 0; -} - -static bool is_non_zero(TB_Node* n) { - TB_NodeInt* i = TB_NODE_GET_EXTRA(n); - return n->type == TB_INTEGER_CONST && i->value != 0; -} - -static bool is_zero(TB_Node* n) { - TB_NodeInt* i = TB_NODE_GET_EXTRA(n); - return n->type == TB_INTEGER_CONST && i->value == 0; -} - -static bool is_one(TB_Node* n) { - TB_NodeInt* i = TB_NODE_GET_EXTRA(n); - return n->type == TB_INTEGER_CONST && i->value == 1; -} - -static bool inverted_cmp(TB_Node* n, TB_Node* n2) { - switch (n->type) { - case TB_CMP_EQ: return n2->type == TB_CMP_NE && n2->inputs[1] == n->inputs[1] && n2->inputs[2] == n->inputs[2]; - case TB_CMP_NE: return n2->type == TB_CMP_EQ && n2->inputs[1] == n->inputs[1] && n2->inputs[2] == n->inputs[2]; - // flipped inputs - case TB_CMP_SLE: return n2->type == TB_CMP_SLT && n2->inputs[2] == n->inputs[1] && n2->inputs[1] == n->inputs[2]; - case TB_CMP_ULE: return n2->type == TB_CMP_ULT && n2->inputs[2] == n->inputs[1] && n2->inputs[1] == n->inputs[2]; - case TB_CMP_SLT: return n2->type == TB_CMP_SLE && n2->inputs[2] == n->inputs[1] && n2->inputs[1] == n->inputs[2]; - case TB_CMP_ULT: return n2->type == TB_CMP_ULE && n2->inputs[2] == n->inputs[1] && n2->inputs[1] == n->inputs[2]; - default: return false; - } -} - -static Lattice* value_sext(TB_Passes* restrict opt, TB_Node* n) { - Lattice* a = lattice_universe_get(opt, n->inputs[1]); - if (a == &TOP_IN_THE_SKY) { - return &TOP_IN_THE_SKY; - } - - int old_bits = n->inputs[1]->dt.data; - - uint64_t sign_range = (1 << (old_bits - 1)) - 1; - uint64_t mask = tb__mask(n->dt.data) & ~tb__mask(old_bits); - - int64_t min, max; - if (a->_int.min >= a->_int.max || a->_int.max < sign_range) { - min = tb__sxt(a->_int.min, old_bits, n->dt.data); - max = tb__sxt(a->_int.max, old_bits, n->dt.data); - } else { - min = lattice_int_min(old_bits) | mask; - max = lattice_int_max(old_bits); - } - - uint64_t zeros = a->_int.known_zeros; - uint64_t ones = a->_int.known_ones; - - // if we know the sign bit then we can know what the extended bits look like - if (zeros >> (old_bits - 1)) { - zeros |= mask; - } else if (ones >> (old_bits - 1)) { - ones |= mask; - } - - return lattice_intern(opt, (Lattice){ LATTICE_INT, ._int = { min, max, zeros, ones } }); -} - -static Lattice* value_zext(TB_Passes* restrict opt, TB_Node* n) { - Lattice* a = lattice_universe_get(opt, n->inputs[1]); - if (a == &TOP_IN_THE_SKY) { - return &TOP_IN_THE_SKY; - } - - uint64_t mask = tb__mask(n->dt.data) & ~tb__mask(n->inputs[1]->dt.data); - - int64_t min = a->_int.min; - int64_t max = a->_int.max; - uint64_t zeros = a->_int.known_zeros | mask; // we know the top bits must be zero - uint64_t ones = a->_int.known_ones; - - return lattice_intern(opt, (Lattice){ LATTICE_INT, ._int = { min, max, zeros, ones } }); -} - -static Lattice* value_trunc(TB_Passes* restrict opt, TB_Node* n) { - Lattice* a = lattice_universe_get(opt, n->inputs[1]); - if (a == &TOP_IN_THE_SKY) { - return &TOP_IN_THE_SKY; - } - - int64_t mask = tb__mask(n->dt.data); - int64_t min = a->_int.min & mask; - int64_t max = a->_int.max & mask; - if (min > max) { - min = 0; - max = mask; - } - - uint64_t zeros = a->_int.known_zeros | ~mask; - uint64_t ones = a->_int.known_ones & mask; - return lattice_intern(opt, (Lattice){ LATTICE_INT, ._int = { min, max, zeros, ones } }); -} - -static Lattice* value_arith(TB_Passes* restrict opt, TB_Node* n) { - Lattice* a = lattice_universe_get(opt, n->inputs[1]); - Lattice* b = lattice_universe_get(opt, n->inputs[2]); - if (a == &TOP_IN_THE_SKY || b == &TOP_IN_THE_SKY) { - return &TOP_IN_THE_SKY; - } - - assert(a->tag == LATTICE_INT && b->tag == LATTICE_INT); - uint64_t mask = tb__mask(n->dt.data); - uint64_t min, max; - bool overflow = false; - switch (n->type) { - case TB_ADD: - overflow |= l_add_overflow(a->_int.min, b->_int.min, mask, &min); - overflow |= l_add_overflow(a->_int.max, b->_int.max, mask, &max); - if (min > max) { overflow = true; } - break; - - case TB_SUB: - overflow |= l_sub_overflow(a->_int.min, b->_int.min, mask, &min); - overflow |= l_sub_overflow(a->_int.max, b->_int.max, mask, &max); - if (min > max) { overflow = true; } - break; - - case TB_MUL: - overflow |= l_mul_overflow(a->_int.min, b->_int.min, mask, &min); - overflow |= l_mul_overflow(a->_int.max, b->_int.max, mask, &max); - if (min > max) { overflow = true; } - break; - } - - // truncate to the size of the raw DataType - min &= mask, max &= mask; - - if (min == max) { - return lattice_intern(opt, (Lattice){ LATTICE_INT, ._int = { min, min, ~min, min } }); - } else if (overflow) { - return lattice_intern(opt, (Lattice){ LATTICE_INT, ._int = { 0, mask } }); - } else { - return lattice_intern(opt, (Lattice){ LATTICE_INT, ._int = { min, max } }); - } -} - -static Lattice* value_bitcast(TB_Passes* restrict opt, TB_Node* n) { - Lattice* a = lattice_universe_get(opt, n->inputs[1]); - if (a == &TOP_IN_THE_SKY) { - return &TOP_IN_THE_SKY; - } - - if (a->tag == LATTICE_INT && a->_int.min == a->_int.max && n->dt.type == TB_PTR) { - // bitcast with a constant leads to fun cool stuff (usually we get constant zeros for NULL) - return a->_int.min ? &XNULL_IN_THE_SKY : &NULL_IN_THE_SKY; - } - - return NULL; -} - -static Lattice* value_unary(TB_Passes* restrict opt, TB_Node* n) { - Lattice* a = lattice_universe_get(opt, n->inputs[1]); - if (a == &TOP_IN_THE_SKY) { - return &TOP_IN_THE_SKY; - } - - if (a->tag == LATTICE_INT) { - uint64_t mask = tb__mask(n->dt.data); - uint64_t min = ~a->_int.min & mask; - uint64_t max = ~a->_int.max & mask; - - if (wrapped_int_lt(max, min, n->dt.data)) { - SWAP(int64_t, min, max); - } - - uint64_t zeros = 0, ones = 0; - if (n->type == TB_NEG) { - // -x => ~x + 1 - // because of this addition we can technically - // overflow... umm? glhf? - uint64_t min_inc = (min+1) & mask; - uint64_t max_inc = (max+1) & mask; - - if (min_inc < min || max_inc < min) { - min = lattice_int_min(n->dt.data); - max = lattice_int_max(n->dt.data); - } else { - min = min_inc; - max = max_inc; - } - } else { - zeros = a->_int.known_ones; - ones = a->_int.known_zeros; - } - - return lattice_intern(opt, (Lattice){ LATTICE_INT, ._int = { min, max, zeros, ones } }); - } else { - return NULL; - } -} - -static Lattice* value_bits(TB_Passes* restrict opt, TB_Node* n) { - Lattice* a = lattice_universe_get(opt, n->inputs[1]); - Lattice* b = lattice_universe_get(opt, n->inputs[2]); - if (a == &TOP_IN_THE_SKY || b == &TOP_IN_THE_SKY) { - return &TOP_IN_THE_SKY; - } - - uint64_t zeros, ones; - switch (n->type) { - case TB_AND: - // 0 if either is zero, 1 if both are 1 - zeros = a->_int.known_zeros | b->_int.known_zeros; - ones = a->_int.known_ones & b->_int.known_ones; - break; - - case TB_OR: - // 0 if both are 0, 1 if either is 1 - zeros = a->_int.known_zeros & b->_int.known_zeros; - ones = a->_int.known_ones | b->_int.known_ones; - break; - - case TB_XOR: - // 0 if both bits are 0 or 1 - // 1 if both bits aren't the same - zeros = (a->_int.known_zeros & b->_int.known_zeros) | (a->_int.known_ones & b->_int.known_ones); - ones = (a->_int.known_zeros & b->_int.known_ones) | (a->_int.known_ones & b->_int.known_zeros); - break; - - default: tb_todo(); - } - - uint64_t mask = tb__mask(n->dt.data); - zeros &= mask, ones &= mask; - - // we can deduce a min and max by assuming the unknown bits are either zeros or ones - int64_t min = ones, max = ~zeros; - if (wrapped_int_lt(max, min, n->dt.data)) { - min = lattice_int_min(n->dt.data); - max = lattice_int_max(n->dt.data); - } - min &= mask, max &= mask; - - return lattice_intern(opt, (Lattice){ LATTICE_INT, ._int = { min, max, zeros, ones } }); -} - -static Lattice* value_shift(TB_Passes* restrict opt, TB_Node* n) { - Lattice* a = lattice_universe_get(opt, n->inputs[1]); - Lattice* b = lattice_universe_get(opt, n->inputs[2]); - if (a == &TOP_IN_THE_SKY || b == &TOP_IN_THE_SKY) { - return &TOP_IN_THE_SKY; - } - - if (b->tag == LATTICE_INT && b->_int.max > b->_int.min) { - return NULL; - } - - uint64_t bits = n->dt.data; - uint64_t mask = tb__mask(n->dt.data); - - // shift that's in-bounds can tell us quite a few nice details - if (b->_int.max <= bits) { - uint64_t min, max, zeros, ones = 0; - switch (n->type) { - case TB_SHL: - min = a->_int.min << b->_int.min; - max = a->_int.max << b->_int.max; - min &= mask, max &= mask; - - if (((a->_int.min & b->_int.min) < 0 && min >= 0) || - (~(a->_int.max | b->_int.max) < 0 && max < 0) || - wrapped_int_lt(max, min, n->dt.data) - ) { - min = lattice_int_min(n->dt.data); - max = lattice_int_max(n->dt.data); - } - - // we at least shifted this many bits therefore we - // at least have this many zeros at the bottom - zeros = (1ull << b->_int.min) - 1ull; - // if we know how many bits we shifted then we know where - // our known ones ones went - if (b->_int.min == b->_int.max) { - ones <<= b->_int.min; - } - break; - - case TB_SAR: - case TB_SHR: - // perform shift logic as unsigned - min = a->_int.min; - max = a->_int.max; - if (min > max) { - min = 0, max = mask; - } - - // the largest value is caused by the lowest shift amount - min >>= b->_int.max; - max >>= b->_int.min; - - // convert range back into signed - if (wrapped_int_lt(max, min, n->dt.data)) { - min = lattice_int_min(n->dt.data); - max = lattice_int_max(n->dt.data); - } - - // TODO(NeGate): we can technically guarentee the top bits are zero - zeros = 0; - // if we know how many bits we shifted then we know where - // our known ones ones went - if (b->_int.min == b->_int.max) { - ones = 0; - zeros = ~(mask >> b->_int.min) & mask; - } - break; - - default: tb_todo(); - } - - return lattice_intern(opt, (Lattice){ LATTICE_INT, ._int = { min, max, zeros, ones } }); - } else { - return NULL; - } -} - -static Lattice* value_cmp(TB_Passes* restrict opt, TB_Node* n) { - Lattice* a = lattice_universe_get(opt, n->inputs[1]); - Lattice* b = lattice_universe_get(opt, n->inputs[2]); - if (a == &TOP_IN_THE_SKY || b == &TOP_IN_THE_SKY) { - return &TOP_IN_THE_SKY; - } - - if (a == &BOT_IN_THE_SKY || b == &BOT_IN_THE_SKY) { - return &BOT_IN_THE_SKY; - } - - TB_DataType dt = n->inputs[1]->dt; - if (dt.type == TB_INT) { - uint64_t mask = tb__mask(dt.data); - uint64_t sign_range = (1 << (dt.data - 1)) - 1; - - bool a_cst = a->_int.min == a->_int.max; - bool b_cst = b->_int.min == b->_int.max; - - int cmp = 2; // 0 or 1 (2 for BOT) - switch (n->type) { - case TB_CMP_EQ: - if (a_cst && b_cst) cmp = a->_int.min == b->_int.min; - break; - - case TB_CMP_NE: - if (a_cst && b_cst) cmp = a->_int.min != b->_int.min; - break; - - case TB_CMP_SLE: - case TB_CMP_SLT: - if (a->_int.max < sign_range && b->_int.max < sign_range) { - if (wrapped_int_lt(a->_int.max, b->_int.min, dt.data)) cmp = 1; - if (wrapped_int_lt(b->_int.max, a->_int.min, dt.data)) cmp = 0; - } - break; - - case TB_CMP_ULT: - case TB_CMP_ULE: - if (a->_int.min <= a->_int.max && b->_int.min <= b->_int.max) { - if ((a->_int.max & mask) < (b->_int.min & mask)) cmp = 1; - if ((b->_int.max & mask) < (a->_int.min & mask)) cmp = 0; - } - break; - } - - if (cmp != 2) { - return lattice_intern(opt, (Lattice){ LATTICE_INT, ._int = { cmp, cmp, ~cmp, cmp } }); - } - } else if (dt.type == TB_PTR && (n->type == TB_CMP_EQ || n->type == TB_CMP_NE)) { - a = lattice_meet(opt, a, &XNULL_IN_THE_SKY, TB_TYPE_PTR); - b = lattice_meet(opt, b, &XNULL_IN_THE_SKY, TB_TYPE_PTR); - - if (n->type == TB_CMP_EQ) { - return a == b ? &TRUE_IN_THE_SKY : &FALSE_IN_THE_SKY; - } else { - return a != b ? &TRUE_IN_THE_SKY : &FALSE_IN_THE_SKY; - } - } - - return NULL; -} - -static TB_Node* ideal_select(TB_Passes* restrict opt, TB_Function* f, TB_Node* n) { - TB_Node* src = n->inputs[1]; - - LatticeTrifecta key_truthy = lattice_truthy(lattice_universe_get(opt, src)); - if (key_truthy == LATTICE_KNOWN_TRUE) { - return n->inputs[2]; - } else if (key_truthy == LATTICE_KNOWN_FALSE) { - return n->inputs[3]; - } - - // select(y <= x, a, b) => select(x < y, b, a) flipped conditions - if (src->type == TB_CMP_SLE || src->type == TB_CMP_ULE) { - TB_Node* new_cmp = tb_alloc_node(f, src->type == TB_CMP_SLE ? TB_CMP_SLT : TB_CMP_ULT, TB_TYPE_BOOL, 3, sizeof(TB_NodeCompare)); - set_input(f, new_cmp, src->inputs[2], 1); - set_input(f, new_cmp, src->inputs[1], 2); - TB_NODE_SET_EXTRA(new_cmp, TB_NodeCompare, .cmp_dt = TB_NODE_GET_EXTRA_T(src, TB_NodeCompare)->cmp_dt); - - set_input(f, n, new_cmp, 1); - tb_pass_mark(opt, new_cmp); - return n; - } - - // T(some_bool ? 1 : 0) => movzx(T, some_bool) - if (src->dt.type == TB_INT && src->dt.data == 1) { - uint64_t on_true, on_false; - bool true_imm = get_int_const(n->inputs[2], &on_true); - bool false_imm = get_int_const(n->inputs[3], &on_false); - - // A ? A : 0 => A (booleans) - if (src == n->inputs[2] && false_imm && on_false == 0) { - return src; - } - - // A ? 0 : !A => A (booleans) - if (inverted_cmp(src, n->inputs[3]) && true_imm && on_true == 0) { - return src; - } - - if (true_imm && false_imm && on_true == 1 && on_false == 0) { - TB_Node* ext_node = tb_alloc_node(f, TB_ZERO_EXT, n->dt, 2, 0); - set_input(f, ext_node, src, 1); - tb_pass_mark(opt, ext_node); - return ext_node; - } - } - - // (select.f32 (v43: cmp.lt.f32 ...) (v41: load.f32 ...) (v42: load.f32 ...)) - if (n->dt.type == TB_FLOAT && src->type == TB_CMP_FLT) { - TB_Node* a = src->inputs[1]; - TB_Node* b = src->inputs[2]; - - // (select (lt A B) A B) => (min A B) - if (n->inputs[2] == a && n->inputs[3] == b) { - TB_Node* new_node = tb_alloc_node(f, TB_FMIN, n->dt, 3, 0); - set_input(f, new_node, a, 1); - set_input(f, new_node, b, 2); - return new_node; - } - - // (select (lt A B) B A) => (max A B) - if (n->inputs[2] == b && n->inputs[3] == a) { - TB_Node* new_node = tb_alloc_node(f, TB_FMAX, n->dt, 3, 0); - set_input(f, new_node, a, 1); - set_input(f, new_node, b, 2); - return new_node; - } - } - - return NULL; -} - -static bool nice_ass_trunc(TB_NodeTypeEnum t) { return t == TB_ADD || t == TB_AND || t == TB_XOR || t == TB_OR || t == TB_MUL || t == TB_SHL || t == TB_SHR || t == TB_SAR || t == TB_SMOD || t == TB_UMOD; } -static TB_Node* ideal_truncate(TB_Passes* restrict opt, TB_Function* f, TB_Node* n) { - TB_Node* src = n->inputs[1]; - - if (src->type == TB_ZERO_EXT && src->inputs[1]->dt.type == TB_INT && n->dt.type == TB_INT) { - int now = n->dt.data; - int before = src->inputs[1]->dt.data; - - if (now != before) { - // we're extending the original value - TB_Node* ext = tb_alloc_node(f, now < before ? TB_TRUNCATE : src->type, n->dt, 2, 0); - set_input(f, ext, src->inputs[1], 1); - return ext; - } else { - return src->inputs[1]; - } - } - - // Trunc(NiceAssBinop(a, b)) => NiceAssBinop(Trunc(a), Trunc(b)) - if (nice_ass_trunc(src->type)) { - TB_Node* left = tb_alloc_node(f, TB_TRUNCATE, n->dt, 2, 0); - set_input(f, left, src->inputs[1], 1); - tb_pass_mark(opt, left); - - TB_Node* right = tb_alloc_node(f, TB_TRUNCATE, n->dt, 2, 0); - set_input(f, right, src->inputs[2], 1); - tb_pass_mark(opt, right); - - TB_Node* new_binop = tb_alloc_node(f, src->type, n->dt, 3, 0); - set_input(f, new_binop, left, 1); - set_input(f, new_binop, right, 2); - return new_binop; - } - - return NULL; -} - -static TB_Node* ideal_extension(TB_Passes* restrict opt, TB_Function* f, TB_Node* n) { - TB_NodeTypeEnum ext_type = n->type; - TB_Node* src = n->inputs[1]; - - if (src->type == ext_type) { - do { - src = src->inputs[1]; - } while (src->type == ext_type); - set_input(f, n, src, 1); - return n; - } - - // Ext(phi(a: con, b: con)) => phi(Ext(a: con), Ext(b: con)) - if (src->type == TB_PHI) { - FOREACH_N(i, 1, src->input_count) { - if (src->inputs[i]->type != TB_INTEGER_CONST) return NULL; - } - - // generate extension nodes - TB_DataType dt = n->dt; - FOREACH_N(i, 1, src->input_count) { - assert(src->inputs[i]->type == TB_INTEGER_CONST); - - TB_Node* ext_node = tb_alloc_node(f, ext_type, dt, 2, 0); - set_input(f, ext_node, src->inputs[i], 1); - set_input(f, src, ext_node, i); - tb_pass_mark(opt, ext_node); - } - - src->dt = dt; - return src; - } - - // Cast(NiceAssBinop(a, b)) => NiceAssBinop(Cast(a), Cast(b)) - if (nice_ass_trunc(src->type)) { - TB_Node* left = tb_alloc_node(f, ext_type, n->dt, 2, 0); - set_input(f, left, src->inputs[1], 1); - tb_pass_mark(opt, left); - - TB_Node* right = tb_alloc_node(f, ext_type, n->dt, 2, 0); - set_input(f, right, src->inputs[2], 1); - tb_pass_mark(opt, right); - - TB_Node* new_binop = tb_alloc_node(f, src->type, n->dt, 3, 0); - set_input(f, new_binop, left, 1); - set_input(f, new_binop, right, 2); - return new_binop; - } - - return NULL; -} - -static int node_pos(TB_Node* n) { - switch (n->type) { - case TB_INTEGER_CONST: - case TB_FLOAT32_CONST: - case TB_FLOAT64_CONST: - return 1; - - default: - return 2; - - case TB_PHI: - return 3; - } -} - -static bool is_shift_op(TB_Node* n) { - return n->type == TB_SHL || n->type == TB_SHR || n->type == TB_SAR; -} - -static bool is_iconst(TB_Passes* p, TB_Node* n) { return lattice_is_const(lattice_universe_get(p, n)); } -static TB_Node* ideal_int_binop(TB_Passes* restrict p, TB_Function* f, TB_Node* n) { - TB_NodeTypeEnum type = n->type; - TB_Node* a = n->inputs[1]; - TB_Node* b = n->inputs[2]; - - // if it's commutative: we wanna have a canonical form. - if (is_commutative(type)) { - // if they're the same rank, then we'll just shuffle for smaller gvn on the right - int ap = node_pos(a); - int bp = node_pos(b); - if (ap < bp || (ap == bp && a->gvn < b->gvn)) { - set_input(f, n, b, 1); - set_input(f, n, a, 2); - return n; - } - } - - // (aa + ab) + b => aa + (ab + b) where ab and b are constant - if (is_associative(type) && a->type == type && is_iconst(p, a->inputs[2]) && is_iconst(p, b)) { - TB_Node* abb = tb_alloc_node(f, type, n->dt, 3, sizeof(TB_NodeBinopInt)); - set_input(f, abb, a->inputs[2], 1); - set_input(f, abb, b, 2); - - Lattice* l = value_arith(p, abb); - assert(l->tag == LATTICE_INT && l->_int.min == l->_int.max); - - violent_kill(f, abb); - - TB_Node* con = make_int_node(f, p, n->dt, l->_int.min); - set_input(f, n, a->inputs[1], 1); - set_input(f, n, con, 2); - return n; - } - - if (type == TB_OR) { - assert(n->dt.type == TB_INT); - int bits = n->dt.data; - - // (or (shl a 24) (shr a 40)) => (rol a 24) - if (a->type == TB_SHL && (b->type == TB_SHR || b->type == TB_SAR)) { - uint64_t shl_amt, shr_amt; - if (a->inputs[1] == b->inputs[1] && - get_int_const(a->inputs[2], &shl_amt) && - get_int_const(b->inputs[2], &shr_amt) && - shl_amt == bits - shr_amt) { - // convert to rotate left - n->type = TB_ROL; - set_input(f, n, a->inputs[1], 1); - set_input(f, n, a->inputs[2], 2); - return n; - } - } - } else if (type == TB_MUL) { - uint64_t rhs; - if (get_int_const(b, &rhs)) { - uint64_t log2 = tb_ffs(rhs) - 1; - if (rhs == (UINT64_C(1) << log2)) { - TB_Node* shl_node = tb_alloc_node(f, TB_SHL, n->dt, 3, sizeof(TB_NodeBinopInt)); - set_input(f, shl_node, a, 1); - set_input(f, shl_node, make_int_node(f, p, n->dt, log2), 2); - - tb_pass_mark(p, shl_node->inputs[1]); - tb_pass_mark(p, shl_node->inputs[2]); - return shl_node; - } - } - } else if (type == TB_CMP_EQ) { - // (a == 0) is !a - TB_Node* cmp = n->inputs[1]; - - uint64_t rhs; - if (get_int_const(n->inputs[2], &rhs) && rhs == 0) { - // !(a < b) is (b <= a) - switch (cmp->type) { - case TB_CMP_EQ: n->type = TB_CMP_EQ; break; - case TB_CMP_NE: n->type = TB_CMP_NE; break; - case TB_CMP_SLT: n->type = TB_CMP_SLE; break; - case TB_CMP_SLE: n->type = TB_CMP_SLT; break; - case TB_CMP_ULT: n->type = TB_CMP_ULE; break; - case TB_CMP_ULE: n->type = TB_CMP_ULT; break; - default: return NULL; - } - - TB_DataType cmp_dt = TB_NODE_GET_EXTRA_T(cmp, TB_NodeCompare)->cmp_dt; - TB_NODE_SET_EXTRA(n, TB_NodeCompare, .cmp_dt = cmp_dt); - - set_input(f, n, cmp->inputs[2], 1); - set_input(f, n, cmp->inputs[1], 2); - return n; - } - } else if (type == TB_SHL || type == TB_SHR || type == TB_SAR) { - // (a << b) >> c = (a << (b - c)) & (ALL >> b) - // (a >> b) << c = (a >> (b - c)) & ((1 << b) - 1) - uint64_t b, c; - if ((n->inputs[1]->type == TB_SHL || n->inputs[1]->type == TB_SHR || n->inputs[1]->type == TB_SAR) && - get_int_const(n->inputs[2], &c) && get_int_const(n->inputs[1]->inputs[2], &b)) { - TB_NodeTypeEnum inner_shift = n->inputs[1]->type; - - // track how far we're shifting (and how many bits need chopping) - int amt = inner_shift == TB_SHL ? b : -b; - uint64_t mask = inner_shift == TB_SHL ? UINT64_MAX << b : UINT64_MAX >> c; - - // apply outer shift - amt += type == TB_SHL ? c : -c; - mask = type == TB_SHL ? mask << b : mask >> b; - - TB_Node* shift = n->inputs[1]->inputs[1]; - if (amt) { - TB_Node* imm = make_int_node(f, p, n->dt, b - c); - tb_pass_mark(p, imm); - - // if we have a negative shift amount, that's a right shift - shift = tb_alloc_node(f, amt < 0 ? TB_SHR : TB_SHL, n->dt, 3, sizeof(TB_NodeBinopInt)); - set_input(f, shift, n->inputs[1]->inputs[1], 1); - set_input(f, shift, imm, 2); - - tb_pass_mark(p, shift); - } - - TB_Node* mask_node = make_int_node(f, p, n->dt, mask); - TB_Node* and_node = tb_alloc_node(f, TB_AND, n->dt, 3, sizeof(TB_NodeBinopInt)); - set_input(f, and_node, shift, 1); - set_input(f, and_node, mask_node, 2); - return and_node; - } - } - - if (type >= TB_CMP_EQ && type <= TB_CMP_ULE) { - // (Cmp Sxt(a) Sxt(b)) => (Cmp a b) - if (n->inputs[1]->type == TB_SIGN_EXT && n->inputs[2]->type == TB_SIGN_EXT) { - TB_DataType dt = n->inputs[1]->inputs[1]->dt; - set_input(f, n, n->inputs[1]->inputs[1], 1); - set_input(f, n, n->inputs[2]->inputs[1], 2); - TB_NODE_SET_EXTRA(n, TB_NodeCompare, .cmp_dt = dt); - return n; - } - } - - return NULL; -} - -static TB_Node* ideal_int_mod(TB_Passes* restrict opt, TB_Function* f, TB_Node* n) { - bool is_signed = n->type == TB_SMOD; - - TB_DataType dt = n->dt; - TB_Node* x = n->inputs[1]; - - uint64_t y = TB_NODE_GET_EXTRA_T(n->inputs[2], TB_NodeInt)->value; - uint64_t log2 = tb_ffs(y) - 1; - if (!is_signed && y == (UINT64_C(1) << log2)) { - TB_Node* and_node = tb_alloc_node(f, TB_AND, dt, 3, sizeof(TB_NodeBinopInt)); - set_input(f, and_node, x, 1); - set_input(f, and_node, make_int_node(f, opt, dt, y - 1), 2); - return and_node; - } - - return NULL; -} - -static TB_Node* ideal_int_div(TB_Passes* restrict opt, TB_Function* f, TB_Node* n) { - bool is_signed = n->type == TB_SDIV; - - // if we have a constant denominator we may be able to reduce the division into a - // multiply and shift-right - if (n->inputs[2]->type != TB_INTEGER_CONST) return NULL; - - // https://gist.github.com/B-Y-P/5872dbaaf768c204480109007f64a915 - TB_DataType dt = n->dt; - TB_Node* x = n->inputs[1]; - - uint64_t y = TB_NODE_GET_EXTRA_T(n->inputs[2], TB_NodeInt)->value; - if (y >= (1ull << 63ull)) { - // we haven't implemented the large int case - return NULL; - } else if (y == 0) { - return tb_alloc_node(f, TB_POISON, dt, 1, 0); - } else if (y == 1) { - return x; - } else { - // (udiv a N) => a >> log2(N) where N is a power of two - uint64_t log2 = tb_ffs(y) - 1; - if (!is_signed && y == (UINT64_C(1) << log2)) { - TB_Node* shr_node = tb_alloc_node(f, TB_SHR, dt, 3, sizeof(TB_NodeBinopInt)); - set_input(f, shr_node, x, 1); - set_input(f, shr_node, make_int_node(f, opt, dt, log2), 2); - return shr_node; - } - } - - // idk how to handle this yet - if (is_signed) return NULL; - - uint64_t sh = (64 - tb_clz64(y)) - 1; // sh = ceil(log2(y)) + w - 64 - - #ifndef NDEBUG - uint64_t sh2 = 0; - while(y > (1ull << sh2)){ sh2++; } // sh' = ceil(log2(y)) - sh2 += 63 - 64; // sh = ceil(log2(y)) + w - 64 - - assert(sh == sh2); - #endif - - // 128bit division here can't overflow - uint64_t a = tb_div128(1ull << sh, y - 1, y); - - // now we can take a and sh and do: - // x / y => mulhi(x, a) >> sh - int bits = dt.data; - if (bits > 32) { - TB_Node* mul_node = tb_alloc_node(f, TB_MULPAIR, TB_TYPE_TUPLE, 3, 0); - set_input(f, mul_node, x, 1); - set_input(f, mul_node, make_int_node(f, opt, dt, a), 2); - - TB_Node* lo = make_proj_node(f, dt, mul_node, 0); - TB_Node* hi = make_proj_node(f, dt, mul_node, 1); - - tb_pass_mark(opt, mul_node); - tb_pass_mark(opt, lo); - tb_pass_mark(opt, hi); - - TB_Node* sh_node = tb_alloc_node(f, TB_SHR, dt, 3, sizeof(TB_NodeBinopInt)); - set_input(f, sh_node, hi, 1); - set_input(f, sh_node, make_int_node(f, opt, dt, sh), 2); - TB_NODE_SET_EXTRA(sh_node, TB_NodeBinopInt, .ab = 0); - - return sh_node; - } else { - TB_DataType big_dt = TB_TYPE_INTN(bits * 2); - sh += bits; // chopping the low half - - a &= (1ull << bits) - 1; - - // extend x - TB_Node* ext_node = tb_alloc_node(f, TB_ZERO_EXT, big_dt, 2, 0); - set_input(f, ext_node, x, 1); - - TB_Node* mul_node = tb_alloc_node(f, TB_MUL, big_dt, 3, sizeof(TB_NodeBinopInt)); - set_input(f, mul_node, ext_node, 1); - set_input(f, mul_node, make_int_node(f, opt, big_dt, a), 2); - TB_NODE_SET_EXTRA(mul_node, TB_NodeBinopInt, .ab = 0); - - TB_Node* sh_node = tb_alloc_node(f, TB_SHR, big_dt, 3, sizeof(TB_NodeBinopInt)); - set_input(f, sh_node, mul_node, 1); - set_input(f, sh_node, make_int_node(f, opt, big_dt, sh), 2); - TB_NODE_SET_EXTRA(sh_node, TB_NodeBinopInt, .ab = 0); - - TB_Node* trunc_node = tb_alloc_node(f, TB_TRUNCATE, dt, 2, 0); - set_input(f, trunc_node, sh_node, 1); - - tb_pass_mark(opt, mul_node); - tb_pass_mark(opt, sh_node); - tb_pass_mark(opt, ext_node); - return trunc_node; - } -} - -//////////////////////////////// -// Integer identities -//////////////////////////////// -// a + 0 => a -// a - 0 => a -// a ^ 0 => a -// a * 0 => 0 -// a / 0 => poison -static TB_Node* identity_int_binop(TB_Passes* restrict p, TB_Function* f, TB_Node* n) { - if (n->type == TB_AND) { - Lattice* aa = lattice_universe_get(p, n->inputs[1]); - Lattice* bb = lattice_universe_get(p, n->inputs[2]); - uint64_t mask = tb__mask(n->dt.data); - - if (aa != &TOP_IN_THE_SKY && bb->tag == LATTICE_INT && bb->_int.min == bb->_int.max) { - uint32_t src = aa->_int.known_zeros; - uint32_t chopped = ~bb->_int.min & mask; - - // if the known zeros is more than those chopped then the mask is useless - if ((src & chopped) == chopped) { - return n->inputs[1]; - } - } - } - - uint64_t b; - if (!get_int_const(n->inputs[2], &b)) { - return n; - } - - if (n->type == TB_MUL && b == 1) { - return n->inputs[1]; - } else if (b == 0) { - switch (n->type) { - default: return n; - - case TB_SHL: - case TB_SHR: - case TB_SAR: - case TB_ADD: - case TB_SUB: - case TB_XOR: - return n->inputs[1]; - - case TB_MUL: - return n->inputs[0]; - - case TB_UDIV: - case TB_SDIV: - case TB_UMOD: - case TB_SMOD: - return make_poison(f, n->dt); - - // (cmp.ne a 0) => a - case TB_CMP_NE: { - // walk up extension - TB_Node* src = n->inputs[1]; - if (src->type == TB_ZERO_EXT || src->type == TB_SIGN_EXT) { - src = src->inputs[1]; - } - - if (src->dt.type == TB_INT && src->dt.data == 1) { - return src; - } - - return n; - } - } - } else { - return n; - } -} - -//////////////////////////////// -// Pointer idealizations -//////////////////////////////// -static TB_Node* identity_member_ptr(TB_Passes* restrict opt, TB_Function* f, TB_Node* n) { - if (TB_NODE_GET_EXTRA_T(n, TB_NodeMember)->offset == 0) { - return n->inputs[1]; - } - return n; -} - -static TB_Node* ideal_member_ptr(TB_Passes* restrict opt, TB_Function* f, TB_Node* n) { - int64_t offset = TB_NODE_GET_EXTRA_T(n, TB_NodeMember)->offset; - TB_Node* base = n->inputs[1]; - - if (base->type == TB_MEMBER_ACCESS) { - offset += TB_NODE_GET_EXTRA_T(base, TB_NodeMember)->offset; - set_input(f, n, base->inputs[1], 1); - - TB_NODE_SET_EXTRA(n, TB_NodeMember, .offset = offset); - return n; - } - - return NULL; -} - -static TB_Node* ideal_array_ptr(TB_Passes* restrict opt, TB_Function* f, TB_Node* n) { - int64_t stride = TB_NODE_GET_EXTRA_T(n, TB_NodeArray)->stride; - TB_Node* base = n->inputs[1]; - TB_Node* index = n->inputs[2]; - - // (array A B 4) => (member A B*4) where B is constant - if (index->type == TB_INTEGER_CONST) { - int64_t src_i = TB_NODE_GET_EXTRA_T(index, TB_NodeInt)->value; - - int64_t offset = src_i * stride; - TB_Node* new_n = tb_alloc_node(f, TB_MEMBER_ACCESS, n->dt, 2, sizeof(TB_NodeMember)); - set_input(f, new_n, base, 1); - TB_NODE_SET_EXTRA(new_n, TB_NodeMember, .offset = offset); - return new_n; - } - - // (array A (shl B C) D) => (array A B C<type == TB_SHL && index->inputs[2]->type == TB_INTEGER_CONST) { - uint64_t scale = TB_NODE_GET_EXTRA_T(index->inputs[2], TB_NodeInt)->value; - set_input(f, n, index->inputs[1], 2); - TB_NODE_SET_EXTRA(n, TB_NodeArray, .stride = stride << scale); - return n; - } - - // (array A (mul B C) D) => (array A B C*D) - if (index->type == TB_MUL && index->inputs[2]->type == TB_INTEGER_CONST) { - uint64_t factor = TB_NODE_GET_EXTRA_T(index->inputs[2], TB_NodeInt)->value; - set_input(f, n, index->inputs[1], 2); - TB_NODE_SET_EXTRA(n, TB_NodeArray, .stride = stride * factor); - return n; - } - - if (index->type == TB_ADD) { - TB_Node* new_index = index->inputs[1]; - TB_Node* add_rhs = index->inputs[2]; - - uint64_t offset; - if (get_int_const(add_rhs, &offset)) { - // (array A (add B C) D) => (member (array A B D) C*D) - offset *= stride; - - TB_Node* new_n = tb_alloc_node(f, TB_ARRAY_ACCESS, TB_TYPE_PTR, 3, sizeof(TB_NodeArray)); - set_input(f, new_n, base, 1); - set_input(f, new_n, new_index, 2); - TB_NODE_SET_EXTRA(new_n, TB_NodeArray, .stride = stride); - - TB_Node* new_member = tb_alloc_node(f, TB_MEMBER_ACCESS, TB_TYPE_PTR, 2, sizeof(TB_NodeMember)); - set_input(f, new_member, new_n, 1); - TB_NODE_SET_EXTRA(new_member, TB_NodeMember, .offset = offset); - - tb_pass_mark(opt, new_n); - tb_pass_mark(opt, new_member); - return new_member; - } else if (add_rhs->type == TB_SHL && add_rhs->inputs[2]->type == TB_INTEGER_CONST) { - // (array A (add B (shl C D)) E) => (array (array A B 1<inputs[1]; - uint64_t amt = 1ull << TB_NODE_GET_EXTRA_T(n, TB_NodeInt)->value; - - TB_Node* new_n = tb_alloc_node(f, TB_ARRAY_ACCESS, TB_TYPE_PTR, 3, sizeof(TB_NodeArray)); - set_input(f, new_n, base, 1); - set_input(f, new_n, second_index, 2); - TB_NODE_SET_EXTRA(new_n, TB_NodeArray, .stride = amt); - - tb_pass_mark(opt, new_n); - set_input(f, n, new_n, 1); - set_input(f, n, new_index, 2); - return n; - } - } - - return NULL; -} diff --git a/vendor/tb/src/opt/gcm.h b/vendor/tb/src/opt/gcm.h deleted file mode 100644 index 5cbda91c..00000000 --- a/vendor/tb/src/opt/gcm.h +++ /dev/null @@ -1,290 +0,0 @@ -#include - -// Scheduling: "Global Code Motion Global Value Numbering", Cliff Click 1995 -// https://courses.cs.washington.edu/courses/cse501/06wi/reading/click-pldi95.pdf -typedef struct Elem { - struct Elem* parent; - TB_ArenaSavepoint sp; - TB_Node* n; - int i; -} Elem; - -static bool swag_to_hoist(TB_Node* n) { - return n->type == TB_LOAD; -} - -//////////////////////////////// -// Late scheduling -//////////////////////////////// -// schedule nodes such that they appear the least common -// ancestor to all their users -static TB_BasicBlock* find_lca(TB_Passes* p, TB_BasicBlock* a, TB_BasicBlock* b) { - if (a == NULL) return b; - - // line both up - while (a->dom_depth > b->dom_depth) a = a->dom; - while (b->dom_depth > a->dom_depth) b = b->dom; - - while (a != b) { - b = b->dom; - a = a->dom; - } - - return a; -} - -// writes out to the p->scheduled array, it's at the top of the tmp_arena -void tb_pass_schedule(TB_Passes* p, TB_CFG cfg, bool renumber) { - assert(p->scheduled == NULL && "make sure when you're done with the schedule, you throw away the old one"); - - CUIK_TIMED_BLOCK("schedule") { - TB_Function* f = p->f; - Worklist* restrict ws = &p->worklist; - - // arraychads stay up - p->scheduled = tb_arena_alloc(tmp_arena, f->node_count * sizeof(TB_BasicBlock*)); - memset(p->scheduled, 0, f->node_count * sizeof(TB_BasicBlock*)); - - TB_ArenaSavepoint sp = tb_arena_save(tmp_arena); - TB_Node** saved = tb_arena_alloc(tmp_arena, cfg.block_count * sizeof(TB_Node*)); - memcpy(saved, &ws->items[0], cfg.block_count * sizeof(TB_Node*)); - - CUIK_TIMED_BLOCK("dominators") { - // jarvis pull up the dommies - tb_compute_dominators(p->f, p, cfg); - - FOREACH_N(i, 0, cfg.block_count) { - TB_Node* n = ws->items[i]; - TB_BasicBlock* bb = &nl_map_get_checked(cfg.node_to_block, n); - - bb->items = nl_hashset_alloc(32); - nl_hashset_put(&bb->items, n); - p->scheduled[n->gvn] = bb; - } - - worklist_clear_visited(ws); - } - - TB_BasicBlock* start_bb = p->scheduled[p->worklist.items[0]->gvn]; - NL_ChunkedArr pins = nl_chunked_arr_alloc(tmp_arena); - - CUIK_TIMED_BLOCK("pinned schedule") { - // schedule root's users - { - TB_Node* root = f->root_node; - FOR_USERS(u, root) { - TB_Node* out = u->n; - if (!worklist_test_n_set(ws, out)) { - dyn_array_put(ws->items, out); - } - } - } - - for (size_t i = cfg.block_count + 1; i < dyn_array_length(ws->items); i++) { - TB_Node* n = ws->items[i]; - - if (is_pinned(n)) { - // a region might refer to itself, but a node within a - // BB will refer to it's parent (who should've been scheduled - // by now) - TB_BasicBlock* bb = NULL; - if (n->type == TB_PROJ && n->inputs[0]->type == TB_ROOT) { - bb = start_bb; - } else if (n->type != TB_ROOT) { - TB_Node* curr = n; - do { - bb = p->scheduled[curr->gvn]; - curr = curr->inputs[0]; - if (curr == NULL || curr->type == TB_ROOT) { - break; - } - } while (!bb); - } - - if (bb) { - nl_hashset_put(&bb->items, n); - p->scheduled[n->gvn] = bb; - nl_chunked_arr_put(&pins, n); - - DO_IF(TB_OPTDEBUG_GCM)(printf("%s: v%u pinned to .bb%d\n", f->super.name, n->gvn, bb->id)); - } - } - - FOR_USERS(u, n) { - TB_Node* out = u->n; - if (!worklist_test_n_set(ws, out)) { - dyn_array_put(ws->items, out); - } - } - } - - DO_IF(TB_OPTDEBUG_GCM)(printf("%s: scheduled %zu nodes (%zu recorded in the graph)\n", f->super.name, dyn_array_length(ws->items) - cfg.block_count, f->node_count)); - - if (renumber) { - // TODO(NeGate): reorder lattice universe - tb_todo(); - - f->node_count = dyn_array_length(ws->items) - cfg.block_count; - - FOREACH_N(i, cfg.block_count, dyn_array_length(ws->items)) { - // we'll reassign IDs here such that we have an easier - // time with node count based tables (quite a few in codegen). - ws->items[i]->gvn = i; - } - } - } - - CUIK_TIMED_BLOCK("early schedule") { - nl_chunked_arr_trim(&pins); - - // we're gonna use this space to store the DFS order, we'll walk it in reverse for - // late sched - worklist_clear_visited(ws); - dyn_array_set_length(ws->items, cfg.block_count); - - for (NL_ArrChunk* restrict chk = pins.first; chk; chk = chk->next) { - FOREACH_N(i, 0, chk->count) { - TB_Node* pin_n = chk->elems[i]; - - TB_ArenaSavepoint sp = tb_arena_save(tmp_arena); - Elem* top = tb_arena_alloc(tmp_arena, sizeof(Elem)); - top->parent = NULL; - top->sp = sp; - top->n = pin_n; - top->i = pin_n->input_count; - - // DFS nodes by inputs - while (top) { - TB_Node* n = top->n; - - if (top->i > 0) { - // push next unvisited in - TB_Node* in = n->inputs[--top->i]; - - // pinned nodes can't be rescheduled - if (in && !is_pinned(in) && !worklist_test_n_set(ws, in)) { - TB_ArenaSavepoint sp = tb_arena_save(tmp_arena); - Elem* new_top = tb_arena_alloc(tmp_arena, sizeof(Elem)); - new_top->parent = top; - new_top->sp = sp; - new_top->n = in; - new_top->i = in->input_count; - top = new_top; - } - continue; - } - - if (n != pin_n) { // only pinned node in the stack - // start at the entry point - int best_depth = 0; - TB_BasicBlock* best = start_bb; - - // choose deepest block - FOREACH_N(i, 0, n->input_count) if (n->inputs[i]) { - if (n->inputs[i]->type == TB_ROOT) { - DO_IF(TB_OPTDEBUG_GCM)(printf(" in v%u @ bb0\n", n->inputs[i]->gvn)); - continue; - } - - TB_BasicBlock* bb = p->scheduled[n->inputs[i]->gvn]; - if (bb == NULL) { - // input has no scheduling... weird? - DO_IF(TB_OPTDEBUG_GCM)(printf(" in v%u @ dead\n", n->inputs[i]->gvn)); - continue; - } - - DO_IF(TB_OPTDEBUG_GCM)(printf(" in v%u @ bb%d\n", n->inputs[i]->gvn, bb->id)); - if (best_depth < bb->dom_depth) { - best_depth = bb->dom_depth; - best = bb; - } - } - - DO_IF(TB_OPTDEBUG_GCM)(printf("%s: v%u into .bb%d\n", p->f->super.name, n->gvn, best->id)); - - p->scheduled[n->gvn] = best; - nl_hashset_put(&best->items, n); - dyn_array_put(ws->items, n); - } - - struct Elem* parent = top->parent; - tb_arena_restore(tmp_arena, top->sp); - top = parent; - } - } - } - } - - // move nodes closer to their usage site - CUIK_TIMED_BLOCK("late schedule") { - FOREACH_REVERSE_N(i, cfg.block_count, dyn_array_length(ws->items)) { - TB_Node* n = ws->items[i]; - - DO_IF(TB_OPTDEBUG_GCM)(printf("%s: try late v%u\n", p->f->super.name, n->gvn)); - - // we're gonna find the least common ancestor - TB_BasicBlock* lca = NULL; - FOR_USERS(use, n) { - TB_Node* y = use->n; - TB_BasicBlock* use_block = p->scheduled[y->gvn]; - if (use_block == NULL) { continue; } // dead - - DO_IF(TB_OPTDEBUG_GCM)(printf(" user v%u @ bb%d\n", y->gvn, use_block->id)); - if (y->type == TB_PHI) { - TB_Node* use_node = y->inputs[0]; - assert(cfg_is_region(use_node)); - - if (y->input_count != use_node->input_count + 1) { - tb_panic("phi has parent with mismatched predecessors"); - } - - ptrdiff_t j = 1; - for (; j < y->input_count; j++) { - if (y->inputs[j] == n) { - break; - } - } - assert(j >= 0); - - TB_BasicBlock* bb = p->scheduled[use_node->inputs[j - 1]->gvn]; - if (bb) { use_block = bb; } - } - - lca = find_lca(p, lca, use_block); - } - - if (lca != NULL) { - TB_BasicBlock* old = p->scheduled[n->gvn]; - // i dont think it should be possible to schedule something here - // which didn't already get scheduled in EARLY - assert(old && "huh?"); - - // replace old BB entry, also if old is a natural loop we might - // be better off hoisting the values if possible. - if (old != lca && lca->dom_depth > old->dom_depth) { - // some ops deserve hoisting more than others (cough cough loads) - if (!swag_to_hoist(n) || !cfg_is_natural_loop(old->start)) { - TB_OPTDEBUG(GCM)( - printf(" LATE v%u into .bb%d: ", n->gvn, lca->id), - print_node_sexpr(n, 0), - printf("\n") - ); - - p->scheduled[n->gvn] = lca; - nl_hashset_remove(&old->items, n); - nl_hashset_put(&lca->items, n); - } - } - } - } - } - - CUIK_TIMED_BLOCK("copy CFG back in") { - memcpy(ws->items, saved, cfg.block_count * sizeof(TB_Node*)); - dyn_array_set_length(ws->items, cfg.block_count); - - worklist_clear_visited(ws); - tb_arena_restore(tmp_arena, sp); - } - } -} diff --git a/vendor/tb/src/opt/gvn.h b/vendor/tb/src/opt/gvn.h deleted file mode 100644 index 629a2740..00000000 --- a/vendor/tb/src/opt/gvn.h +++ /dev/null @@ -1,190 +0,0 @@ -#include "../tb_internal.h" - -static size_t extra_bytes(TB_Node* n) { - switch (n->type) { - case TB_INTEGER_CONST: return sizeof(TB_NodeInt); - case TB_FLOAT32_CONST: return sizeof(TB_NodeFloat32); - case TB_FLOAT64_CONST: return sizeof(TB_NodeFloat64); - case TB_SYMBOL: return sizeof(TB_NodeSymbol); - case TB_LOCAL: return sizeof(TB_NodeLocal); - - case TB_LOOKUP: { - TB_NodeLookup* l = TB_NODE_GET_EXTRA(n); - return sizeof(TB_NodeLookup) + (l->entry_count * sizeof(TB_LookupEntry)); - } - - case TB_BRANCH: { - TB_NodeBranch* br = TB_NODE_GET_EXTRA(n); - return sizeof(TB_NodeBranch) + ((br->succ_count - 1) * sizeof(int64_t)); - } - - case TB_SAFEPOINT_POLL: - return sizeof(TB_NodeSafepoint); - - case TB_AND: - case TB_OR: - case TB_XOR: - case TB_ADD: - case TB_SUB: - case TB_MUL: - case TB_SHL: - case TB_SHR: - case TB_SAR: - case TB_ROL: - case TB_ROR: - case TB_UDIV: - case TB_SDIV: - case TB_UMOD: - case TB_SMOD: - return sizeof(TB_NodeBinopInt); - - case TB_MEMBER_ACCESS: - return sizeof(TB_NodeMember); - - case TB_ARRAY_ACCESS: - return sizeof(TB_NodeArray); - - case TB_CALLGRAPH: - case TB_TRUNCATE: - case TB_UINT2FLOAT: - case TB_FLOAT2UINT: - case TB_INT2FLOAT: - case TB_FLOAT2INT: - case TB_FLOAT_EXT: - case TB_SIGN_EXT: - case TB_ZERO_EXT: - case TB_BITCAST: - case TB_FADD: - case TB_FSUB: - case TB_FMUL: - case TB_FDIV: - case TB_FMAX: - case TB_FMIN: - case TB_NEG: - case TB_NOT: - case TB_PHI: - case TB_CLZ: - case TB_CTZ: - case TB_ADC: - case TB_VA_START: - case TB_POISON: - case TB_SELECT: - case TB_MERGEMEM: - case TB_DEAD: - case TB_NULL: - case TB_UNREACHABLE: - case TB_DEBUGBREAK: - case TB_CYCLE_COUNTER: - case TB_MULPAIR: - case TB_READ: - case TB_WRITE: - case TB_ROOT: - case TB_RETURN: - return 0; - - case TB_SPLITMEM: - return sizeof(TB_NodeMemSplit); - - case TB_REGION: - case TB_NATURAL_LOOP: - case TB_AFFINE_LOOP: - return sizeof(TB_NodeRegion); - - case TB_CALL: - case TB_SYSCALL: - return sizeof(TB_NodeCall); - - case TB_TAILCALL: - return sizeof(TB_NodeTailcall); - - case TB_LOAD: - case TB_STORE: - case TB_MEMCPY: - case TB_MEMSET: - return sizeof(TB_NodeMemAccess); - - case TB_ATOMIC_LOAD: - case TB_ATOMIC_XCHG: - case TB_ATOMIC_ADD: - case TB_ATOMIC_SUB: - case TB_ATOMIC_AND: - case TB_ATOMIC_XOR: - case TB_ATOMIC_OR: - case TB_ATOMIC_CAS: - return sizeof(TB_NodeAtomic); - - case TB_CMP_EQ: - case TB_CMP_NE: - case TB_CMP_ULT: - case TB_CMP_ULE: - case TB_CMP_SLT: - case TB_CMP_SLE: - case TB_CMP_FLT: - case TB_CMP_FLE: - return sizeof(TB_NodeCompare); - - case TB_PREFETCH: - return sizeof(TB_NodePrefetch); - - case TB_PROJ: - return sizeof(TB_NodeProj); - - default: tb_todo(); - } -} - -uint32_t gvn_hash(void* a) { - uint32_t h; - CUIK_TIMED_BLOCK("hash") { - TB_Node* n = a; - size_t extra = extra_bytes(n); - h = n->type + n->dt.raw + n->input_count + extra; - - if (n->type == TB_LOCAL) { - h += (uintptr_t) n->gvn; - } - - FOREACH_N(i, 0, n->input_count) { - h += n->inputs[i] ? n->inputs[i]->gvn : 0; - } - - // fnv1a the extra space - uint32_t* extra_arr = (uint32_t*) n->extra; - FOREACH_N(i, 0, extra / 4) { - h += extra_arr[i]; - } - - FOREACH_N(i, extra & ~0x7, extra) { - h += n->extra[i]; - } - - // fib hashing amirite - h = ((uint64_t) h * 11400714819323198485llu) >> 32llu; - } - - return h; -} - -bool gvn_compare(void* a, void* b) { - TB_Node *x = a, *y = b; - - // early outs - if (x->type != y->type || x->input_count != y->input_count || x->dt.raw != y->dt.raw) { - return false; - } - - // current exception to the GVN rule - if (x->type == TB_LOCAL) { - return x == y; - } - - // match up inputs - FOREACH_N(i, 0, x->input_count) { - if (x->inputs[i] != y->inputs[i]) { - return false; - } - } - - size_t extra = extra_bytes(x); - return extra == 0 || memcmp(x->extra, y->extra, extra) == 0; -} diff --git a/vendor/tb/src/opt/lattice.h b/vendor/tb/src/opt/lattice.h deleted file mode 100644 index da4fcb62..00000000 --- a/vendor/tb/src/opt/lattice.h +++ /dev/null @@ -1,362 +0,0 @@ -#include - -static Lattice TOP_IN_THE_SKY = { LATTICE_TOP }; -static Lattice BOT_IN_THE_SKY = { LATTICE_BOT }; -static Lattice CTRL_IN_THE_SKY = { LATTICE_CTRL }; -static Lattice XCTRL_IN_THE_SKY = { LATTICE_XCTRL }; -static Lattice XNULL_IN_THE_SKY = { LATTICE_XNULL }; -static Lattice NULL_IN_THE_SKY = { LATTICE_NULL }; -static Lattice PTR_IN_THE_SKY = { LATTICE_BOTPTR }; -static Lattice FALSE_IN_THE_SKY = { LATTICE_INT, ._int = { 0, 0, 1, 0 } }; -static Lattice TRUE_IN_THE_SKY = { LATTICE_INT, ._int = { 1, 1, 0, 1 } }; - -static Lattice* lattice_from_dt(TB_Passes* p, TB_DataType dt); - -static uint32_t lattice_hash(void* a) { - size_t s = sizeof(Lattice); - Lattice* l = a; - if (l->tag == LATTICE_TUPLE) { - s += l->_tuple.count*sizeof(Lattice*); - } - - return tb__murmur3_32(a, s); -} - -static bool lattice_cmp(void* a, void* b) { - Lattice *aa = a, *bb = b; - if (aa->tag != bb->tag) { - return false; - } - - if (aa->tag == LATTICE_TUPLE) { - if (aa->_tuple.count != bb->_tuple.count) { - return false; - } - - return memcmp(aa, bb, sizeof(Lattice) + aa->_tuple.count*sizeof(Lattice*)) == 0; - } else { - return memcmp(aa, bb, sizeof(Lattice)) == 0; - } -} - -static bool lattice_is_const_int(Lattice* l) { return l->_int.min == l->_int.max; } -static bool lattice_is_const(Lattice* l) { return l->tag == LATTICE_INT && l->_int.min == l->_int.max; } - -static void lattice_universe_grow(TB_Passes* p, size_t top) { - size_t new_cap = tb_next_pow2(top + 16); - p->types = tb_platform_heap_realloc(p->types, new_cap * sizeof(Lattice*)); - - // clear new space - FOREACH_N(i, p->type_cap, new_cap) { - p->types[i] = NULL; - } - - p->type_cap = new_cap; -} - -static bool lattice_universe_map_progress(TB_Passes* p, TB_Node* n, Lattice* l) { - // reserve cap, slow path :p - if (UNLIKELY(n->gvn >= p->type_cap)) { - lattice_universe_grow(p, n->gvn); - } - - Lattice* old = p->types[n->gvn]; - p->types[n->gvn] = l; - return old != l; -} - -static void lattice_universe_map(TB_Passes* p, TB_Node* n, Lattice* l) { - // reserve cap, slow path :p - if (UNLIKELY(n->gvn >= p->type_cap)) { - lattice_universe_grow(p, n->gvn); - } - - p->types[n->gvn] = l; -} - -static Lattice* lattice_intern(TB_Passes* p, Lattice l) { - assert(l.tag != LATTICE_TUPLE); - Lattice* k = nl_hashset_get2(&p->type_interner, &l, lattice_hash, lattice_cmp); - if (k != NULL) { - return k; - } - - // allocate new node - k = tb_arena_alloc(tmp_arena, sizeof(Lattice)); - memcpy(k, &l, sizeof(l)); - nl_hashset_put2(&p->type_interner, k, lattice_hash, lattice_cmp); - return k; -} - -static bool lattice_top_or_bot(Lattice* l) { - return l->tag <= LATTICE_TOP; -} - -LatticeTrifecta lattice_truthy(Lattice* l) { - switch (l->tag) { - case LATTICE_INT: - if (l->_int.min == l->_int.max) { - return l->_int.min ? LATTICE_KNOWN_TRUE : LATTICE_KNOWN_FALSE; - } - return LATTICE_UNKNOWN; - - case LATTICE_FLOAT32: - case LATTICE_FLOAT64: - return LATTICE_UNKNOWN; - - case LATTICE_NULL: return false; - case LATTICE_XNULL: return true; - - default: - return LATTICE_UNKNOWN; - } -} - -static int64_t lattice_int_min(int bits) { return 1ll << (bits - 1); } -static int64_t lattice_int_max(int bits) { return (1ll << (bits - 1)) - 1; } -static uint64_t lattice_uint_max(int bits) { return UINT64_MAX >> (64 - bits); } - -static Lattice* lattice_from_dt(TB_Passes* p, TB_DataType dt) { - switch (dt.type) { - case TB_INT: { - assert(dt.data <= 64); - if (dt.data == 0) { - return &BOT_IN_THE_SKY; - } - - return lattice_intern(p, (Lattice){ LATTICE_INT, ._int = { 0, lattice_uint_max(dt.data) } }); - } - - case TB_FLOAT: { - assert(dt.data == TB_FLT_32 || dt.data == TB_FLT_64); - return lattice_intern(p, (Lattice){ dt.data == TB_FLT_64 ? LATTICE_FLOAT64 : LATTICE_FLOAT32, ._float = { LATTICE_UNKNOWN } }); - } - - case TB_PTR: { - return &PTR_IN_THE_SKY; - } - - case TB_MEMORY: { - return p->root_mem; - } - - case TB_CONTROL: return &CTRL_IN_THE_SKY; - default: return &BOT_IN_THE_SKY; - } -} - -Lattice* lattice_universe_get(TB_Passes* p, TB_Node* n) { - // reserve cap, slow path :p - if (UNLIKELY(n->gvn >= p->type_cap)) { - lattice_universe_grow(p, n->gvn); - } - - if (p->types[n->gvn] == NULL) { - p->types[n->gvn] = lattice_from_dt(p, n->dt); - } - - return p->types[n->gvn]; -} - -static Lattice* lattice_tuple_from_node(TB_Passes* p, TB_Node* n) { - assert(n->dt.type == TB_TUPLE); - // count projs - int projs = 0; - FOR_USERS(u, n) { - if (u->n->type == TB_PROJ) projs++; - } - - size_t size = sizeof(Lattice) + projs*sizeof(Lattice*); - Lattice* l = tb_arena_alloc(tmp_arena, size); - *l = (Lattice){ LATTICE_TUPLE, ._tuple = { projs } }; - FOR_USERS(u, n) { - if (u->n->type != TB_PROJ) continue; - - int index = TB_NODE_GET_EXTRA_T(u->n, TB_NodeProj)->index; - l->elems[index] = lattice_from_dt(p, u->n->dt); - } - - Lattice* k = nl_hashset_put2(&p->type_interner, l, lattice_hash, lattice_cmp); - if (k) { - tb_arena_free(tmp_arena, l, size); - return k; - } else { - return l; - } -} - -// known X ^ known X => known X or -// known X ^ unknown => unknown (commutative btw) -#define TRIFECTA_MEET(a, b) ((a).trifecta == (b).trifecta ? (a).trifecta : LATTICE_UNKNOWN) - -#define MASK_UPTO(pos) (UINT64_MAX >> (64 - pos)) -#define BEXTR(src,pos) (((src) >> (pos)) & 1) -uint64_t tb__sxt(uint64_t src, uint64_t src_bits, uint64_t dst_bits) { - uint64_t sign_bit = BEXTR(src, src_bits-1); - uint64_t mask = MASK_UPTO(dst_bits) & ~MASK_UPTO(src_bits); - - uint64_t dst = src & ~mask; - return dst | (sign_bit ? mask : 0); -} - -static bool lattice_signed(LatticeInt* l) { return l->min > l->max; } - -static LatticeInt lattice_into_unsigned(LatticeInt i, int bits) { - if (i.min > i.max) { - i.min = 0; - i.max = lattice_uint_max(bits); - return i; - } else { - return i; - } -} - -static Lattice* lattice_gimme_int(TB_Passes* p, int64_t min, int64_t max) { - assert(min <= max); - return lattice_intern(p, (Lattice){ LATTICE_INT, ._int = { min, max } }); -} - -static Lattice* lattice_gimme_uint(TB_Passes* p, uint64_t min, uint64_t max) { - assert(min <= max); - return lattice_intern(p, (Lattice){ LATTICE_INT, ._int = { min, max } }); -} - -static Lattice* lattice_new_alias(TB_Passes* p) { - return lattice_intern(p, (Lattice){ LATTICE_MEM, ._mem = { p->alias_n++ } }); -} - -static Lattice* lattice_alias(TB_Passes* p, int alias_idx) { - return lattice_intern(p, (Lattice){ LATTICE_MEM, ._mem = { alias_idx } }); -} - -static bool l_add_overflow(uint64_t x, uint64_t y, uint64_t mask, uint64_t* out) { - *out = (x + y) & mask; - return x && *out < x; -} - -static bool l_mul_overflow(uint64_t x, uint64_t y, uint64_t mask, uint64_t* out) { - *out = (x * y) & mask; - return x && *out < x; -} - -static bool l_sub_overflow(uint64_t x, uint64_t y, uint64_t mask, uint64_t* out) { - *out = (x - y) & mask; - return x && *out > x; -} - -static bool wrapped_int_lt(int64_t x, int64_t y, int bits) { - return (int64_t)tb__sxt(x, bits, 64) < (int64_t)tb__sxt(y, bits, 64); -} - -static LatticeInt lattice_meet_int(LatticeInt a, LatticeInt b, TB_DataType dt) { - // [amin, amax] ^ [bmin, bmax] => [min(amin, bmin), max(amax, bmax)] - int bits = dt.data; - uint64_t mask = tb__mask(dt.data); - - bool aas = a.min > a.max; - bool bbs = b.min > b.max; - if (aas && bbs) { - if (wrapped_int_lt(b.min, a.min, bits)) a.min = b.min; - if (wrapped_int_lt(a.max, b.max, bits)) a.max = b.max; - } else { - if (aas) a = lattice_into_unsigned(a, bits); - if (bbs) b = lattice_into_unsigned(b, bits); - - if (b.min < a.min) a.min = b.min; - if (a.max < b.max) a.max = b.max; - } - - a.known_zeros &= b.known_zeros; - a.known_ones &= b.known_ones; - return a; -} - -static Lattice* lattice_dual(TB_Passes* p, Lattice* type) { - switch (type->tag) { - case LATTICE_BOT: { - return &TOP_IN_THE_SKY; - } - - case LATTICE_INT: { - LatticeInt i = type->_int; - return lattice_intern(p, (Lattice){ LATTICE_INT, ._int = { i.max, i.min, i.known_ones, i.known_zeros } }); - } - - default: - return type; - } -} - -// generates the greatest lower bound between a and b -static Lattice* lattice_meet(TB_Passes* p, Lattice* a, Lattice* b, TB_DataType dt) { - // a ^ a = a - if (a == b) return a; - - // it's commutative, so let's simplify later code this way. - // now a will always be smaller tag than b. - if (a->tag > b->tag) { - SWAP(Lattice*, a, b); - } - - switch (a->tag) { - case LATTICE_BOT: return &BOT_IN_THE_SKY; - case LATTICE_TOP: return b; - - case LATTICE_INT: { - if (b->tag != LATTICE_INT) { - return &BOT_IN_THE_SKY; - } - - LatticeInt i = lattice_meet_int(a->_int, b->_int, dt); - return lattice_intern(p, (Lattice){ LATTICE_INT, ._int = i }); - } - - case LATTICE_FLOAT32: - case LATTICE_FLOAT64: { - if (b->tag != a->tag) { - return &BOT_IN_THE_SKY; - } - - LatticeFloat f = { .trifecta = TRIFECTA_MEET(a->_float, b->_float) }; - return lattice_intern(p, (Lattice){ a->tag, ._float = f }); - } - - // all cases that reached down here are bottoms - case LATTICE_BOTPTR: - case LATTICE_NULL: - return &PTR_IN_THE_SKY; - - // ~null ^ sym = ~null - case LATTICE_XNULL: { - if (b->tag == LATTICE_PTRCON) { - return a; - } else { - return &BOT_IN_THE_SKY; - } - } - - // symA ^ symB = ~null - case LATTICE_PTRCON: { - if (b->tag == LATTICE_PTRCON) { - assert(a->_ptr.sym != b->_ptr.sym); - return &XNULL_IN_THE_SKY; - } else { - return &BOT_IN_THE_SKY; - } - } - - case LATTICE_CTRL: - case LATTICE_XCTRL: { - // ctrl ^ ctrl = ctrl - // ctrl ^ xctrl = bot - // xctrl ^ xctrl = xctrl - return a == b ? a : &BOT_IN_THE_SKY; - } - - // if we make it here, they're not the same mem - case LATTICE_MEM: - return &BOT_IN_THE_SKY; - - default: tb_todo(); - } -} diff --git a/vendor/tb/src/opt/legalizer.h b/vendor/tb/src/opt/legalizer.h deleted file mode 100644 index 59f91f2e..00000000 --- a/vendor/tb/src/opt/legalizer.h +++ /dev/null @@ -1,171 +0,0 @@ - -typedef struct { - TB_DataType dt; - uint16_t size; - uint16_t splits; -} LegalType; - -typedef struct LegalizeCtx LegalizeCtx; -typedef bool (*LegalizeFn)(LegalizeCtx* ctx, TB_DataType dt, LegalType* out); - -struct LegalizeCtx { - TB_Arch arch; - - TB_Function* f; - TB_Passes* p; - - Worklist* ws; - LegalizeFn fn; -}; - -static bool legalize_64bit_machine(LegalizeCtx* ctx, TB_DataType dt, LegalType* out) { - // if it's not 1,8,16,32 or 64 bit we need to legalize - if (dt.type == TB_INT && dt.data != 1 && dt.data != 8 && dt.data != 16 && dt.data != 32 && dt.data != 64) { - out->dt = TB_TYPE_I64; - out->size = 8; - out->splits = (dt.data + 63) / 64; - return true; - } - - return false; -} - -static bool legalize_32bit_machine(LegalizeCtx* ctx, TB_DataType dt, LegalType* out) { - // if it's not 1,8,16 or 32 bit we need to legalize - if (dt.type == TB_INT && dt.data != 1 && dt.data != 8 && dt.data != 16 && dt.data != 32) { - out->dt = TB_TYPE_I32; - out->size = 4; - out->splits = (dt.data + 31) / 32; - return true; - } - - return false; -} - -static TB_Node* make_member(TB_Function* f, TB_Node* src, int32_t offset) { - if (offset == 0) { return src; } - - TB_Node* n = tb_alloc_node(f, TB_MEMBER_ACCESS, TB_TYPE_PTR, 2, sizeof(TB_NodeMember)); - set_input(f, n, src, 1); - TB_NODE_SET_EXTRA(n, TB_NodeMember, .offset = offset); - return tb__gvn(f, n, 0); -} - -static TB_Node** legalize_node(LegalizeCtx* ctx, TB_Node* n, LegalType valid) { - TB_ArenaSavepoint sp = tb_arena_save(tmp_arena); - TB_Node** pieces = tb_arena_alloc(tmp_arena, valid.splits * sizeof(TB_Node*)); - - TB_Function* f = ctx->f; - if (n->type == TB_INTEGER_CONST) { - uint64_t imm = TB_NODE_GET_EXTRA_T(n, TB_NodeInt)->value; - uint64_t mask = tb__mask(valid.size*8); - - FOREACH_N(i, 0, valid.splits) { - pieces[i] = make_int_node(f, ctx->p, valid.dt, imm & mask); - imm >>= valid.size*8; - } - } else if (n->type == TB_LOAD) { - int32_t align = TB_NODE_GET_EXTRA_T(n, TB_NodeMemAccess)->align; - if (align < valid.size) { align = valid.size; } - - int32_t off = 0; - FOREACH_N(i, 0, valid.splits) { - TB_Node* addr = make_member(f, n->inputs[2], off); - off += valid.size; - - // make split load - TB_Node* ld = tb_alloc_node(f, TB_LOAD, valid.dt, 3, sizeof(TB_NodeMemAccess)); - set_input(f, ld, n->inputs[0], 0); - set_input(f, ld, n->inputs[1], 1); - set_input(f, ld, addr, 2); - TB_NODE_SET_EXTRA(ld, TB_NodeMemAccess, .align = align); - - pieces[i] = ld; - } - } else if (n->type == TB_ADD) { - TB_Node** a = legalize_node(ctx, n->inputs[1], valid); - TB_Node** b = legalize_node(ctx, n->inputs[2], valid); - - TB_Node* carry = NULL; - FOREACH_N(i, 0, valid.splits) { - TB_Node* n = tb_alloc_node(f, TB_ADC, TB_TYPE_TUPLE, 4, 0); - set_input(f, n, a[i], 1); - set_input(f, n, b[i], 2); - set_input(f, n, carry, 3); - - pieces[i] = make_proj_node(f, valid.dt, n, 0); - carry = make_proj_node(f, TB_TYPE_BOOL, n, 1); - } - } else { - tb_todo(); - } - return pieces; -} - -static TB_Node* walk_node(LegalizeCtx* ctx, TB_Node* n) { - if (worklist_test_n_set(ctx->ws, n)) { - return n; - } - - TB_Function* f = ctx->f; - - // let's find ops we need to scale up - LegalType valid; - if (n->type == TB_STORE && ctx->fn(ctx, n->inputs[3]->dt, &valid)) { - // split store - TB_Node** op = legalize_node(ctx, n->inputs[3], valid); - - int32_t align = TB_NODE_GET_EXTRA_T(n, TB_NodeMemAccess)->align; - if (align < valid.size) { align = valid.size; } - - TB_Node* prev_st = n->inputs[1]; - int32_t off = 0; - FOREACH_N(i, 0, valid.splits - 1) { - TB_Node* addr = make_member(f, n->inputs[2], off); - off += valid.size; - - // make split stores - TB_Node* st = tb_alloc_node(f, TB_STORE, TB_TYPE_MEMORY, 4, sizeof(TB_NodeMemAccess)); - set_input(f, st, n->inputs[0], 0); - set_input(f, st, prev_st, 1); - set_input(f, st, addr, 2); - set_input(f, st, op[i], 3); - TB_NODE_SET_EXTRA(st, TB_NodeMemAccess, .align = align); - prev_st = st; - } - - nl_hashset_remove2(&f->gvn_nodes, n, gvn_hash, gvn_compare); - - // we'll replace our original store with the first load (so all the - // stores are in order) - TB_Node* addr = make_member(f, n->inputs[2], off); - set_input(f, n, prev_st, 1); - set_input(f, n, addr, 2); - set_input(f, n, op[valid.splits - 1], 3); - } - - // replace all input edges - FOREACH_N(i, 0, n->input_count) { - TB_Node* in = n->inputs[i]; - if (in) { - TB_Node* k = walk_node(ctx, in); - if (in != k) { set_input(ctx->f, n, k, i); } - } - } - - return n; -} - -// This is the post-optimize pass which gets rid of weird integer types -void tb_pass_legalize(TB_Passes* p, TB_Arch arch) { - CUIK_TIMED_BLOCK("global sched") { - TB_Function* f = p->f; - - LegalizeCtx ctx = { arch, f, p, &p->worklist }; - ctx.fn = legalize_64bit_machine; - - // bottom-up rewrite - f->root_node = walk_node(&ctx, f->root_node); - } -} - diff --git a/vendor/tb/src/opt/libcalls.h b/vendor/tb/src/opt/libcalls.h deleted file mode 100644 index ec9f75c6..00000000 --- a/vendor/tb/src/opt/libcalls.h +++ /dev/null @@ -1,47 +0,0 @@ - -// this is a peephole lmao -static TB_Node* ideal_libcall(TB_Passes* restrict passes, TB_Function* f, TB_Node* n) { - if (n->inputs[2]->type != TB_SYMBOL) { - return NULL; - } - - const TB_Symbol* sym = TB_NODE_GET_EXTRA_T(n->inputs[2], TB_NodeSymbol)->sym; - - bool is_memcpy = strcmp(sym->name, "memcpy") == 0; - bool is_memset = strcmp(sym->name, "memset") == 0; - if (is_memcpy || is_memset) { - // remove from callgraph - FOR_USERS(u, n) { - if (u->n->type == TB_CALLGRAPH) { - TB_Node* last = u->n->inputs[u->n->input_count - 1]; - set_input(f, u->n, NULL, u->n->input_count - 1); - set_input(f, u->n, last, u->slot); - u->n->input_count--; - break; - } - } - - TB_Node* n2 = tb_alloc_node(f, is_memset ? TB_MEMSET : TB_MEMCPY, TB_TYPE_MEMORY, 5, sizeof(TB_NodeMemAccess)); - set_input(f, n2, n->inputs[0], 0); // ctrl - set_input(f, n2, n->inputs[1], 1); // mem - set_input(f, n2, n->inputs[3], 2); // dst - set_input(f, n2, n->inputs[4], 3); // val - set_input(f, n2, n->inputs[5], 4); // size - TB_NODE_SET_EXTRA(n2, TB_NodeMemAccess, .align = 1); - - TB_Node* dst_ptr = n->inputs[2]; - TB_Node* ctrl = n->inputs[0]; - - // returns the destination pointer, convert any users of that to dst - // and CALL has a projection we wanna get rid of - TB_NodeCall* c = TB_NODE_GET_EXTRA_T(n, TB_NodeCall); - subsume_node(f, c->projs[0], ctrl); - subsume_node(f, c->projs[1], n2); - subsume_node(f, c->projs[2], dst_ptr); - - return dst_ptr; - } - - return NULL; -} - diff --git a/vendor/tb/src/opt/loop.h b/vendor/tb/src/opt/loop.h deleted file mode 100644 index 62e32dbb..00000000 --- a/vendor/tb/src/opt/loop.h +++ /dev/null @@ -1,670 +0,0 @@ - -// clone anything except control edges and phis to region -static TB_Node* loop_clone_node(TB_Passes* restrict p, TB_Function* f, TB_Node* region, TB_Node* n, int phi_index) { - TB_Node* cloned = n; - if (n->type == TB_PHI && n->inputs[0] == region) { - // replace OG with phi's edge - cloned = n->inputs[phi_index]; - } else if (cfg_is_region(n) || n->type == TB_ROOT || (n->type == TB_PROJ && n->inputs[0]->type == TB_ROOT)) { - // doesn't clone - } else { - size_t extra = extra_bytes(n); - cloned = tb_alloc_node(f, n->type, n->dt, n->input_count, extra); - - // clone extra data (i hope it's that easy lol) - memcpy(cloned->extra, n->extra, extra); - - // mark new node - tb_pass_mark(p, cloned); - - // fill cloned edges - FOREACH_N(i, 1, n->input_count) { - TB_Node* in = loop_clone_node(p, f, region, n->inputs[i], phi_index); - - cloned->inputs[i] = in; - add_user(f, cloned, in, i, NULL); - } - - if (n->inputs[0]) { - cloned = tb__gvn(f, cloned, extra); - } - } - - #if TB_OPTDEBUG_LOOP - printf("CLONE: "); - print_node_sexpr(n, 0); - printf(" => "); - print_node_sexpr(cloned, 0); - printf("\n"); - #endif - - return cloned; -} - -static uint64_t* iconst(TB_Node* n) { - return n->type == TB_INTEGER_CONST ? &TB_NODE_GET_EXTRA_T(n, TB_NodeInt)->value : NULL; -} - -static void swap_nodes(TB_Function* f, TB_Node* n, int i, int j) { - TB_Node* old = n->inputs[i]; - set_input(f, n, n->inputs[j], i); - set_input(f, n, old, j); -} - -static TB_Node* upcast(TB_Passes* p, TB_Function* f, TB_Node* src, TB_DataType dt) { - TB_Node* cast = tb_alloc_node(f, TB_ZERO_EXT, dt, 2, 0); - set_input(f, cast, src, 1); - tb_pass_mark(p, cast); - return cast; -} - -static bool indvar_simplify(TB_Passes* p, TB_Node* phi, TB_Node* cond, TB_Node* op) { - TB_Function* f = p->f; - - // -1 means signed, 1 means unsigned, 0 means don't care - int best_uses = 0; - TB_DataType best_dt = phi->dt; - - // in the simple case we wanna see if our induction var is usually casted - int phi_uses = 0; - FOR_USERS(u, phi) { - TB_Node* use_n = u->n; - if (use_n == phi) continue; - if (use_n == cond) continue; - if (use_n == op) continue; - - int node_uses = 0; - FOR_USERS(u2, u->n) node_uses += 1; - - // if we've got any casts, let's try to see which is most common - if (use_n->type == TB_SIGN_EXT || use_n->type == TB_ZERO_EXT) { - if (node_uses > best_uses) { - best_uses = node_uses; - best_dt = use_n->dt; - } - } else { - phi_uses += 1; - } - } - - if (best_uses > phi_uses) { - // replace all uses with a different size - TB_Node* cast = tb_alloc_node(f, TB_TRUNCATE, phi->dt, 2, 0); - set_input(f, cast, phi, 1); - - size_t new_node_mark = f->node_count; - User* users = phi->users; - while (users) { - User* next = users->next; - TB_Node* use_n = users->n; - int use_i = users->slot; - - if (use_n == cond && cond != phi) { - assert(use_i == 1); - TB_NODE_SET_EXTRA(cond, TB_NodeCompare, .cmp_dt = best_dt); - - // upcast the other comparand - TB_Node* b_cast = upcast(p, f, cond->inputs[2], best_dt); - set_input(f, cond, b_cast, 2); - } else if (use_n == op) { - assert(use_i == 1); - op->dt = best_dt; - - TB_Node* b_cast = upcast(p, f, op->inputs[2], best_dt); - set_input(f, op, b_cast, 2); - } else if (use_n != cast) { - if (use_n->dt.raw == best_dt.raw && (use_n->type == TB_ZERO_EXT || use_n->type == TB_SIGN_EXT)) { - // no cast necessary - subsume_node2(f, use_n, phi); - } else { - // generic upgrade - set_input(f, use_n, cast, use_i); - tb_pass_mark_users(p, use_n); - } - } - - users = next; - } - - tb_pass_mark_users(p, cast); - tb_pass_mark_users(p, phi); - tb_pass_mark(p, cast); - tb_pass_mark(p, phi); - - // upscale init var - TB_Node* init_cast = upcast(p, f, phi->inputs[1], best_dt); - set_input(f, phi, init_cast, 1); - tb_pass_mark(p, init_cast); - - phi->dt = best_dt; - return true; - } else { - return false; - } -} - -// only case we handle rn is making a bigger stride when we proved the induction var never goes past -static bool indvar_strength_reduction(TB_Passes* p, TB_Node* phi, TB_Node* cond, TB_Node* end_cond, TB_Node* op) { - TB_Function* f = p->f; - - uint64_t* init = iconst(phi->inputs[1]); - uint64_t* end = end_cond ? iconst(end_cond) : NULL; - - return false; -} - -void tb_pass_loop(TB_Passes* p) { - cuikperf_region_start("loop", NULL); - verify_tmp_arena(p); - - size_t block_count = tb_pass_update_cfg(p, &p->worklist, true); - TB_Node** blocks = &p->worklist.items[0]; - - TB_Function* f = p->f; - - // canonicalize regions into natural loop headers (or affine loops) - DynArray(ptrdiff_t) backedges = NULL; - FOREACH_N(i, 0, block_count) { - TB_Node* header = blocks[i]; - if (!cfg_is_region(header) || header->input_count < 2) { - continue; - } - - // find all backedges - dyn_array_clear(backedges); - FOREACH_N(j, 0, header->input_count) { - TB_Node* pred = cfg_get_pred(&p->cfg, header, j); - if (slow_dommy(&p->cfg, header, pred)) { - dyn_array_put(backedges, j); - } - } - - // found a loop :) - if (dyn_array_length(backedges) == 0) { - continue; - } - - TB_OPTDEBUG(LOOP)(printf("found loop on .bb%zu (v%u) with %zu backedges\n", i, header->gvn, dyn_array_length(backedges))); - ptrdiff_t single_backedge = backedges[0]; - - // as part of loop simplification we convert backedges into one, this - // makes it easier to analyze the exit condition. - if (dyn_array_length(backedges) > 1) { - tb_todo(); - } - - // somehow we couldn't simplify the loop? welp - if (single_backedge < 0 || header->input_count != 2) { - continue; - } - - // guarentee that the dominator is inputs[0] - if (single_backedge == 0) { - swap_nodes(f, header, 0, 1); - FOR_USERS(phi, header) { - if (phi->n->type == TB_PHI) { - swap_nodes(f, phi->n, 1, 2); - } - } - single_backedge = 1; - } - header->type = TB_NATURAL_LOOP; - - // given the induction var exists and is representable by an affine function we'll reorder things - // such that it's the first backedge in the header's preds (inputs[1]) alongside making the header - // into a TB_AFFINE_LOOP. - // - // if we don't have the latch in the header BB... ngmi - TB_BasicBlock* header_info = &nl_map_get_checked(p->cfg.node_to_block, header); - TB_Node* latch = header_info->end; - if (latch->type != TB_BRANCH && TB_NODE_GET_EXTRA_T(latch, TB_NodeBranch)->succ_count != 2) { - break; - } - - TB_Node* ind_var = NULL; - TB_Node* cond = latch->inputs[1]; - TB_Node* end_cond = NULL; - TB_Node* phi = cond; - - // detect induction var for affine loops (only valid if it's a phi on the header) - if (cond->type >= TB_CMP_EQ && cond->type <= TB_CMP_SLE && - cond->inputs[1]->type == TB_PHI && cond->inputs[1]->inputs[0] == header) { - phi = cond->inputs[1]; - end_cond = cond->inputs[2]; - } - - // affine loop's induction var should loop like: - // i = phi(init, i + step) where step is constant. - if (phi->inputs[0] == header && phi->dt.type == TB_INT) { - TB_Node* op = phi->inputs[2]; - if ((op->type == TB_ADD || op->type == TB_SUB) && op->inputs[2]->type == TB_INTEGER_CONST) { - // we're a real affine loop now! - header->type = TB_AFFINE_LOOP; - - int64_t step = TB_NODE_GET_EXTRA_T(op->inputs[2], TB_NodeInt)->value; - if (phi->inputs[2]->type == TB_SUB) { - step = -step; - } - - uint64_t* init = iconst(phi->inputs[1]); - uint64_t* end = end_cond ? iconst(end_cond) : NULL; - - if (0 && init && end) { - int64_t trips = (*end - *init) / step; - if (cond->type == TB_CMP_SLT || cond->type == TB_CMP_ULT) { - // if we know the right number of trips we can just append that to the value info - // TODO(NeGate): we should be using a JOIN op to narrow. - Lattice* range = lattice_gimme_uint(p, *init, *init + trips*step); - - tb_pass_mark(p, phi); - if (lattice_universe_map_progress(p, phi, range)) { - tb_pass_mark_users(p, phi); - } - } - } - - #if TB_OPTDEBUG_LOOP - if (init) { - printf(" affine loop: v%u = %"PRId64"*x + %"PRId64"\n", phi->gvn, step, *init); - } else { - printf(" affine loop: v%u = %"PRId64"*x + v%u\n", phi->gvn, step, phi->inputs[1]->gvn); - } - - if (end) { - printf(" latch: %s(v%u, %"PRId64")\n", tb_node_get_name(cond), phi->gvn, *end); - } else { - printf(" latch: %s(v%u, v%u)\n", tb_node_get_name(cond), phi->gvn, end_cond->gvn); - } - - // fixed trip count loops aren't *uncommon* - if (init && end) { - printf(" trips: %"PRId64" (%"PRId64" ... %"PRId64")\n", (*end - *init) / step, *init, *end); - } - #endif - - // we'll just run these alongside the loop detection but maybe they - // should be a separate pass? - indvar_simplify(p, phi, cond, op); - // indvar_strength_reduction(p, phi, cond, end_cond, op); - } - } - } - dyn_array_destroy(backedges); - - #if 0 - // find & canonicalize loops - DynArray(ptrdiff_t) backedges = NULL; - FOREACH_N(i, 0, block_count) { - TB_Node* header = blocks[i]; - if (header->type != TB_REGION || header->input_count < 2) { - continue; - } - - // find all backedges - dyn_array_clear(backedges); - FOREACH_N(j, 0, header->input_count) { - TB_Node* pred = cfg_get_pred(&p->cfg, header, j); - if (slow_dommy(&p->cfg, header, pred)) { - dyn_array_put(backedges, j); - } - } - - // found a loop :) - if (dyn_array_length(backedges) == 0) { - continue; - } - - TB_OPTDEBUG(LOOP)(printf("found loop on .bb%zu with %zu backedges\n", i, dyn_array_length(backedges))); - TB_NODE_GET_EXTRA_T(header, TB_NodeRegion)->freq = 10.0f; - ptrdiff_t single_backedge = backedges[0]; - - // as part of loop simplification we convert backedges into one, this - // makes it easier to analyze the exit condition. - if (dyn_array_length(backedges) > 1) { - tb_todo(); - } - - // somehow we couldn't simplify the loop? welp - if (single_backedge < 0 || header->input_count != 2) { - continue; - } - - // canonicalize loop - TB_NodeRegion* r = TB_NODE_GET_EXTRA(header); - r->natty = true; - if (single_backedge == 0) { - SWAP(TB_Node*, header->inputs[0], header->inputs[1]); - FOR_USERS(phi, header) { - if (phi->n->type == TB_PHI) { - SWAP(TB_Node*, phi->n->inputs[1], phi->n->inputs[2]); - } - } - single_backedge = 1; - } - - // if it's already rotated don't do it again - /* if (header->inputs[single_backedge]->type == TB_PROJ && header->inputs[single_backedge]->inputs[0]->type == TB_BRANCH) { - continue; - } */ - - // if we don't have the latch in the header BB... ngmi - TB_BasicBlock* header_info = &nl_map_get_checked(p->cfg.node_to_block, header); - TB_Node* latch = header_info->end; - if (latch->type != TB_BRANCH && TB_NODE_GET_EXTRA_T(latch, TB_NodeBranch)->succ_count != 2) { - break; - } - - // detect induction var for affine loops (only valid if it's a phi on the header) - TB_Node* ind_var = NULL; - TB_Node* cond = latch->inputs[1]; - TB_Node* end_cond = NULL; - TB_Node* phi = cond; - - if (cond->type >= TB_CMP_EQ && cond->type <= TB_CMP_SLE && cond->inputs[1]->type == TB_PHI) { - phi = cond->inputs[1]; - end_cond = cond->inputs[2]; - } - - // affine loop's induction var should loop like: - // - // i = phi(init, i + step) where step is constant. - if (phi->inputs[0] == header && phi->dt.type == TB_INT) { - TB_Node* op = phi->inputs[2]; - if ((op->type == TB_ADD || op->type == TB_SUB) && op->inputs[2]->type == TB_INTEGER_CONST) { - int64_t step = TB_NODE_GET_EXTRA_T(op->inputs[2], TB_NodeInt)->value; - if (phi->inputs[2]->type == TB_SUB) { - step = -step; - } - - uint64_t* init = iconst(phi->inputs[1]); - if (init) { - TB_OPTDEBUG(LOOP)(printf(" affine loop: v%u = %"PRId64"*x + %"PRId64"\n", phi->gvn, step, *init)); - } else { - TB_OPTDEBUG(LOOP)(printf(" affine loop: v%u = %"PRId64"*x + v%u\n", phi->gvn, step, phi->inputs[1]->gvn)); - } - - __debugbreak(); - - #if 0 - // track the kinds of casts/ops users do and then see who's the winner - int cnt = 0; - bool legal_scale = true; - - int keys[4] = { -1, -1, -1, -1 }; - int uses[4] = { 0 }; - uint32_t use_signed = 0; - TB_DataType use_dt[4]; - - IndVar iv = { - .users = nl_chunked_arr_alloc(tmp_arena) - }; - - indvar_push(&indvar, phi); - - // find the best option - for (NL_ArrChunk* restrict chk = iv.users; chk; chk = chk->next) { - FOREACH_N(i, 0, chk->count) { - TB_Node* use = chk->elems[i]; - __debugbreak(); - } - } - - nl_chunked_arr_reset(&projs); - - FOR_USERS(u, phi) { - if (u->n == op) continue; - if (u->n == cond) continue; - if (u->n == latch) continue; - - // look past the casts, we do wanna know if we benefit from using a - // before type tho. - if (u->n->type == TB_SIGN_EXT || u->n->type == TB_ZERO_EXT) { - is_signed = u->n->type == TB_SIGN_EXT; - stride = 1; - dt = u->n->dt; - } - - bool is_signed = false; - ptrdiff_t stride = 0; - TB_DataType dt = phi->dt; - if (u->n->type == TB_ARRAY_ACCESS) { - stride = TB_NODE_GET_EXTRA_T(u->n, TB_NodeArray)->stride; - } else if (u->n->type == TB_SIGN_EXT || u->n->type == TB_ZERO_EXT) { - is_signed = u->n->type == TB_SIGN_EXT; - stride = 1; - dt = u->n->dt; - } else { - TB_OPTDEBUG(LOOP)(printf(" * cannot scale because we don't understand %s\n", tb_node_get_name(u->n))); - - legal_scale = false; - break; - } - - int node_uses = 0; - FOR_USERS(u2, u->n) node_uses += 1; - - TB_OPTDEBUG(LOOP)(printf(" * scaled by %tu (with %d uses)\n", stride, node_uses)); - - // track uses of ARRAY - int i = 0; - for (; i < cnt; i++) { - if (keys[i] == stride) break; - } - - if (i == cnt) { - if (cnt == 4) break; - keys[cnt++] = stride; - } - - use_signed |= 1 << i; - use_dt[i] = u->n->dt; - uses[i] += node_uses; - } - - // TODO(NeGate): support figuring out scaling when things - // get wacky (*2 vs *6, if we pick the larger number it might - // be more useful but also we'd need to div3 to make this work - // which isn't worth it) - if (legal_scale && cnt == 1) { - // choose high use step to scale by - int biggest = 0, big_use = uses[0]; - FOREACH_N(i, 1, cnt) if (uses[i] > big_use) { - big_use = uses[i]; - biggest = i; - } - - int best_stride = keys[biggest]; - - // rescale the step - TB_Node* scale_n = make_int_node(f, p, phi->dt, best_stride); - set_input(f, op, scale_n, 2); - tb_pass_mark(p, op); - tb_pass_mark(p, scale_n); - - // rescale the limit - if (cond->type != TB_PHI) { - assert(cond->type >= TB_CMP_EQ && cond->type <= TB_CMP_SLE); - assert(cond->inputs[1] == phi); - - TB_Node* limit_scaled = tb_alloc_node(f, TB_MUL, phi->dt, 3, sizeof(TB_NodeBinopInt)); - set_input(f, limit_scaled, cond->inputs[2], 1); - set_input(f, limit_scaled, scale_n, 2); - TB_NODE_GET_EXTRA_T(limit_scaled, TB_NodeBinopInt)->ab = 0; - - set_input(f, cond, limit_scaled, 2); - - tb_pass_mark(p, cond); - tb_pass_mark(p, limit_scaled); - } - - FOR_USERS(u, phi) { - TB_Node* use = u->n; - - // we'll just scale each stride down - if (!worklist_test_n_set(&p->worklist, use)) { - dyn_array_put(p->worklist.items, use); - - if (use->type == TB_ARRAY_ACCESS) { - TB_NodeArray* arr = TB_NODE_GET_EXTRA(use); - arr->stride /= best_stride; - } else { - tb_todo(); - } - } - } - } - #endif - } - } - - if (0) { - TB_Node* backedge_bb = cfg_get_pred(&p->cfg, header, single_backedge); - - // check which paths lead to exitting the loop (don't dominate the single backedge) - int exit_proj_i = -1; - TB_Node* projs[2]; - for (User* u = latch->users; u; u = u->next) { - if (u->n->type == TB_PROJ) { - TB_Node* succ = cfg_next_bb_after_cproj(u->n); - - int proj_i = TB_NODE_GET_EXTRA_T(u->n, TB_NodeProj)->index; - projs[proj_i] = u->n; - - if (!fast_dommy(succ, backedge_bb)) { - // if we have multiple exit latches... just leave - if (exit_proj_i >= 0) { - exit_proj_i = -1; - break; - } - - exit_proj_i = proj_i; - } else { - // body must not have any PHIs - if (cfg_has_phis(succ)) { - exit_proj_i = -1; - break; - } - } - } - } - - // infinite loop? yikes - if (exit_proj_i < 0) { - continue; - } - - // update latch before killing it (we want it's current users to know shit's - // in motion) - tb_pass_mark_users(p, latch); - - // convert to rotated loops - // - // header: latch: - // ... ... - // if (A) body else exit if (A) body else exit - // body: body: - // ... ... - // jmp body if (A) body else exit - // exit: exit: - // - // clone latch and intercept the backedge (making sure to replace all phis with - // the value that's not in the body) - TB_Node* exit_region; - { - int init_edge = 1 - single_backedge; - TB_Node *bot_cloned = NULL, *top_cloned = NULL; - for (TB_Node* curr = latch; curr != header; curr = curr->inputs[0]) { - TB_Node* cloned = loop_clone_node(p, f, header, curr, 1 + init_edge); - tb_pass_mark_users(p, cloned); - - // attach control edge - if (top_cloned) { - set_input(f, top_cloned, cloned, 0); - } else { - bot_cloned = cloned; - } - top_cloned = cloned; - } - - // create cprojs - assert(bot_cloned->type == TB_BRANCH); - TB_Node* proj0 = make_proj_node(f, TB_TYPE_CONTROL, bot_cloned, 0); - TB_Node* proj1 = make_proj_node(f, TB_TYPE_CONTROL, bot_cloned, 1); - - tb_pass_mark(p, proj0); - tb_pass_mark(p, proj1); - - // add zero trip count check - TB_Node* ztc_check = tb_alloc_node(f, TB_REGION, TB_TYPE_CONTROL, 1, sizeof(TB_NodeRegion)); - set_input(f, ztc_check, header->inputs[init_edge], 0); - TB_NODE_GET_EXTRA_T(ztc_check, TB_NodeRegion)->freq = 1.0f; - tb_pass_mark(p, ztc_check); - - // intercept the init path on the header - set_input(f, top_cloned, ztc_check, 0); - set_input(f, header, exit_proj_i ? proj0 : proj1, init_edge); - - exit_region = tb_alloc_node(f, TB_REGION, TB_TYPE_CONTROL, 2, sizeof(TB_NodeRegion)); - TB_NODE_GET_EXTRA_T(exit_region, TB_NodeRegion)->freq = 1.0f; - tb_pass_mark(p, exit_region); - set_input(f, exit_region, exit_proj_i ? proj1 : proj0, 0); - - // connect exit projection to exit region, then connect the exit region to - // what the exit successor wanted - User* after_exit = cfg_next_user(projs[exit_proj_i]); - assert(after_exit != NULL && "no successors after exit?"); - - set_input(f, exit_region, projs[exit_proj_i], 1); - set_input(f, after_exit->n, exit_region, after_exit->slot); - - DO_IF(TB_OPTDEBUG_LOOP)(TB_NODE_GET_EXTRA_T(ztc_check, TB_NodeRegion)->tag = lil_name(f, "loop.ztc.%d", i)); - DO_IF(TB_OPTDEBUG_LOOP)(TB_NODE_GET_EXTRA_T(exit_region, TB_NodeRegion)->tag = lil_name(f, "loop.exit.%d", i)); - DO_IF(TB_OPTDEBUG_LOOP)(TB_NODE_GET_EXTRA_T(header, TB_NodeRegion)->tag = lil_name(f, "loop.body.%d", i)); - } - - // insert duplicated branch for backedge - { - TB_Node *bot_cloned = NULL, *top_cloned = NULL; - for (TB_Node* curr = latch; curr != header; curr = curr->inputs[0]) { - TB_Node* cloned = loop_clone_node(p, f, header, curr, 1 + single_backedge); - - // attach control edge - if (top_cloned) { - set_input(f, top_cloned, cloned, 0); - } else { - bot_cloned = cloned; - } - top_cloned = cloned; - } - - // create cprojs - assert(bot_cloned->type == TB_BRANCH); - TB_Node* proj0 = make_proj_node(f, TB_TYPE_CONTROL, bot_cloned, 0); - TB_Node* proj1 = make_proj_node(f, TB_TYPE_CONTROL, bot_cloned, 1); - tb_pass_mark(p, proj0); - tb_pass_mark(p, proj1); - - set_input(f, top_cloned, header->inputs[single_backedge], 0); - set_input(f, header, exit_proj_i ? proj0 : proj1, single_backedge); - - // remove initial latch now - User* after_next = cfg_next_user(projs[1 - exit_proj_i]); - assert(after_next != NULL && "no successors after next?"); - assert(after_next->slot == 0 && "pretty sure it should've been passed through in[0]"); - - set_input(f, exit_region, exit_proj_i ? proj1 : proj0, 1); - set_input(f, after_next->n, latch->inputs[0], after_next->slot); - - tb_pass_kill_node(f, projs[exit_proj_i]); - tb_pass_kill_node(f, projs[1 - exit_proj_i]); - tb_pass_kill_node(f, latch); - } - } - - skip:; - } - #endif - - dyn_array_destroy(backedges); - cuikperf_region_end(); -} diff --git a/vendor/tb/src/opt/mem2reg.h b/vendor/tb/src/opt/mem2reg.h deleted file mode 100644 index 52c4d948..00000000 --- a/vendor/tb/src/opt/mem2reg.h +++ /dev/null @@ -1,426 +0,0 @@ - -typedef enum { - // we had some escape so we can't rewrite - RENAME_NONE, - - // the value has no escapes but requires pointer - // arithmetic, we can at least make equivalence - // classes for them & split edges. - RENAME_MEMORY, - - // best stuff, simple names + phis - RENAME_VALUE, -} RenameMode; - -typedef struct { - TB_Node* addr; - int alias_idx; -} Rename; - -typedef struct { - TB_Node* n; - int slot; - int reason; -} MemOp; - -typedef struct { - int local_count; - Rename* renames; - NL_Table phi2local; -} LocalSplitter; - -// used by phi2local whenever a memory phi is marked as split (it's not bound to any -// new name but shouldn't count as NULL) -static Rename RENAME_DUMMY = { 0 }; - -static int bits_in_data_type(int pointer_size, TB_DataType dt); - -static bool good_mem_op(TB_Passes* p, TB_Node* n) { // ld, st, memcpy, memset - if (n->type == TB_LOAD) { - return true; - } else if (n->type >= TB_STORE && n->type <= TB_MEMSET) { - Lattice* l = lattice_universe_get(p, n); - return l == &TOP_IN_THE_SKY || l == p->root_mem; - } else { - return false; - } -} - -static bool same_base(TB_Node* a, TB_Node* b) { - while (b->type == TB_MEMBER_ACCESS || b->type == TB_ARRAY_ACCESS) { - if (a == b) return true; - b = b->inputs[1]; - } - - return a == b; -} - -static TB_Node* next_mem_user(TB_Node* n) { - FOR_USERS(u, n) { - if (is_mem_out_op(u->n)) { - return u->n; - } - } - - return NULL; -} - -static int categorize_alias_idx(LocalSplitter* restrict ctx, TB_Node* n) { - // skip any member or array accesses - while (n->type == TB_ARRAY_ACCESS || n->type == TB_MEMBER_ACCESS) { - n = n->inputs[1]; - } - - FOREACH_N(i, 0, ctx->local_count) { - if (ctx->renames[i].addr == n) return i; - } - - return -1; -} - -enum { - MEM_FORK, MEM_JOIN, MEM_END, MEM_USE -}; - -static bool all_stores_dead(TB_Node* n) { - FOR_USERS(u, n) { - if (u->slot != 2 || u->n->type != TB_STORE) { - return false; - } - } - - return true; -} - -static void expunge(TB_Function* f, TB_Node* n) { - // delete users without weird iteration invalidation issues - while (n->users) { - TB_Node* use_n = n->users->n; - int use_i = n->users->slot; - - assert(use_i == 2); - n->users = n->users->next; - use_n->inputs[2] = NULL; - - FOR_USERS(u, use_n) { - // kill connected phis - if (u->n->type == TB_PHI) { - set_input(f, u->n, NULL, 0); - } - } - - tb_pass_kill_node(f, use_n); - } - tb_pass_kill_node(f, n); -} - -static TB_Node* node_or_poison(TB_Function* f, TB_Node* n, TB_DataType dt) { - return n ? n : make_poison(f, dt); -} - -static void fixup_mem_node(TB_Function* f, TB_Passes* restrict p, LocalSplitter* restrict ctx, TB_Node* curr, TB_Node** latest) { - int user_cnt = 0; - MemOp* users = NULL; - TB_ArenaSavepoint sp = tb_arena_save(tmp_arena); - - // walk past each memory effect and categorize it - while (curr) { - // printf("WALK v%u\n", curr->gvn); - - int user_cap = 0; - FOR_USERS(u, curr) { user_cap++; } - - sp = tb_arena_save(tmp_arena); - user_cnt = 0; - users = tb_arena_alloc(tmp_arena, user_cap * sizeof(MemOp)); - FOR_USERS(u, curr) { - TB_Node* use_n = u->n; - int use_i = u->slot; - - // we can either be followed by: - // * merge: we're done now, stitch in the latest ops - // - // * phi: whoever makes it here first follows it through and generates the - // parallel memory phis, everyone else just fills in the existing nodes. - // - // * normie memory op: just advances (maybe forking) - int reason = -1; - - if (0) {} - else if (use_n->type == TB_RETURN) { reason = MEM_END; } - else if (use_n->type == TB_PHI) { reason = MEM_JOIN; } - else if (is_mem_only_in_op(u->n)) { reason = MEM_USE; } - else if (cfg_is_mproj(use_n) || (use_i == 1 && is_mem_out_op(use_n))) { - reason = MEM_FORK; - } - - if (reason >= 0) { - users[user_cnt].n = u->n; - users[user_cnt].slot = u->slot; - users[user_cnt].reason = reason; - user_cnt += 1; - } - } - - // skip past projections - TB_Node* st_val = NULL; - if (curr->type >= TB_STORE && curr->type <= TB_MEMSET) { - int cat = categorize_alias_idx(ctx, curr->inputs[2]); - if (cat < 0) { - set_input(f, curr, latest[0], 1); - latest[0] = curr; - } else { - int alias_idx = ctx->renames[cat].alias_idx; - if (alias_idx >= 0) { - // rewrite memory edge - set_input(f, curr, latest[1 + cat], 1); - latest[1 + cat] = curr; - - // invalidate old memory type - tb_pass_mark(p, curr); - lattice_universe_map(p, curr, NULL); - } else { - // get rid of the store, we don't need it - latest[1 + cat] = curr->inputs[3]; - - // printf("* KILL %%%u (latest[%d] = %%%u)\n", curr->gvn, 1 + cat, curr->inputs[3]->gvn); - tb_pass_kill_node(f, curr); - } - } - } else if (curr->dt.type == TB_TUPLE) { - // skip to mproj - assert(curr->type != TB_SPLITMEM); - curr = next_mem_user(curr); - } - - // fixup any connected loads - FOREACH_N(i, 0, user_cnt) { - TB_Node* use_n = users[i].n; - int use_i = users[i].slot; - int reason = users[i].reason; - - if (reason == MEM_USE) { - int cat = categorize_alias_idx(ctx, use_n->inputs[2]); - if (cat < 0) { - set_input(f, use_n, latest[0], 1); - } else { - int alias_idx = ctx->renames[cat].alias_idx; - if (alias_idx >= 0) { - // rewrite edge - set_input(f, use_n, latest[1 + cat], 1); - tb_pass_mark(p, use_n); - } else { - assert(use_n->type == TB_LOAD); - - TB_Node* val = node_or_poison(f, latest[1 + cat], use_n->dt); - subsume_node(f, use_n, val); - } - } - } - } - - // "tail" such that we don't make another stack frame and more - // importantly another "latest" array - if (user_cnt == 1 && users[0].reason == MEM_FORK) { - curr = users[0].n; - tb_arena_restore(tmp_arena, sp); - } else { - break; - } - } - - FOREACH_N(i, 0, user_cnt) { - TB_Node* use_n = users[i].n; - int use_i = users[i].slot; - int reason = users[i].reason; - - switch (reason) { - case MEM_FORK: { - TB_ArenaSavepoint sp = tb_arena_save(tmp_arena); - TB_Node** new_latest = tb_arena_alloc(tmp_arena, (1 + ctx->local_count) * sizeof(TB_Node*)); - FOREACH_N(i, 0, 1 + ctx->local_count) { - new_latest[i] = latest[i]; - } - - fixup_mem_node(f, p, ctx, use_n, new_latest); - tb_arena_restore(tmp_arena, sp); - break; - } - - case MEM_JOIN: { - // stitch latest state to phis - TB_Node* region = use_n->inputs[0]; - - Rename* v = nl_table_get(&ctx->phi2local, use_n); - if (v == NULL) { - nl_table_put(&ctx->phi2local, use_n, &RENAME_DUMMY); - - // convert single phi into parallel phis (use_n will become the leftovers mem) - TB_Node** new_latest = tb_arena_alloc(tmp_arena, (1 + ctx->local_count) * sizeof(TB_Node*)); - - // convert single phi into multiple parallel phis (first one will be replaced - // with the root mem) - set_input(f, use_n, latest[0], use_i); - lattice_universe_map(p, use_n, NULL); - - assert(use_n->dt.type == TB_MEMORY); - new_latest[0] = use_n; - - // make extra alias phis - FOREACH_N(i, 0, ctx->local_count) { - // let's hope the first datatype we get from the phi is decent, if not the - // peepholes will ideally fix it. - TB_Node* val = latest[1 + i]; - TB_DataType dt = TB_TYPE_MEMORY; - - if (val != NULL) { - if (ctx->renames[i].alias_idx < 0) { - dt = val->dt; - } - - TB_Node* new_phi = tb_alloc_node(f, TB_PHI, dt, use_n->input_count, 0); - set_input(f, new_phi, region, 0); - set_input(f, new_phi, val, use_i); - new_latest[1 + i] = new_phi; - tb_pass_mark(p, new_phi); - - nl_table_put(&ctx->phi2local, new_phi, &ctx->renames[i]); - lattice_universe_map(p, new_phi, NULL); - } else { - new_latest[1 + i] = NULL; - } - } - - // first entry, every other time we'll just be stitching phis - tb_pass_mark(p, use_n); - fixup_mem_node(f, p, ctx, use_n, new_latest); - } else if (v == &RENAME_DUMMY) { - FOR_USERS(phi, region) if (phi->n->type == TB_PHI) { - Rename* name = nl_table_get(&ctx->phi2local, phi->n); - if (name == &RENAME_DUMMY) { - set_input(f, phi->n, latest[0], use_i); - } else if (name) { - if (name->alias_idx < 0) { - TB_Node* val = latest[1 + (name - ctx->renames)]; - if (val->dt.raw != phi->n->dt.raw) { - // insert bitcast - TB_Node* cast = tb_alloc_node(f, TB_BITCAST, phi->n->dt, 2, 0); - set_input(f, cast, val, 1); - val = cast; - } - - set_input(f, phi->n, val, use_i); - } else { - set_input(f, phi->n, latest[1 + (name - ctx->renames)], use_i); - } - } - } - } - break; - } - - case MEM_END: { - // stitch the latest nodes to the merge or return - assert(use_n->type == TB_RETURN); - set_input(f, use_n, latest[0], 1); - - /* size_t j = 0; - FOREACH_N(i, 0, ctx->local_count) if (ctx->renames[i].alias_idx > 0) { - assert(latest[i] != NULL && "TODO we should place a poison?"); - set_input(f, use_n, latest[i], 2+j); - j += 1; - } */ - break; - } - } - } - - tb_arena_restore(tmp_arena, sp); -} - -void tb_pass_locals(TB_Passes* p) { - TB_Function* f = p->f; - - do { - cuikperf_region_start("locals", NULL); - assert(dyn_array_length(p->worklist.items) == 0); - - // find all locals - LocalSplitter ctx = { 0 }; - NL_ChunkedArr locals = nl_chunked_arr_alloc(tmp_arena); - FOR_USERS(u, f->root_node) { - if (u->n->type != TB_LOCAL) continue; - nl_chunked_arr_put(&locals, u->n); - ctx.local_count++; - } - - // find reasons for renaming - ctx.renames = tb_arena_alloc(tmp_arena, ctx.local_count * sizeof(Rename)); - - size_t j = 0; - bool needs_to_rewrite = false; - for (NL_ArrChunk* restrict chk = locals.first; chk; chk = chk->next) { - FOREACH_N(i, 0, chk->count) { - TB_Node* addr = chk->elems[i]; - RenameMode mode = RENAME_VALUE; - - FOR_USERS(mem, addr) { - if (mem->slot == 1 && (mem->n->type == TB_MEMBER_ACCESS || mem->n->type == TB_ARRAY_ACCESS)) { - // TODO(NeGate): pointer arith are also fair game, since they'd stay in bounds (given no UB) - // mode = RENAME_MEMORY; - mode = RENAME_NONE; - break; - } else if (mem->slot != 2 || !good_mem_op(p, mem->n)) { - mode = RENAME_NONE; - break; - } - } - - if (mode != RENAME_NONE) { - // allocate new alias index - if (mode == RENAME_MEMORY) { - ctx.renames[j].alias_idx = p->alias_n++; - needs_to_rewrite = true; - } else if (mode == RENAME_VALUE) { - ctx.renames[j].alias_idx = -1; - needs_to_rewrite = true; - } - - ctx.renames[j].addr = addr; - j += 1; - } - } - } - - if (!needs_to_rewrite) { - nl_chunked_arr_reset(&locals); - cuikperf_region_end(); - break; - } - - ctx.local_count = j; - ctx.phi2local = nl_table_alloc(200); - - // let's rewrite values & memory - TB_Node* first_mem = next_mem_user(f->params[1]); - if (first_mem) { - TB_Node** latest = tb_arena_alloc(tmp_arena, (1 + ctx.local_count) * sizeof(TB_Node*)); - FOREACH_N(i, 1, 1 + ctx.local_count) { latest[i] = NULL; } - latest[0] = f->params[1]; - - fixup_mem_node(f, p, &ctx, first_mem, latest); - } - - // ok if they're now value edges we can delete the LOCAL - FOREACH_N(i, 0, ctx.local_count) if (ctx.renames[i].alias_idx < 0) { - tb_pass_kill_node(f, ctx.renames[i].addr); - } - - nl_table_free(ctx.phi2local); - nl_chunked_arr_reset(&locals); - cuikperf_region_end(); - - // run a round of peepholes, lots of new room to explore :) - tb_pass_peephole(p); - } while (true); -} diff --git a/vendor/tb/src/opt/mem_opt.h b/vendor/tb/src/opt/mem_opt.h deleted file mode 100644 index fb873e7f..00000000 --- a/vendor/tb/src/opt/mem_opt.h +++ /dev/null @@ -1,295 +0,0 @@ -// Certain aliasing optimizations technically count as peepholes lmao, these can get fancy -// so the sliding window notion starts to break down but there's no global analysis and -// i can make them incremental technically so we'll go wit it. -typedef struct { - TB_Node* base; - int64_t offset; -} KnownPointer; - -static bool is_local_ptr(TB_Node* n) { - // skip past ptr arith - while (n->type == TB_MEMBER_ACCESS || n->type == TB_ARRAY_ACCESS) { - n = n->inputs[1]; - } - - return n->type == TB_LOCAL; -} - -static KnownPointer known_pointer(TB_Node* n) { - if (n->type == TB_MEMBER_ACCESS) { - return (KnownPointer){ n->inputs[1], TB_NODE_GET_EXTRA_T(n, TB_NodeMember)->offset }; - } else { - return (KnownPointer){ n, 0 }; - } -} - -typedef struct FoundPhis { - struct FoundPhis* prev; - TB_Node* old; - TB_Node* new; -} FoundPhis; - -static TB_Node* data_phi_from_memory_phi(TB_Passes* restrict p, TB_Function* f, TB_Node* n, TB_Node* addr, TB_Node* mem, FoundPhis* prev, int steps) { - // convert memory phis into data phis - assert(mem->type == TB_PHI); - assert(mem->dt.type == TB_MEMORY && "memory input should be memory"); - - TB_DataType dt = n->dt; - - TB_ArenaSavepoint sp = tb_arena_save(tmp_arena); - size_t path_count = mem->input_count - 1; - TB_Node** paths = tb_arena_alloc(tmp_arena, path_count * sizeof(TB_Node*)); - - TB_Node* phi = tb_alloc_node(f, TB_PHI, dt, 1 + path_count, 0); - - bool fail = false; - FOREACH_N(i, 0, path_count) { - TB_Node* st = mem->inputs[1+i]; - if (st->type == TB_PHI && steps > 0) { - TB_Node* k = NULL; - - // if we've already seen this phi before, let's just reference the OG - for (FoundPhis* p = prev; p; p = p->prev) { - if (p->old == st) k = p->new; - } - - if (k == NULL) { - FoundPhis next = { prev, mem, phi }; - k = data_phi_from_memory_phi(p, f, n, addr, st, &next, steps - 1); - if (k == NULL) { - fail = true; - break; - } - } - - paths[i] = k; - } else if (st->type != TB_STORE || st->inputs[2] != addr || st->inputs[3]->dt.raw != dt.raw) { - fail = true; - break; - } else { - paths[i] = st->inputs[3]; - } - } - - if (fail) { - violent_kill(f, phi); - return NULL; - } - - set_input(f, phi, mem->inputs[0], 0); - FOREACH_N(i, 0, path_count) { - set_input(f, phi, paths[i], 1+i); - } - tb_arena_restore(tmp_arena, sp); - - return phi; -} - -static TB_Node* ideal_load(TB_Passes* restrict p, TB_Function* f, TB_Node* n) { - TB_Node* ctrl = n->inputs[0]; - TB_Node* mem = n->inputs[1]; - TB_Node* addr = n->inputs[2]; - - if (mem->type == TB_PHI) { - TB_Node* k = data_phi_from_memory_phi(p, f, n, addr, mem, NULL, 2); - if (k) return k; - } - - if (ctrl != NULL) { - // we've dependent on code which must always be run (ROOT.mem) - if (n->inputs[0]->type == TB_PROJ && n->inputs[0]->inputs[0]->type == TB_ROOT) { - set_input(f, n, NULL, 0); - return n; - } else { - TB_Node* base = addr; - while (base->type == TB_MEMBER_ACCESS || base->type == TB_ARRAY_ACCESS) { - base = base->inputs[1]; - } - - // loads based on LOCALs don't need control-dependence, it's actually kinda annoying - if (base->type == TB_LOCAL) { - set_input(f, n, NULL, 0); - return n; - } - } - - // if all paths are dominated by a load of some address then it's safe - // to relax ctrl deps. - ICodeGen* cg = f->super.module->codegen; - int bits_read = bits_in_data_type(cg->pointer_size, n->dt); - - for (User* u = addr->users; u; u = u->next) { - // find other users of the address which read the same size (or more) - TB_NodeTypeEnum type = 0; - if (u->n != n && u->slot == 2 && u->n->type == TB_LOAD) { - TB_DataType mem_dt = n->type == TB_LOAD ? n->dt : n->inputs[3]->dt; - int other_bits_read = bits_in_data_type(cg->pointer_size, mem_dt); - if (bits_read <= other_bits_read) { - TB_Node* other_ctrl = u->n->inputs[0]; - if (other_ctrl == NULL || (fast_dommy(other_ctrl, ctrl) && other_ctrl != ctrl)) { - set_input(f, n, other_ctrl, 0); - return n; - } - } - } - } - } - - return NULL; -} - -static TB_Node* identity_load(TB_Passes* restrict p, TB_Function* f, TB_Node* n) { - // god i need a pattern matcher - // (load (store X A Y) A) => Y - TB_Node *mem = n->inputs[1], *addr = n->inputs[2]; - if (mem->type == TB_STORE && mem->inputs[2] == addr && - n->dt.raw == mem->inputs[3]->dt.raw) { - return mem->inputs[3]; - } - - return n; -} - -static Lattice* value_split_mem(TB_Passes* restrict p, TB_Node* n) { - TB_NodeMemSplit* s = TB_NODE_GET_EXTRA(n); - - size_t size = sizeof(Lattice) + s->alias_cnt*sizeof(Lattice*); - Lattice* l = tb_arena_alloc(tmp_arena, size); - *l = (Lattice){ LATTICE_TUPLE, ._tuple = { s->alias_cnt } }; - FOREACH_N(i, 0, s->alias_cnt) { - l->elems[i] = lattice_alias(p, s->alias_idx[i]); - } - - Lattice* k = nl_hashset_put2(&p->type_interner, l, lattice_hash, lattice_cmp); - if (k) { - tb_arena_free(tmp_arena, l, size); - return k; - } else { - return l; - } -} - -static TB_Node* ideal_merge_mem(TB_Passes* restrict p, TB_Function* f, TB_Node* n) { - bool progress = false; - TB_Node* split_node = n->inputs[1]; - TB_NodeMemSplit* split = TB_NODE_GET_EXTRA(split_node); - - // remove useless split edges - size_t i = 2; - while (i < n->input_count) { - if (n->inputs[i]->type == TB_PROJ && n->inputs[i]->inputs[0] == split_node && single_use(n->inputs[i])) { - int j = i - 2; - progress = true; - - assert(split->alias_cnt > 0); - assert(split->alias_cnt + 2 == n->input_count); - - split->alias_cnt -= 1; - split->alias_idx[j] = split->alias_idx[split->alias_cnt]; - - User* proj = proj_with_index(split_node, split->alias_cnt); - assert(proj->n); - TB_NODE_SET_EXTRA(proj->n, TB_NodeProj, .index = j); - - tb_pass_kill_node(f, n->inputs[i]); - - // simple remove-swap - set_input(f, n, n->inputs[n->input_count - 1], i); - set_input(f, n, NULL, n->input_count - 1); - n->input_count -= 1; - - // we didn't *really* change the memory type, just reordered it (all the live projs - // are the same so we don't need to push them onto the worklist) - Lattice* new_split_type = value_split_mem(p, split_node); - lattice_universe_map(p, split_node, new_split_type); - } else { - i += 1; - } - } - - return progress ? n : NULL; -} - -static TB_Node* ideal_store(TB_Passes* restrict p, TB_Function* f, TB_Node* n) { - TB_Node *mem = n->inputs[1], *addr = n->inputs[2], *val = n->inputs[3]; - TB_DataType dt = val->dt; - - // if a store has only one user in this chain it means it's only job was - // to facilitate the creation of that user store... if we can detect that - // user store is itself dead, everything in the middle is too. - if (mem->type == TB_STORE && single_use(mem) && mem->inputs[2] == addr && mem->inputs[3]->dt.raw == dt.raw) { - // choose the bigger alignment (we wanna keep this sort of info) - TB_NodeMemAccess* a = TB_NODE_GET_EXTRA(mem); - TB_NodeMemAccess* b = TB_NODE_GET_EXTRA(n); - if (a->align > b->align) b->align = a->align; - - // make sure to kill the stores to avoid problems - TB_Node* parent = mem->inputs[1]; - tb_pass_kill_node(f, mem); - - set_input(f, n, parent, 1); - return n; - } - - return NULL; -} - -static bool is_cool(uint64_t x) { return x == 1 || x == 2 || x == 4 || x == 8; } -static TB_Node* ideal_memset(TB_Passes* restrict p, TB_Function* f, TB_Node* n) { - // convert small memsets into stores - uint64_t count, val; - if (get_int_const(n->inputs[4], &count) && get_int_const(n->inputs[3], &val) && is_cool(count)) { - // fill rest of the bytes - FOREACH_N(i, 1, count) { - val |= (val & 0xFF) << (i*8); - } - - TB_DataType dt = TB_TYPE_INTN(count*8); - set_input(f, n, make_int_node(f, p, dt, val), 3); - set_input(f, n, NULL, 4); - n->input_count = 4; - n->type = TB_STORE; - return n; - } - - return NULL; -} - -static TB_Node* ideal_return(TB_Passes* restrict p, TB_Function* f, TB_Node* n) { - return NULL; -} - -static TB_Node* ideal_memcpy(TB_Passes* restrict p, TB_Function* f, TB_Node* n) { - // convert small memsets into ld+st pairs - uint64_t count, val; - if (get_int_const(n->inputs[4], &count) && is_cool(count)) { - TB_Node* ctrl = n->inputs[0]; - TB_Node* mem = n->inputs[1]; - TB_Node* src = n->inputs[3]; - - TB_DataType dt = TB_TYPE_INTN(count*8); - TB_Node* ld = tb_alloc_node(f, TB_LOAD, dt, 3, sizeof(TB_NodeBinopInt)); - set_input(f, ld, ctrl, 0); - set_input(f, ld, mem, 1); - set_input(f, ld, src, 2); - tb_pass_mark(p, ld); - - // convert to store, remove extra input - n->type = TB_STORE; - set_input(f, n, ld, 3); - set_input(f, n, NULL, 4); - n->input_count = 4; - return n; - } - - return NULL; -} - -static Lattice* value_merge_mem(TB_Passes* restrict p, TB_Node* n) { - return &BOT_IN_THE_SKY; -} - -static Lattice* value_mem(TB_Passes* restrict p, TB_Node* n) { - // just inherit memory from parent - return lattice_universe_get(p, n->inputs[1]); -} diff --git a/vendor/tb/src/opt/optimizer.c b/vendor/tb/src/opt/optimizer.c deleted file mode 100644 index 6796fa78..00000000 --- a/vendor/tb/src/opt/optimizer.c +++ /dev/null @@ -1,1757 +0,0 @@ -// Let's just explain the architecture of the optimizer here. -// -// # Peephole optimizations -// These are the kind which work locally like 2+2=4 and in TB's design they're -// performed incrementally which means that certain mutations must go through -// functions to guarentee they update correctly. Let's go over those: -// -// set_input(f, n, in, slot) -// basically `n->inputs[slot] = in` except it correctly updates the user set -// -// # How to implement peepholes -// TODO -// -#include "passes.h" -#include - -thread_local TB_Arena* tmp_arena; - -// helps us do some matching later -static User* remove_user(TB_Node* n, int slot); -static void remove_input(TB_Function* f, TB_Node* n, size_t i); -static void violent_kill(TB_Function* f, TB_Node* n); - -static void subsume_node(TB_Function* f, TB_Node* n, TB_Node* new_n); -static void subsume_node2(TB_Function* f, TB_Node* n, TB_Node* new_n); -static void print_lattice(Lattice* l, TB_DataType dt); - -// node creation helpers -TB_Node* make_poison(TB_Function* f, TB_DataType dt); -TB_Node* dead_node(TB_Function* f, TB_Passes* restrict p); -TB_Node* make_int_node(TB_Function* f, TB_Passes* restrict p, TB_DataType dt, uint64_t x); -TB_Node* make_proj_node(TB_Function* f, TB_DataType dt, TB_Node* src, int i); - -static size_t tb_pass_update_cfg(TB_Passes* p, Worklist* ws, bool preserve); - -//////////////////////////////// -// Worklist -//////////////////////////////// -void worklist_alloc(Worklist* restrict ws, size_t initial_cap) { - ws->visited_cap = (initial_cap + 63) / 64; - ws->visited = tb_platform_heap_alloc(ws->visited_cap * sizeof(uint64_t)); - ws->items = dyn_array_create(uint64_t, ws->visited_cap * 64); - FOREACH_N(i, 0, ws->visited_cap) { - ws->visited[i] = 0; - } -} - -void worklist_free(Worklist* restrict ws) { - tb_platform_heap_free(ws->visited); - dyn_array_destroy(ws->items); -} - -void worklist_clear_visited(Worklist* restrict ws) { - CUIK_TIMED_BLOCK("clear visited") { - memset(ws->visited, 0, ws->visited_cap * sizeof(uint64_t)); - } -} - -void worklist_clear(Worklist* restrict ws) { - CUIK_TIMED_BLOCK("clear worklist") { - memset(ws->visited, 0, ws->visited_cap * sizeof(uint64_t)); - dyn_array_clear(ws->items); - } -} - -void worklist_remove(Worklist* restrict ws, TB_Node* n) { - uint64_t gvn_word = n->gvn / 64; // which word this ID is at - if (gvn_word >= ws->visited_cap) return; - - uint64_t gvn_mask = 1ull << (n->gvn % 64); - ws->visited[gvn_word] &= ~gvn_mask; -} - -// checks if node is visited but doesn't push item -bool worklist_test(Worklist* restrict ws, TB_Node* n) { - uint64_t gvn_word = n->gvn / 64; // which word this ID is at - if (gvn_word >= ws->visited_cap) return false; - - uint64_t gvn_mask = 1ull << (n->gvn % 64); - return ws->visited[gvn_word] & gvn_mask; -} - -bool worklist_test_n_set(Worklist* restrict ws, TB_Node* n) { - uint64_t gvn_word = n->gvn / 64; // which word this ID is at - - // resize? - if (gvn_word >= ws->visited_cap) { - size_t new_cap = gvn_word + 16; - ws->visited = tb_platform_heap_realloc(ws->visited, new_cap * sizeof(uint64_t)); - - // clear new space - FOREACH_N(i, ws->visited_cap, new_cap) { - ws->visited[i] = 0; - } - - ws->visited_cap = new_cap; - } - - uint64_t gvn_mask = 1ull << (n->gvn % 64); - if (ws->visited[gvn_word] & gvn_mask) { - return true; - } else { - ws->visited[gvn_word] |= gvn_mask; - return false; - } -} - -void worklist_push(Worklist* restrict ws, TB_Node* restrict n) { - if (!worklist_test_n_set(ws, n)) { - dyn_array_put(ws->items, n); - } -} - -TB_Node* worklist_pop(Worklist* ws) { - if (dyn_array_length(ws->items)) { - TB_Node* n = dyn_array_pop(ws->items); - uint64_t gvn_word = n->gvn / 64; - uint64_t gvn_mask = 1ull << (n->gvn % 64); - - ws->visited[gvn_word] &= ~gvn_mask; - return n; - } else { - return NULL; - } -} - -int worklist_popcount(Worklist* ws) { - int sum = 0; - for (size_t i = 0; i < ws->visited_cap; i++) { - sum += tb_popcount64(ws->visited[i]); - } - return sum; -} - -void verify_tmp_arena(TB_Passes* p) { - // once passes are run on a thread, they're pinned to it. - TB_Module* m = p->f->super.module; - TB_ThreadInfo* info = tb_thread_info(m); - - if (p->pinned_thread == NULL) { - p->pinned_thread = info; - tb_arena_clear(p->pinned_thread->tmp_arena); - } else if (p->pinned_thread != info) { - tb_panic( - "TB_Passes are bound to a thread, you can't switch which threads they're run on\n\n" - "NOTE: if you really need to run across threads you'll need to exit the passes and\n" - "start anew... though you pay a performance hit everytime you start one" - ); - } - - tmp_arena = p->pinned_thread->tmp_arena; -} - -static int bits_in_data_type(int pointer_size, TB_DataType dt) { - switch (dt.type) { - case TB_INT: return dt.data; - case TB_PTR: return pointer_size; - case TB_FLOAT: - if (dt.data == TB_FLT_32) return 32; - if (dt.data == TB_FLT_64) return 64; - return 0; - default: return 0; - } -} - -static char* lil_name(TB_Function* f, const char* fmt, ...) { - char* buf = TB_ARENA_ALLOC(tmp_arena, 30); - - va_list ap; - va_start(ap, fmt); - vsnprintf(buf, 30, fmt, ap); - va_end(ap); - return buf; -} - -static TB_Node* mem_user(TB_Passes* restrict p, TB_Node* n, int slot) { - FOR_USERS(u, n) { - if ((u->n->type == TB_PROJ && u->n->dt.type == TB_MEMORY) || - (u->slot == slot && is_mem_out_op(u->n))) { - return u->n; - } - } - - return NULL; -} - -static bool single_use(TB_Node* n) { - return n->users->next == NULL; -} - -static bool is_same_align(TB_Node* a, TB_Node* b) { - TB_NodeMemAccess* aa = TB_NODE_GET_EXTRA(a); - TB_NodeMemAccess* bb = TB_NODE_GET_EXTRA(b); - return aa->align == bb->align; -} - -static bool is_empty_bb(TB_Passes* restrict p, TB_Node* end) { - assert(end->type == TB_BRANCH || end->type == TB_UNREACHABLE); - if (!cfg_is_bb_entry(end->inputs[0])) { - return false; - } - - TB_Node* bb = end->inputs[0]; - FOR_USERS(use, bb) { - TB_Node* n = use->n; - if (use->n != end) return false; - } - - return true; -} - -static bool is_if_branch(TB_Node* n, uint64_t* falsey) { - if (n->type == TB_BRANCH && n->input_count == 2 && TB_NODE_GET_EXTRA_T(n, TB_NodeBranch)->succ_count == 2) { - *falsey = TB_NODE_GET_EXTRA_T(n, TB_NodeBranch)->keys[0].key; - return true; - } - - return false; -} - -// incremental dominators, plays nice with peepholes and has -// a limited walk of 20 steps. -static TB_Node* fast_idom(TB_Node* bb) { - int steps = 0; - - // note that "subtypes" of region like TB_NATURAL_LOOP and TB_AFFINE_LOOP are - // valid for fast doms since they guarentee they're dominated by inputs[0] - while (steps < FAST_IDOM_LIMIT && bb->type != TB_REGION && bb->type != TB_ROOT) { - bb = bb->inputs[0]; - steps++; - } - - return bb; -} - -static bool fast_dommy(TB_Node* expected_dom, TB_Node* bb) { - int steps = 0; - - // note that "subtypes" of region like TB_NATURAL_LOOP and TB_AFFINE_LOOP are - // valid for fast doms since they guarentee they're dominated by inputs[0] - while (steps < FAST_IDOM_LIMIT && bb != expected_dom && bb->type != TB_REGION && bb->type != TB_ROOT) { - bb = bb->inputs[0]; - steps++; - } - - return bb == expected_dom; -} - -// unity build with all the passes -#include "lattice.h" -#include "cfg.h" -#include "gvn.h" -#include "fold.h" -#include "mem_opt.h" -#include "sroa.h" -#include "loop.h" -#include "branches.h" -#include "print_c.h" -#include "print.h" -#include "print_dumb.h" -#include "gcm.h" -#include "libcalls.h" -#include "mem2reg.h" -#include "scheduler.h" -#include "legalizer.h" - -static void violent_kill(TB_Function* f, TB_Node* n) { - // remove from GVN if we're murdering it - size_t extra = extra_bytes(n); - nl_hashset_remove2(&f->gvn_nodes, n, gvn_hash, gvn_compare); - - // remove users - FOREACH_REVERSE_N(i, 0, n->input_count) { - User* u = remove_user(n, i); - if (u) { tb_arena_free(f->arena, u, sizeof(User)); } - - n->inputs[i] = NULL; - } - - // try free - tb_arena_free(f->arena, n->inputs, n->input_cap * sizeof(TB_Node*)); - tb_arena_free(f->arena, n, sizeof(TB_Node) + extra); - - n->input_count = 0; - n->type = TB_NULL; -} - -static Lattice* value_int(TB_Passes* restrict p, TB_Node* n) { - assert(n->type == TB_INTEGER_CONST); - TB_NodeInt* num = TB_NODE_GET_EXTRA(n); - if (n->dt.type == TB_PTR) { - return num->value ? &XNULL_IN_THE_SKY : &NULL_IN_THE_SKY; - } else { - return lattice_intern(p, (Lattice){ LATTICE_INT, ._int = { num->value, num->value, ~num->value, num->value } }); - } -} - -static Lattice* value_proj(TB_Passes* restrict p, TB_Node* n) { - assert(n->type == TB_PROJ); - Lattice* l = lattice_universe_get(p, n->inputs[0]); - if (l == &TOP_IN_THE_SKY) { - return &TOP_IN_THE_SKY; - } else if (l == &BOT_IN_THE_SKY) { - return lattice_from_dt(p, n->dt); - } else { - assert(l->tag == LATTICE_TUPLE); - int index = TB_NODE_GET_EXTRA_T(n, TB_NodeProj)->index; - return l->elems[index]; - } -} - -static Lattice* value_ctrl(TB_Passes* restrict p, TB_Node* n) { - return lattice_universe_get(p, n->inputs[0]); -} - -static Lattice* value_ptr_vals(TB_Passes* restrict p, TB_Node* n) { - if (n->type == TB_LOCAL) { - return &XNULL_IN_THE_SKY; - } else { - assert(n->type == TB_SYMBOL); - return lattice_intern(p, (Lattice){ LATTICE_PTRCON, ._ptr = { TB_NODE_GET_EXTRA_T(n, TB_NodeSymbol)->sym } }); - } -} - -static Lattice* value_lookup(TB_Passes* restrict p, TB_Node* n) { - TB_NodeLookup* l = TB_NODE_GET_EXTRA(n); - TB_DataType dt = n->dt; - assert(dt.type == TB_INT); - - LatticeInt a = { l->entries[0].val, l->entries[0].val, l->entries[0].val, ~l->entries[0].val }; - FOREACH_N(i, 1, n->input_count) { - LatticeInt b = { l->entries[i].val, l->entries[i].val, l->entries[i].val, ~l->entries[i].val }; - a = lattice_meet_int(a, b, dt); - } - - return lattice_intern(p, (Lattice){ LATTICE_INT, ._int = a }); -} - -static Lattice* value_region(TB_Passes* restrict p, TB_Node* n) { - assert(cfg_is_region(n)); - FOREACH_N(i, 0, n->input_count) { - Lattice* edge = lattice_universe_get(p, n->inputs[i]); - if (edge == &CTRL_IN_THE_SKY) { - return &CTRL_IN_THE_SKY; - } - } - - return &TOP_IN_THE_SKY; -} - -static Lattice* value_phi(TB_Passes* restrict p, TB_Node* n) { - // wait for region to check first - TB_Node* r = n->inputs[0]; - if (lattice_universe_get(p, r) == &TOP_IN_THE_SKY) return &TOP_IN_THE_SKY; - - Lattice* l = lattice_universe_get(p, n); - l = lattice_dual(p, l); - - FOREACH_N(i, 1, n->input_count) { - Lattice* ctrl = lattice_universe_get(p, r->inputs[i - 1]); - if (ctrl == &CTRL_IN_THE_SKY) { - Lattice* edge = lattice_universe_get(p, n->inputs[i]); - l = lattice_meet(p, l, edge, n->dt); - } - } - - return l; -} - -static Lattice* value_select(TB_Passes* restrict p, TB_Node* n) { - Lattice* a = lattice_universe_get(p, n->inputs[2]); - Lattice* b = lattice_universe_get(p, n->inputs[3]); - return lattice_meet(p, a, b, n->dt); -} - -// this is where the vtable goes for all peepholes -#include "peeps.h" - -TB_Node* tb_pass_gvn_node(TB_Function* f, TB_Node* n) { - size_t extra = extra_bytes(n); - return tb__gvn(f, n, extra); -} - -TB_Node* tb__gvn(TB_Function* f, TB_Node* n, size_t extra) { - // try GVN, if we succeed, just delete the node and use the old copy - TB_Node* k = nl_hashset_put2(&f->gvn_nodes, n, gvn_hash, gvn_compare); - if (k && k != n) { - // remove users - FOREACH_REVERSE_N(i, 0, n->input_count) { - User* u = remove_user(n, i); - if (u) { tb_arena_free(f->arena, u, sizeof(User)); } - - n->inputs[i] = NULL; - } - - // try free - tb_arena_free(f->arena, n->inputs, n->input_cap * sizeof(TB_Node*)); - tb_arena_free(f->arena, n, sizeof(TB_Node) + extra); - return k; - } else { - return n; - } -} - -TB_Node* make_poison(TB_Function* f, TB_DataType dt) { - TB_Node* n = tb_alloc_node(f, TB_POISON, dt, 1, 0); - set_input(f, n, f->root_node, 0); - return tb__gvn(f, n, 0); -} - -TB_Node* make_int_node(TB_Function* f, TB_Passes* restrict p, TB_DataType dt, uint64_t x) { - uint64_t mask = tb__mask(dt.data); - x &= mask; - - TB_Node* n = tb_alloc_node(f, TB_INTEGER_CONST, dt, 1, sizeof(TB_NodeInt)); - TB_NodeInt* i = TB_NODE_GET_EXTRA(n); - i->value = x; - - set_input(f, n, f->root_node, 0); - - Lattice* l; - if (dt.type == TB_INT) { - l = lattice_intern(p, (Lattice){ LATTICE_INT, ._int = { x, x, ~x & mask, x } }); - } else { - l = x ? &XNULL_IN_THE_SKY : &NULL_IN_THE_SKY; - } - lattice_universe_map(p, n, l); - return tb__gvn(f, n, sizeof(TB_NodeInt)); -} - -TB_Node* dead_node(TB_Function* f, TB_Passes* p) { - TB_Node* n = tb_alloc_node(f, TB_DEAD, TB_TYPE_VOID, 1, 0); - set_input(f, n, f->root_node, 0); - lattice_universe_map(p, n, &TOP_IN_THE_SKY); - return tb__gvn(f, n, 0); -} - -TB_Node* make_proj_node(TB_Function* f, TB_DataType dt, TB_Node* src, int i) { - TB_Node* n = tb_alloc_node(f, TB_PROJ, dt, 1, sizeof(TB_NodeProj)); - set_input(f, n, src, 0); - TB_NODE_SET_EXTRA(n, TB_NodeProj, .index = i); - return n; -} - -static void remove_input(TB_Function* f, TB_Node* n, size_t i) { - // remove swap - n->input_count--; - if (n->input_count > 0) { - if (n->input_count != i) { - set_input(f, n, n->inputs[n->input_count], i); - } - set_input(f, n, NULL, n->input_count); - } -} - -void tb_pass_kill_node(TB_Function* f, TB_Node* n) { - // remove from GVN if we're murdering it - nl_hashset_remove2(&f->gvn_nodes, n, gvn_hash, gvn_compare); - - FOREACH_N(i, 0, n->input_count) { - remove_user(n, i); - n->inputs[i] = NULL; - } - - // assert(n->users == NULL && "we can't kill nodes with users, that's fucking rude"); - n->input_count = 0; - n->type = TB_NULL; -} - -static User* remove_user(TB_Node* n, int slot) { - // early out: there was no previous input - if (n->inputs[slot] == NULL) return NULL; - - TB_Node* old = n->inputs[slot]; - User* old_use = old->users; - if (old_use == NULL) return NULL; - - // remove old user (this must succeed unless our users go desync'd) - for (User* prev = NULL; old_use; prev = old_use, old_use = old_use->next) { - if (old_use->slot == slot && old_use->n == n) { - // remove - if (prev != NULL) { - prev->next = old_use->next; - } else { - old->users = old_use->next; - } - - return old_use; - } - } - - tb_panic("Failed to remove non-existent user %p from %p (slot %d)", old, n, slot); -} - -void set_input(TB_Function* f, TB_Node* n, TB_Node* in, int slot) { - // recycle the user - User* old_use = remove_user(n, slot); - - n->inputs[slot] = in; - if (in != NULL) { - add_user(f, n, in, slot, old_use); - } -} - -// we sometimes get the choice to recycle users because we just deleted something -void add_user(TB_Function* f, TB_Node* n, TB_Node* in, int slot, User* recycled) { - User* use = recycled ? recycled : TB_ARENA_ALLOC(f->arena, User); - use->next = in->users; - use->n = n; - use->slot = slot; - in->users = use; -} - -static void tb_pass_mark_users_raw(TB_Passes* restrict p, TB_Node* n) { - FOR_USERS(use, n) { - tb_pass_mark(p, use->n); - } -} - -void tb_pass_mark(TB_Passes* opt, TB_Node* n) { - worklist_push(&opt->worklist, n); -} - -void tb_pass_mark_users(TB_Passes* restrict p, TB_Node* n) { - FOR_USERS(use, n) { - tb_pass_mark(p, use->n); - TB_NodeTypeEnum type = use->n->type; - - // tuples changing means their projections did too. - if (type == TB_PROJ) { - tb_pass_mark_users(p, use->n); - } - - // (br (cmp a b)) => ... - // (or (shl a 24) (shr a 40)) => ... - // (trunc (mul a b)) => ... - // (phi ...) => ... (usually converting into branchless ops) - if ((type >= TB_CMP_EQ && type <= TB_CMP_FLE) || type == TB_SHL || type == TB_SHR || type == TB_MUL || type == TB_PHI) { - tb_pass_mark_users_raw(p, use->n); - } - } -} - -static void push_for_death(TB_Passes* restrict p, TB_Node* n) { - FOREACH_N(i, 0, n->input_count) { - TB_Node* in = n->inputs[i]; - - // we can guarentee it's got a user... that's us lmao - if (in && in->users->next == NULL) { - tb_pass_mark(p, in); - } - } -} - -static void push_all_nodes(TB_Passes* restrict p, Worklist* restrict ws, TB_Function* f) { - CUIK_TIMED_BLOCK("push_all_nodes") { - worklist_test_n_set(ws, f->root_node); - dyn_array_put(ws->items, f->root_node); - - for (size_t i = 0; i < dyn_array_length(ws->items); i++) { - TB_Node* n = ws->items[i]; - - FOR_USERS(use, n) { - TB_Node* out = use->n; - if (!worklist_test_n_set(ws, out)) { - dyn_array_put(ws->items, out); - } - } - } - - CUIK_TIMED_BLOCK("reversing") { - size_t last = dyn_array_length(ws->items) - 1; - FOREACH_N(i, 0, dyn_array_length(ws->items) / 2) { - SWAP(TB_Node*, ws->items[i], ws->items[last - i]); - } - } - } -} - -static void cool_print_type(TB_Node* n) { - TB_DataType dt = n->dt; - if (n->type != TB_ROOT && !cfg_is_region(n) && !(n->type == TB_BRANCH && n->input_count == 1)) { - if (n->type == TB_STORE) { - dt = n->inputs[3]->dt; - } else if (n->type == TB_BRANCH) { - dt = n->inputs[1]->dt; - } else if (n->type == TB_ROOT) { - dt = n->input_count > 1 ? n->inputs[1]->dt : TB_TYPE_VOID; - } else if (n->type >= TB_CMP_EQ && n->type <= TB_CMP_FLE) { - dt = TB_NODE_GET_EXTRA_T(n, TB_NodeCompare)->cmp_dt; - } - printf("."); - print_type(dt); - } -} - -void print_node_sexpr(TB_Node* n, int depth) { - if (n->type == TB_INTEGER_CONST) { - TB_NodeInt* num = TB_NODE_GET_EXTRA(n); - if (n->dt.type == TB_PTR) { - printf("%#"PRIx64, num->value); - } else { - printf("%"PRId64, tb__sxt(num->value, n->dt.data, 64)); - } - } else if (n->type == TB_SYMBOL) { - TB_Symbol* sym = TB_NODE_GET_EXTRA_T(n, TB_NodeSymbol)->sym; - if (sym->name[0]) { - printf("%s", sym->name); - } else { - printf("sym%p", sym); - } - } else if (depth >= 1) { - printf("(v%u: %s", n->gvn, tb_node_get_name(n)); - cool_print_type(n); - printf(" ...)"); - } else { - depth -= (n->type == TB_PROJ); - - printf("(v%u: %s", n->gvn, tb_node_get_name(n)); - cool_print_type(n); - FOREACH_N(i, 0, n->input_count) if (n->inputs[i]) { - if (i == 0) printf(" @"); - else printf(" "); - - print_node_sexpr(n->inputs[i], depth + 1); - } - - switch (n->type) { - case TB_ARRAY_ACCESS: - printf(" %"PRId64, TB_NODE_GET_EXTRA_T(n, TB_NodeArray)->stride); - break; - - case TB_MEMBER_ACCESS: - printf(" %"PRId64, TB_NODE_GET_EXTRA_T(n, TB_NodeMember)->offset); - break; - - case TB_PROJ: - printf(" %d", TB_NODE_GET_EXTRA_T(n, TB_NodeProj)->index); - break; - } - printf(")"); - } -} - -// Returns NULL or a modified node (could be the same node, we can stitch it back into place) -static TB_Node* idealize(TB_Passes* restrict p, TB_Function* f, TB_Node* n) { - NodeIdealize ideal = vtables[n->type].idealize; - return ideal ? ideal(p, f, n) : NULL; -} - -static TB_Node* identity(TB_Passes* restrict p, TB_Function* f, TB_Node* n) { - NodeIdentity identity = vtables[n->type].identity; - return identity ? identity(p, f, n) : n; -} - -static Lattice* value_of(TB_Passes* restrict p, TB_Node* n, bool optimistic) { - NodeValueOf value = vtables[n->type].value; - Lattice* type = value ? value(p, n) : NULL; - - // no type provided? just make a not-so-form fitting bottom type - if (type == NULL) { - if (optimistic) { - return lattice_from_dt(p, n->dt); - } else { - Lattice* old_type = lattice_universe_get(p, n); - return old_type != &TOP_IN_THE_SKY ? old_type : lattice_from_dt(p, n->dt); - } - } else { - return type; - } -} - -static bool is_dead_ctrl(TB_Passes* restrict p, TB_Node* n, bool optimistic) { - if (n->type == TB_DEAD) { - return true; - } else { - Lattice* l = lattice_universe_get(p, n); - return l == &XCTRL_IN_THE_SKY || (optimistic && l == &TOP_IN_THE_SKY); - } -} - -// converts constant Lattice into constant node -static TB_Node* try_as_const(TB_Passes* restrict p, TB_Node* n, Lattice* l, bool optimistic) { - // already a constant? - if (n->type == TB_SYMBOL || n->type == TB_INTEGER_CONST || n->type == TB_FLOAT32_CONST || n->type == TB_FLOAT64_CONST) { - return NULL; - } - - // Dead node? kill - TB_Function* f = p->f; - if (cfg_is_region(n)) { - // remove dead predeccessors - bool changes = false; - - size_t i = 0, extra_edges = 0; - while (i < n->input_count) { - if (is_dead_ctrl(p, n->inputs[i], optimistic)) { - changes = true; - remove_input(f, n, i); - - // update PHIs - FOR_USERS(use, n) { - if (use->n->type == TB_PHI && use->slot == 0) { - remove_input(f, use->n, i + 1); - } - } - } else { - i += 1; - } - } - - if (n->input_count == 0) { - tb_pass_kill_node(f, n); - return dead_node(f, p); - } else if (n->input_count == 1) { - // check for any phi nodes, because we're single entry they're all degens - User* use = n->users; - while (use != NULL) { - User* next = use->next; - if (use->n->type == TB_PHI) { - assert(use->n->input_count == 2); - subsume_node(f, use->n, use->n->inputs[1]); - } - use = next; - } - - return n->inputs[0]; - } else if (changes) { - return n; - } else { - return NULL; - } - } else if (vtables[n->type].flags & NODE_IS_CTRL) { - if (is_dead_ctrl(p, n->inputs[0], optimistic)) { - if (n->dt.type == TB_TUPLE) { - TB_Node* dead = dead_node(f, p); - while (n->users) { - TB_Node* use_n = n->users->n; - int use_i = n->users->slot; - - if (use_n->type == TB_CALLGRAPH) { - TB_Node* last = use_n->inputs[use_n->input_count - 1]; - set_input(f, use_n, NULL, use_n->input_count - 1); - if (use_i != use_n->input_count - 1) { - set_input(f, use_n, last, use_i); - } - use_n->input_count--; - } else if (use_n->type == TB_PROJ) { - TB_Node* replacement = use_n->dt.type == TB_CONTROL - ? dead - : make_poison(f, use_n->dt); - - subsume_node(f, use_n, replacement); - } else { - tb_todo(); - } - } - - return dead; - } else { - return dead_node(f, p); - } - } - } - - switch (l->tag) { - case LATTICE_INT: { - // degenerate range - if (l->_int.min == l->_int.max) { - return make_int_node(p->f, p, n->dt, l->_int.max); - } - - // all bits are known - uint64_t mask = tb__mask(n->dt.data); - if ((l->_int.known_zeros | l->_int.known_ones) == mask) { - return make_int_node(p->f, p, n->dt, l->_int.known_ones); - } - - return NULL; - } - - case LATTICE_NULL: - return make_int_node(p->f, p, n->dt, 0); - - case LATTICE_TUPLE: { - if (n->type != TB_BRANCH) return NULL; - - // check if tuple is constant path - int trues = 0; - FOREACH_N(i, 0, l->_tuple.count) { - if (l->elems[i] == &CTRL_IN_THE_SKY) { - trues++; - } - } - - if (trues == 1) { - TB_Node* dead = dead_node(p->f, p); - TB_Node* ctrl = n->inputs[0]; - - NL_ChunkedArr projs = nl_chunked_arr_alloc(tmp_arena); - FOR_USERS(u, n) { - if (u->n->type == TB_PROJ) { - nl_chunked_arr_put(&projs, u->n); - } - } - - for (NL_ArrChunk* restrict chk = projs.first; chk; chk = chk->next) { - FOREACH_N(i, 0, chk->count) { - TB_Node* proj = chk->elems[i]; - int index = TB_NODE_GET_EXTRA_T(proj, TB_NodeProj)->index; - TB_Node* in = l->elems[index] == &CTRL_IN_THE_SKY ? ctrl : dead; - - set_input(f, proj, NULL, 0); - subsume_node(p->f, proj, in); - } - } - nl_chunked_arr_reset(&projs); - - // no more projections, kill the branch - tb_pass_kill_node(p->f, n); - tb_pass_mark_users(p, dead); - return ctrl; - } else { - return NULL; - } - } - - default: return NULL; - } -} - -static void validate_node_users(TB_Node* n) { - if (n != NULL) { - FOR_USERS(use, n) { - tb_assert(use->n->inputs[use->slot] == n, "Mismatch between def-use and use-def data"); - } - } -} - -static void print_lattice(Lattice* l, TB_DataType dt) { - switch (l->tag) { - case LATTICE_BOT: printf("bot"); break; - case LATTICE_TOP: printf("top"); break; - - case LATTICE_CTRL: printf("ctrl"); break; - case LATTICE_XCTRL: printf("~ctrl"); break; - - case LATTICE_FLOAT32: printf("f32"); break; - case LATTICE_FLOAT64: printf("f64"); break; - - case LATTICE_NULL: printf("null"); break; - case LATTICE_XNULL: printf("~null"); break; - case LATTICE_BOTPTR: printf("allptr"); break; - case LATTICE_PTRCON: printf("%s", l->_ptr.sym->name); break; - - case LATTICE_MEM: printf("$mem%d", l->_mem.alias_idx); break; - - case LATTICE_TUPLE: { - printf("["); - FOREACH_N(i, 0, l->_tuple.count) { - if (i) printf(", "); - print_lattice(l->elems[i], TB_TYPE_I64); - } - printf("]"); - break; - } - case LATTICE_INT: { - assert(dt.type == TB_INT); - - printf("["); - if (l->_int.min == l->_int.max) { - printf("%"PRId64, tb__sxt(l->_int.min, dt.data, 64)); - } else if (l->_int.min == 0 && l->_int.max == 1) { - printf("bool"); - } else if (l->_int.min == INT16_MIN && l->_int.max == INT16_MAX) { - printf("i8"); - } else if (l->_int.min == 0 && l->_int.max == UINT8_MAX) { - printf("u8"); - } else if (l->_int.min == INT16_MIN && l->_int.max == INT16_MAX) { - printf("i16"); - } else if (l->_int.min == 0 && l->_int.max == UINT16_MAX) { - printf("u16"); - } else if (l->_int.min == INT32_MIN && l->_int.max == INT32_MAX) { - printf("i32"); - } else if (l->_int.min == 0 && l->_int.max == UINT32_MAX) { - printf("u32"); - } else if (l->_int.min == INT64_MIN && l->_int.max == INT64_MAX) { - printf("i64"); - } else if (l->_int.min == 0 && l->_int.max == UINT64_MAX) { - printf("u64"); - } else if (l->_int.min > l->_int.max) { - printf("%"PRId64",%"PRId64, tb__sxt(l->_int.min, dt.data, 64), tb__sxt(l->_int.max, dt.data, 64)); - } else { - printf("%"PRIu64",%"PRIu64, l->_int.min, l->_int.max); - } - - uint64_t known = l->_int.known_zeros | l->_int.known_ones; - if (known && known != UINT64_MAX) { - printf("; zeros=%#"PRIx64", ones=%#"PRIx64, l->_int.known_zeros, l->_int.known_ones); - } - printf("]"); - break; - } - - default: - break; - } -} - -static int node_sort_cmp(const void* a, const void* b) { - TB_Node* const* aa = a; - TB_Node* const* bb = b; - return aa[0]->gvn - bb[0]->gvn; -} - -// because certain optimizations apply when things are the same -// we mark ALL users including the ones who didn't get changed -// when subsuming. -TB_Node* tb_pass_peephole_node(TB_Passes* p, TB_Node* n) { - TB_Function* f = p->f; - - // idealize can modify the node, make sure it's not in the GVN pool at the time - nl_hashset_remove2(&f->gvn_nodes, n, gvn_hash, gvn_compare); - - // idealize node (this can technically run an arbitrary number of times - // but in practice we should only hit a node like once or twice) - TB_Node* k = idealize(p, f, n); - DO_IF(TB_OPTDEBUG_PEEP)(int loop_count=0); - while (k != NULL) { - DO_IF(TB_OPTDEBUG_STATS)(p->stats.rewrites++); - DO_IF(TB_OPTDEBUG_PEEP)(printf(" => \x1b[32m"), print_node_sexpr(k, 0), printf("\x1b[0m")); - - // transfer users from n -> k - if (n != k) { - subsume_node(f, n, k); - n = k; - } - - // mark post subsume since previous users of n might have - // name equality based opts. - tb_pass_mark_users(p, n); - - // try again, maybe we get another transformation - k = idealize(p, f, n); - DO_IF(TB_OPTDEBUG_PEEP)(if (++loop_count > 5) { log_warn("%p: we looping a lil too much dawg...", n); }); - } - - // pessimistic constant prop - { - Lattice* new_type = value_of(p, n, false); - - // print fancy type - DO_IF(TB_OPTDEBUG_PEEP)(printf(" => \x1b[93m["), print_lattice(new_type, n->dt), printf("]\x1b[0m")); - - TB_Node* k = try_as_const(p, n, new_type, false); - if (k != NULL) { - DO_IF(TB_OPTDEBUG_PEEP)(printf(" => \x1b[96m"), print_node_sexpr(k, 0), printf("\x1b[0m")); - - push_for_death(p, n); - subsume_node(f, n, k); - tb_pass_mark_users(p, k); - return k; - } else if (lattice_universe_map_progress(p, n, new_type)) { - tb_pass_mark_users(p, n); - } - } - - // convert into matching identity - k = identity(p, f, n); - if (n != k) { - DO_IF(TB_OPTDEBUG_STATS)(p->stats.identities++); - DO_IF(TB_OPTDEBUG_PEEP)(printf(" => \x1b[33m"), print_node_sexpr(k, 0), printf("\x1b[0m")); - - push_for_death(p, n); - subsume_node(f, n, k); - tb_pass_mark_users(p, k); - return k; - } - - // global value numbering - #if TB_OPTDEBUG_GVN - DynArray(TB_Node*) arr = dyn_array_create(TB_Node*, 64); - nl_hashset_for(p, &f->gvn_nodes) { - dyn_array_put(arr, *p); - } - qsort(arr, dyn_array_length(arr), sizeof(TB_Node*), node_sort_cmp); - dyn_array_for(i, arr) { - printf(" * "); - print_node_sexpr(arr[i], 0); - if (gvn_compare(arr[i], n)) { - printf(" <-- HERE"); - } - printf(" (hash=%#x)\n", gvn_hash(arr[i])); - } - #endif - - k = nl_hashset_put2(&f->gvn_nodes, n, gvn_hash, gvn_compare); - if (k && (k != n)) { - DO_IF(TB_OPTDEBUG_STATS)(p->stats.gvn_hit++); - DO_IF(TB_OPTDEBUG_PEEP)(printf(" => \x1b[95mGVN v%u\x1b[0m", k->gvn)); - - subsume_node(f, n, k); - tb_pass_mark_users(p, k); - return k; - } else { - DO_IF(TB_OPTDEBUG_STATS)(p->stats.gvn_miss++); - } - - return n; -} - -static void subsume_node2(TB_Function* f, TB_Node* n, TB_Node* new_n) { - CUIK_TIMED_BLOCK("subsume") { - User* use = n->users; - while (use != NULL) { - tb_assert(use->n->inputs[use->slot] == n, "Mismatch between def-use and use-def data"); - - // set_input will delete 'use' so we can't use it afterwards - TB_Node* use_n = use->n; - User* next = use->next; - - set_input(f, use->n, new_n, use->slot); - use = next; - } - } -} - -static void subsume_node(TB_Function* f, TB_Node* n, TB_Node* new_n) { - subsume_node2(f, n, new_n); - tb_pass_kill_node(f, n); -} - -TB_Passes* tb_pass_enter(TB_Function* f, TB_Arena* arena) { - assert(f->root_node && "missing root node"); - - TB_Passes* p = tb_platform_heap_alloc(sizeof(TB_Passes)); - *p = (TB_Passes){ .f = f }; - - f->arena = arena; - - verify_tmp_arena(p); - worklist_alloc(&p->worklist, f->node_count); - - // generate work list (put everything) - CUIK_TIMED_BLOCK("gen worklist") { - push_all_nodes(p, &p->worklist, f); - DO_IF(TB_OPTDEBUG_STATS)(p->stats.initial = worklist_popcount(&p->worklist)); - } - - DO_IF(TB_OPTDEBUG_PEEP)(log_debug("%s: starting passes with %d nodes", f->super.name, f->node_count)); - - return p; -} - -void tb_pass_sroa(TB_Passes* p) { - CUIK_TIMED_BLOCK("sroa") { - verify_tmp_arena(p); - - TB_Function* f = p->f; - Worklist* ws = &p->worklist; - - int pointer_size = f->super.module->codegen->pointer_size; - TB_Node* root = f->root_node; - - // write initial locals - FOR_USERS(u, root) { - if (u->n->type == TB_LOCAL) { - worklist_push(&p->worklist, u->n); - } - } - - // i think the SROA'd pieces can't themselves split more? that should something we check - size_t local_count = dyn_array_length(ws->items); - for (size_t i = 0; i < local_count; i++) { - assert(ws->items[i]->type == TB_LOCAL); - sroa_rewrite(p, pointer_size, root, ws->items[i]); - } - } -} - -typedef union { - uint64_t i; - User* ctrl; -} Value; - -typedef struct { - Worklist* ws; - Value* vals; - bool* ready; - int phi_i; -} Interp; - -static Value* in_val(Interp* vm, TB_Node* n, int i) { return &vm->vals[n->inputs[i]->gvn]; } -static Value eval(Interp* vm, TB_Node* n) { - printf(" EVAL v%u\n", n->gvn); - switch (n->type) { - case TB_INTEGER_CONST: return (Value){ .i = TB_NODE_GET_EXTRA_T(n, TB_NodeInt)->value }; - - case TB_ADD: { - uint64_t a = in_val(vm, n, 1)->i; - uint64_t b = in_val(vm, n, 2)->i; - return (Value){ .i = a + b }; - } - - case TB_CMP_SLT: { - uint64_t a = in_val(vm, n, 1)->i; - uint64_t b = in_val(vm, n, 2)->i; - return (Value){ .i = a < b }; - } - - case TB_BRANCH: { - TB_NodeBranch* br = TB_NODE_GET_EXTRA(n); - uint64_t key = in_val(vm, n, 1)->i; - int index = 0; - - FOREACH_N(i, 0, br->succ_count - 1) { - if (key == br->keys[i].key) { - index = i + 1; - break; - } - } - - User* ctrl = proj_with_index(n, index); - return (Value){ .ctrl = ctrl }; - } - - case TB_REGION: - case TB_NATURAL_LOOP: - case TB_AFFINE_LOOP: - return (Value){ .ctrl = cfg_next_user(n) }; - - case TB_PROJ: - if (n->dt.type == TB_MEMORY) { - return (Value){ .i = 0 }; - } else if (n->dt.type == TB_CONTROL) { - return (Value){ .ctrl = cfg_next_user(n) }; - } else { - tb_todo(); - } - - // control nodes - case TB_ROOT: { - uint64_t v = in_val(vm, n, 3)->i; - - printf("END %"PRIu64"\n", v); - return (Value){ .ctrl = NULL }; - } - - default: tb_todo(); - } -} - -static bool is_ready(Interp* vm, TB_Node* n) { - FOREACH_N(i, 1, n->input_count) { - if (!vm->ready[n->inputs[i]->gvn]) { - return false; - } - } - - return true; -} - -static void dirty_deps(Interp* vm, TB_Node* n) { - printf(" DIRTY v%u\n", n->gvn); - vm->ready[n->gvn] = false; - - FOR_USERS(u, n) { - if (u->n->type != TB_PHI && vm->ready[u->n->gvn]) { - dirty_deps(vm, u->n); - } - } -} - -void dummy_interp(TB_Passes* p) { - TB_Function* f = p->f; - TB_Arena* arena = get_temporary_arena(f->super.module); - - TB_Node* ip = cfg_next_control(f->root_node); - - Interp vm = { - .ws = &p->worklist, - .vals = tb_arena_alloc(arena, f->node_count * sizeof(Value)), - .ready = tb_arena_alloc(arena, f->node_count * sizeof(bool)) - }; - - int last_edge = 0; - while (ip) { - printf("IP = v%u\n", ip->gvn); - - worklist_clear(&p->worklist); - - // push all direct users of the parent's users (our antideps) - FOR_USERS(u, ip->inputs[last_edge]) { - if (is_ready(&vm, u->n)) { - worklist_push(&p->worklist, u->n); - } - } - - if (!cfg_is_region(ip)) { - FOREACH_N(i, 1, ip->input_count) { - worklist_push(&p->worklist, ip->inputs[i]); - } - } - - if (is_ready(&vm, ip)) { - worklist_push(&p->worklist, ip); - } - - size_t i = 0; - for (; i < dyn_array_length(p->worklist.items); i++) { - TB_Node* n = p->worklist.items[i]; - if (n->type == TB_PHI) continue; - - vm.vals[n->gvn] = eval(&vm, n); - vm.ready[n->gvn] = true; - - FOR_USERS(u, n) { - if (is_ready(&vm, u->n)) { - worklist_push(&p->worklist, u->n); - } - } - - // advance now - if (n == ip) { - dyn_array_set_length(p->worklist.items, i + 1); - break; - } - } - - if (cfg_is_region(ip)) { - vm.vals[ip->gvn] = eval(&vm, ip); - vm.ready[ip->gvn] = true; - } else { - assert(is_ready(&vm, ip)); - } - - User* succ = vm.vals[ip->gvn].ctrl; - if (succ == NULL) { - break; - } - - last_edge = succ->slot; - ip = succ->n; - - if (cfg_is_region(ip)) { - FOR_USERS(u, ip) { - TB_Node* phi = u->n; - if (phi->type == TB_PHI) { - TB_Node* in = phi->inputs[1 + last_edge]; - if (is_ready(&vm, in)) { - worklist_push(&p->worklist, in); - } - } - } - - for (; i < dyn_array_length(p->worklist.items); i++) { - TB_Node* n = p->worklist.items[i]; - if (n->type == TB_PHI) continue; - - vm.vals[n->gvn] = eval(&vm, n); - vm.ready[n->gvn] = true; - } - - FOR_USERS(u, ip) { - TB_Node* phi = u->n; - if (phi->type == TB_PHI) { - printf(" PHI = v%u (v%u)\n", phi->gvn, phi->inputs[1 + last_edge]->gvn); - - Value* v = &vm.vals[phi->inputs[1 + last_edge]->gvn]; - vm.vals[phi->gvn] = *v; - - dirty_deps(&vm, phi); - vm.ready[phi->gvn] = true; - } - } - } - } -} - -static void push_non_bottoms(TB_Passes* restrict p, TB_Node* n) { - // if it's a bottom there's no more steps it can take, don't recompute it - Lattice* l = lattice_universe_get(p, n); - if (l != lattice_from_dt(p, n->dt)) { - // printf(" * MARK v%u\n", n->gvn); - worklist_push(&p->worklist, n); - } else { - // printf(" * ALREADY THERE v%u\n", n->gvn); - } -} - -static void push_non_bottom_users(TB_Passes* restrict p, TB_Node* n) { - FOR_USERS(use, n) { - push_non_bottoms(p, use->n); - - if (cfg_is_region(use->n)) { - FOR_USERS(phi, use->n) if (phi->n->type == TB_PHI) { - push_non_bottoms(p, phi->n); - } - } - } -} - -static void tb_pass_const(TB_Passes* p) { - assert(dyn_array_length(p->worklist.items) == 0); - - TB_Function* f = p->f; - - // Pass 1: find constants. - // reset all types into TOP - FOREACH_N(i, 0, p->type_cap) { - p->types[i] = &TOP_IN_THE_SKY; - } - - // start at ROOT - p->types[f->root_node->gvn] = lattice_tuple_from_node(p, f->root_node); - FOR_USERS(use, f->root_node) { - tb_pass_mark(p, use->n); - } - - CUIK_TIMED_BLOCK("sccp") { - TB_Node* n; - while ((n = worklist_pop(&p->worklist))) { - Lattice* old_type = lattice_universe_get(p, n); - Lattice* new_type = value_of(p, n, true); - if (old_type != new_type) { - DO_IF(TB_OPTDEBUG_SCCP)(printf("TYPE t=%d? ", ++p->stats.time), print_node_sexpr(n, 0), printf(" => \x1b[93m["), print_lattice(new_type, n->dt), printf("]\x1b[0m\n")); - - lattice_universe_map(p, n, new_type); - push_non_bottom_users(p, n); - } else { - // DO_IF(TB_OPTDEBUG_SCCP)(printf("TYPE t=%d? ", ++p->stats.time), print_node_sexpr(n, 0), printf(" => \x1b[93m["), print_lattice(new_type, n->dt), printf("] (STILL)\x1b[0m\n")); - } - } - } - - // Pass 2: ok replace with constants now - // we need a separate worklist for SCCP - Worklist ws = { 0 }; - worklist_alloc(&ws, f->node_count); - - // root node can't constant fold - worklist_test_n_set(&ws, f->root_node); - dyn_array_put(ws.items, f->root_node); - - for (size_t i = 0; i < dyn_array_length(ws.items); i++) { - TB_Node* n = ws.items[i]; - TB_Node* k = try_as_const(p, n, lattice_universe_get(p, n), true); - - // DO_IF(TB_OPTDEBUG_SCCP)(printf("CONST t=%d? ", ++p->stats.time), print_node_sexpr(n, 0)); - if (k != NULL) { - // DO_IF(TB_OPTDEBUG_SCCP)(printf(" => \x1b[96m"), print_node_sexpr(k, 0), printf("\x1b[0m")); - - subsume_node(f, n, k); - tb_pass_mark_users(p, n); - n = k; - } - // DO_IF(TB_OPTDEBUG_SCCP)(printf("\n")); - - FOR_USERS(use, n) { - TB_Node* out = use->n; - if (!worklist_test_n_set(&ws, out)) { - dyn_array_put(ws.items, out); - } - } - } - worklist_free(&ws); -} - -void tb_pass_optimize(TB_Passes* p) { - tb_pass_peephole(p); - - // mostly just SSA construction from memory edges - tb_pass_locals(p); - - // const prop leaves work for the peephole optimizer - tb_pass_const(p); - tb_pass_peephole(p); - - tb_pass_loop(p); - tb_pass_peephole(p); - - // tb_dumb_print(p->f, p); - // dummy_interp(p); -} - -static size_t tb_pass_update_cfg(TB_Passes* p, Worklist* ws, bool preserve) { - TB_Function* f = p->f; - - p->cfg = tb_compute_rpo2(f, ws); - tb_compute_dominators2(f, ws, p->cfg); - - if (!preserve) { - tb_free_cfg(&p->cfg); - } - - return p->cfg.block_count; -} - -void tb_pass_prep(TB_Passes* p) { - TB_Function* f = p->f; - - if (p->types == NULL) { - TB_ThreadInfo* info = tb_thread_info(f->super.module); - - CUIK_TIMED_BLOCK("allocate type array") { - size_t count = (f->node_count + 63ull) & ~63ull; - p->type_interner = nl_hashset_alloc(64); - p->type_cap = count; - p->types = tb_platform_heap_alloc(count * sizeof(Lattice*)); - - FOREACH_N(i, 0, count) { - p->types[i] = NULL; - } - - nl_hashset_put2(&p->type_interner, &BOT_IN_THE_SKY, lattice_hash, lattice_cmp); - nl_hashset_put2(&p->type_interner, &TOP_IN_THE_SKY, lattice_hash, lattice_cmp); - nl_hashset_put2(&p->type_interner, &CTRL_IN_THE_SKY, lattice_hash, lattice_cmp); - nl_hashset_put2(&p->type_interner, &XCTRL_IN_THE_SKY, lattice_hash, lattice_cmp); - nl_hashset_put2(&p->type_interner, &NULL_IN_THE_SKY, lattice_hash, lattice_cmp); - nl_hashset_put2(&p->type_interner, &XNULL_IN_THE_SKY, lattice_hash, lattice_cmp); - nl_hashset_put2(&p->type_interner, &PTR_IN_THE_SKY, lattice_hash, lattice_cmp); - nl_hashset_put2(&p->type_interner, &FALSE_IN_THE_SKY, lattice_hash, lattice_cmp); - nl_hashset_put2(&p->type_interner, &TRUE_IN_THE_SKY, lattice_hash, lattice_cmp); - - // place ROOT type - p->root_mem = lattice_new_alias(p); - - // pessimistic constant prop - CUIK_TIMED_BLOCK("pessimize") { - FOREACH_N(i, 0, dyn_array_length(p->worklist.items)) { - TB_Node* n = p->worklist.items[i]; - p->types[n->gvn] = lattice_from_dt(p, n->dt); - } - } - - p->types[f->root_node->gvn] = lattice_tuple_from_node(p, f->root_node); - } - } -} - -void tb_pass_peephole(TB_Passes* p) { - verify_tmp_arena(p); - TB_Function* f = p->f; - - // allocates shit, imporant shit - tb_pass_prep(p); - - CUIK_TIMED_BLOCK("peephole") { - TB_Node* n; - while ((n = worklist_pop(&p->worklist))) { - DO_IF(TB_OPTDEBUG_STATS)(p->stats.peeps++); - DO_IF(TB_OPTDEBUG_PEEP)(printf("PEEP t=%d? ", ++p->stats.time), print_node_sexpr(n, 0)); - - // must've dead sometime between getting scheduled and getting here. - if (n->type != TB_PROJ && n->users == NULL) { - DO_IF(TB_OPTDEBUG_PEEP)(printf(" => \x1b[196mKILL\x1b[0m\n")); - push_for_death(p, n); - tb_pass_kill_node(f, n); - } else { - tb_pass_peephole_node(p, n); - DO_IF(TB_OPTDEBUG_PEEP)(printf("\n")); - } - } - } -} - -void tb_pass_exit(TB_Passes* p) { - CUIK_TIMED_BLOCK("exit") { - verify_tmp_arena(p); - - TB_Function* f = p->f; - - #if TB_OPTDEBUG_STATS - push_all_nodes(p, &p->worklist, f); - int final_count = worklist_popcount(&p->worklist); - double factor = ((double) final_count / (double) p->stats.initial) * 100.0; - - printf("%s: stats:\n", f->super.name); - printf(" %4d -> %4d nodes (%.2f%%)\n", p->stats.initial, final_count, factor); - printf(" %4d GVN hit %4d GVN miss\n", p->stats.gvn_hit, p->stats.gvn_miss); - printf(" %4d peepholes %4d rewrites %4d identities\n", p->stats.peeps, p->stats.rewrites, p->stats.identities); - #endif - - worklist_free(&p->worklist); - - if (p->types != NULL) { - nl_hashset_free(p->type_interner); - tb_platform_heap_free(p->types); - } - - tb_arena_clear(tmp_arena); - tb_platform_heap_free(p); - } -} - -typedef struct { - bool on_stack; - int index, low_link; -} SCCNode; - -typedef struct { - TB_Arena* arena; - size_t fn_count; - NL_Table nodes; - - size_t stk_cnt; - TB_Function** stk; - - int index; -} SCC; - -static TB_Function* static_call_site(TB_Node* n) { - // is this call site a static function call - assert(n->type == TB_CALL || n->type == TB_TAILCALL); - if (n->inputs[2]->type != TB_SYMBOL) return NULL; - - TB_Symbol* target = TB_NODE_GET_EXTRA_T(n->inputs[2], TB_NodeSymbol)->sym; - if (atomic_load_explicit(&target->tag, memory_order_relaxed) != TB_SYMBOL_FUNCTION) return NULL; - - return (TB_Function*) target; -} - -static SCCNode* scc_walk(SCC* restrict scc, IPOSolver* ipo, TB_Function* f) { - SCCNode* n = tb_arena_alloc(scc->arena, sizeof(SCCNode)); - n->index = scc->index; - n->low_link = scc->index; - n->on_stack = true; - scc->index += 1; - nl_table_put(&scc->nodes, f, n); - - scc->stk[scc->stk_cnt++] = f; - - // consider the successors - TB_Node* callgraph = f->root_node->inputs[0]; - assert(callgraph->type == TB_CALLGRAPH); - FOREACH_N(i, 1, callgraph->input_count) { - TB_Node* call = callgraph->inputs[i]; - TB_Function* target = static_call_site(call); - if (target != NULL) { - SCCNode* succ = nl_table_get(&scc->nodes, target); - if (succ == NULL) { - succ = scc_walk(scc, ipo, target); - if (n->low_link > succ->low_link) { n->low_link = succ->low_link; } - } else if (succ->on_stack) { - if (n->low_link > succ->index) { n->low_link = succ->index; } - } - } - } - - // we're the root, construct an SCC - if (n->low_link == n->index) { - TB_Function* kid_f; - do { - assert(scc->stk_cnt > 0); - kid_f = scc->stk[--scc->stk_cnt]; - - SCCNode* kid_n = nl_table_get(&scc->nodes, kid_f); - kid_n->on_stack = false; - ipo->ws[ipo->ws_cnt++] = kid_f; - } while (kid_f != f); - } - - return n; -} - -static void inline_into(TB_Arena* arena, TB_Function* f, TB_Node* call_site, TB_Function* kid); -bool tb_module_ipo(TB_Module* m) { - // fill initial worklist with all external function calls :) - // - // two main things we wanna know are if something is alive and when to inline (eventually - // we can incorporate IPSCCP) - SCC scc = { 0 }; - scc.arena = get_temporary_arena(m); - scc.fn_count = m->symbol_count[TB_SYMBOL_FUNCTION]; - - IPOSolver ipo = { 0 }; - ipo.ws_cap = scc.fn_count; - ipo.ws = tb_arena_alloc(scc.arena, scc.fn_count * sizeof(TB_Function*)); - - CUIK_TIMED_BLOCK("build SCC") { - TB_ArenaSavepoint sp = tb_arena_save(scc.arena); - scc.stk = tb_arena_alloc(scc.arena, scc.fn_count * sizeof(TB_Function*)); - scc.nodes = nl_table_arena_alloc(scc.arena, scc.fn_count); - - // build strongly connected components - TB_ThreadInfo* info = atomic_load_explicit(&m->first_info_in_module, memory_order_relaxed); - while (info != NULL) { - TB_Symbol** syms = (TB_Symbol**) info->symbols.data; - if (syms == NULL) continue; - - FOREACH_N(i, 0, 1ull << info->symbols.exp) { - TB_Symbol* s = syms[i]; - if (s == NULL || s == NL_HASHSET_TOMB) continue; - if (atomic_load_explicit(&s->tag, memory_order_relaxed) != TB_SYMBOL_FUNCTION) continue; - - if (nl_table_get(&scc.nodes, s) == NULL) { - scc_walk(&scc, &ipo, (TB_Function*) s); - } - } - - info = info->next_in_module; - } - tb_arena_restore(scc.arena, sp); - } - - // we've got our bottom up ordering on the worklist... start trying to inline callsites - bool progress = false; - - TB_OPTDEBUG(INLINE)(printf("BOTTOM-UP ORDER:\n")); - FOREACH_N(i, 0, ipo.ws_cnt) { - TB_Function* f = ipo.ws[i]; - - TB_OPTDEBUG(INLINE)(printf("* FUNCTION: %s\n", f->super.name)); - - TB_Node* callgraph = f->root_node->inputs[0]; - assert(callgraph->type == TB_CALLGRAPH); - FOREACH_N(i, 1, callgraph->input_count) { - TB_Node* call = callgraph->inputs[i]; - TB_Function* target = static_call_site(call); - if (target == NULL) { - continue; - } - - // TODO(NeGate): do some heuristics on inlining - // TB_OPTDEBUG(INLINE)(printf(" -> %s (from v%u)\n", target->super.name, call->gvn)); - // inline_into(scc.arena, f, call, target); - // progress = true; - } - } - - return progress; -} - -static TB_Node* inline_clone_node(TB_Function* f, TB_Node* call_site, TB_Node** clones, TB_Node* n) { - // special cases - if (n->type == TB_PROJ && n->inputs[0]->type == TB_ROOT) { - // this is a parameter, just hook it directly to the inputs of - // the callsite. - // - // 0:ctrl, 1:mem, 2:rpc, 3... params - int index = TB_NODE_GET_EXTRA_T(n, TB_NodeProj)->index; - clones[n->gvn] = call_site->inputs[index]; - - assert(clones[n->gvn]); - return clones[n->gvn]; - } else if (clones[n->gvn] != NULL) { - return clones[n->gvn]; - } - - size_t extra = extra_bytes(n); - TB_Node* cloned = tb_alloc_node(f, n->type, n->dt, n->input_count, extra); - - // clone extra data (i hope it's that easy lol) - memcpy(cloned->extra, n->extra, extra); - clones[n->gvn] = cloned; - - // fill cloned edges - FOREACH_N(i, 0, n->input_count) if (n->inputs[i]) { - TB_Node* in = inline_clone_node(f, call_site, clones, n->inputs[i]); - - cloned->inputs[i] = in; - add_user(f, cloned, in, i, NULL); - } - - #if TB_OPTDEBUG_INLINE - printf("CLONE: "); - print_node_sexpr(n, 0); - printf(" => "); - print_node_sexpr(cloned, 0); - printf("\n"); - #endif - - return cloned; - - /* TB_Node* k = tb__gvn(f, cloned, extra); - if (k != cloned) { - #if TB_OPTDEBUG_INLINE - printf(" => GVN"); - #endif - } - printf("\n"); - return = cloned; */ -} - -static void inline_into(TB_Arena* arena, TB_Function* f, TB_Node* call_site, TB_Function* kid) { - TB_ArenaSavepoint sp = tb_arena_save(arena); - TB_Node** clones = tb_arena_alloc(arena, kid->node_count * sizeof(TB_Node*)); - memset(clones, 0, kid->node_count * sizeof(TB_Node*)); - - // find all nodes - Worklist ws = { 0 }; - worklist_alloc(&ws, kid->node_count); - { - worklist_test_n_set(&ws, kid->root_node); - dyn_array_put(ws.items, kid->root_node); - - for (size_t i = 0; i < dyn_array_length(ws.items); i++) { - TB_Node* n = ws.items[i]; - - FOR_USERS(u, n) { - TB_Node* out = u->n; - if (!worklist_test_n_set(&ws, out)) { - dyn_array_put(ws.items, out); - } - } - } - } - - // clone all nodes in kid into f (GVN while we're at it) - FOREACH_REVERSE_N(i, 0, dyn_array_length(ws.items)) { - inline_clone_node(f, call_site, clones, ws.items[i]); - } - worklist_free(&ws); - - tb_pass_kill_node(f, call_site); - - { - // TODO(NeGate): region-ify the exit point - TB_Node* kid_root = clones[kid->root_node->gvn]; - assert(kid_root->type == TB_ROOT); - assert(kid_root->input_count == 2); - - TB_Node* ret = kid_root->inputs[1]; - assert(ret->type == TB_RETURN); - - User* users = call_site->users; - while (users) { - User* next = users->next; - TB_Node* use_n = users->n; - int use_i = users->slot; - - // replace returning projection with one of our return vals - if (use_n->type == TB_PROJ) { - int index = TB_NODE_GET_EXTRA_T(use_n, TB_NodeProj)->index; - subsume_node(f, use_n, ret->inputs[index]); - } - users = next; - } - - subsume_node(f, kid_root, f->root_node); - } - - // kill edge in callgraph - TB_Node* callgraph = f->root_node->inputs[0]; - assert(callgraph->type == TB_CALLGRAPH); - - FOREACH_N(i, 1, callgraph->input_count) { - if (callgraph->inputs[i] == call_site) { - set_input(f, callgraph, callgraph->inputs[callgraph->input_count - 1], i); - callgraph->input_count--; - break; - } - } - - tb_arena_restore(arena, sp); -} - diff --git a/vendor/tb/src/opt/passes.h b/vendor/tb/src/opt/passes.h deleted file mode 100644 index 712aa8bf..00000000 --- a/vendor/tb/src/opt/passes.h +++ /dev/null @@ -1,551 +0,0 @@ -#pragma once -#include "../tb_internal.h" -#include "properties.h" - -enum { - FAST_IDOM_LIMIT = 20 -}; - -#define TB_OPTDEBUG_STATS 0 -#define TB_OPTDEBUG_PEEP 0 -#define TB_OPTDEBUG_SCCP 0 -#define TB_OPTDEBUG_LOOP 0 -#define TB_OPTDEBUG_SROA 0 -#define TB_OPTDEBUG_GCM 0 -#define TB_OPTDEBUG_MEM2REG 0 -#define TB_OPTDEBUG_CODEGEN 0 -#define TB_OPTDEBUG_DATAFLOW 0 -#define TB_OPTDEBUG_INLINE 0 -#define TB_OPTDEBUG_REGALLOC 0 -#define TB_OPTDEBUG_GVN 0 - -// for toggling ANSI colors -#define TB_OPTDEBUG_ANSI 1 - -#define TB_OPTDEBUG(cond) CONCAT(DO_IF_, CONCAT(TB_OPTDEBUG_, cond)) - -#define DO_IF(cond) CONCAT(DO_IF_, cond) -#define DO_IF_0(...) -#define DO_IF_1(...) __VA_ARGS__ - -#define BB_LOW_FREQ 1e-4 - -#define FOR_USERS(u, n) for (User* u = n->users; u; u = u->next) - -//////////////////////////////// -// SCCP -//////////////////////////////// -typedef struct Lattice Lattice; - -// TODO(NeGate): implement dual? from there i can do join with -// dual(dual(x) ^ dual(y)) = join(x, y) -typedef struct { - uint64_t min, max; - - // for known bit analysis - uint64_t known_zeros; - uint64_t known_ones; -} LatticeInt; - -// a simplification of the set of all pointers (or floats) -typedef enum { - LATTICE_UNKNOWN, // bottom aka {nan, non-nan} or for pointers {null, non-null} - - LATTICE_KNOWN_FALSE = 1, // {false} - LATTICE_KNOWN_TRUE, // {true} -} LatticeTrifecta; - -typedef struct { - LatticeTrifecta trifecta; -} LatticeFloat; - -typedef struct { - TB_Symbol* sym; -} LatticePtrConst; - -typedef struct { - size_t count; -} LatticeTuple; - -typedef struct { - int alias_idx; -} LatticeMem; - -// Represents the fancier type system within the optimizer, it's -// all backed by my shitty understanding of lattice theory -struct Lattice { - enum { - LATTICE_BOT, // bot ^ x = bot - LATTICE_TOP, // top ^ x = x - - LATTICE_INT, - LATTICE_FLOAT32, - LATTICE_FLOAT64, - LATTICE_TUPLE, - - // pointers: - // top - // / \ - // / \ - // / /|\ - // | / | \ - // | a b ... # ptrcon - // | \ | / - // null ~null - // \ / - // botptr - LATTICE_BOTPTR, - LATTICE_NULL, - LATTICE_XNULL, - LATTICE_PTRCON, - - // memory: - // top - // / | \ - // a b ... - // \ | / - // bot - // - // alias idx on each memory type. - LATTICE_MEM, - - // control tokens - LATTICE_CTRL, - LATTICE_XCTRL, - } tag; - uint32_t pad; - union { - LatticeInt _int; - LatticeFloat _float; - LatticePtrConst _ptr; - LatticeTuple _tuple; - LatticeMem _mem; - }; - Lattice* elems[]; -}; - -//////////////////////////////// -// CFG -//////////////////////////////// -typedef struct { - size_t stride; - uint64_t arr[]; -} TB_DominanceFrontiers; - -static void tb_dommy_fronts_put(TB_DominanceFrontiers* df, size_t i, size_t j) { - size_t word_i = j / 64; - df->arr[i * df->stride + word_i] |= 1ull << (j % 64); -} - -static bool tb_dommy_fronts_get(TB_DominanceFrontiers* df, size_t i, size_t j) { - size_t word_i = j / 64; - return df->arr[i * df->stride + word_i] & (1ull << (j % 64)); -} - -typedef struct { - TB_Node *phi, *n; - int dst, src; -} PhiVal; - -typedef struct TB_BasicBlock TB_BasicBlock; -struct TB_BasicBlock { - TB_BasicBlock* dom; - - // whatever regions say, everything else gets 1 - float freq; - - TB_Node* start; - TB_Node* end; - int id, dom_depth; - - NL_HashSet items; -}; - -typedef struct TB_CFG { - size_t block_count; - NL_Map(TB_Node*, TB_BasicBlock) node_to_block; -} TB_CFG; - -//////////////////////////////// -// Core optimizer -//////////////////////////////// -typedef struct { - DynArray(TB_Node*) items; - - // uses gvn as key - size_t visited_cap; // in words - uint64_t* visited; -} Worklist; - -typedef void (*TB_Scheduler)(TB_Passes* passes, TB_CFG* cfg, Worklist* ws, DynArray(PhiVal)* phi_vals, TB_BasicBlock* bb, TB_Node* end); - -struct TB_Passes { - TB_Function* f; - - // we use this to verify that we're on the same thread - // for the entire duration of the TB_Passes. - TB_ThreadInfo* pinned_thread; - - Worklist worklist; - - // tracks the fancier type system - // hash-consing because there's a lot of - // redundant types we might construct. - struct { - NL_HashSet type_interner; - - // for memory alias indices - int alias_n; - Lattice* root_mem; - - // track a lattice per node (basically all get one so a compact array works) - size_t type_cap; - Lattice** types; - }; - - // might be out of date if you haven't called tb_pass_update_cfg - TB_CFG cfg; - - // value number -> TB_BasicBlock* - TB_BasicBlock** scheduled; - - // debug shit: - TB_Node* error_n; - - // nice stats - struct { - #if TB_OPTDEBUG_PEEP || TB_OPTDEBUG_SCCP - int time; - #endif - - #if TB_OPTDEBUG_STATS - int initial; - int gvn_hit, gvn_miss; - int peeps, identities, rewrites; - #endif - } stats; -}; - -typedef struct { - TB_Module* mod; - NL_HashSet visited; - - size_t ws_cap; - size_t ws_cnt; - TB_Function** ws; -} IPOSolver; - -static uint64_t tb__mask(uint64_t bits) { - return ~UINT64_C(0) >> (64 - bits); -} - -static bool ctrl_out_as_cproj_but_not_branch(TB_Node* n) { - return n->type == TB_CALL || n->type == TB_TAILCALL || n->type == TB_SYSCALL || n->type == TB_READ || n->type == TB_WRITE; -} - -static bool cfg_is_mproj(TB_Node* n) { - return n->type == TB_PROJ && n->dt.type == TB_MEMORY; -} - -// includes tuples which have control flow -static bool cfg_is_control(TB_Node* n) { - // easy case - if (n->dt.type == TB_CONTROL) return true; - if (n->dt.type != TB_TUPLE) return false; - - // harder case is figuring out which tuples have control outputs (without manually - // checking which is annoying and slow) - // - // branch, debugbreak, trap, unreachable, dead OR call, syscall, safepoint - return n->type == TB_ROOT || (n->type >= TB_BRANCH && n->type <= TB_UNREACHABLE) || (n->type >= TB_CALL && n->type <= TB_SAFEPOINT_POLL); -} - -static bool cfg_is_bb_entry(TB_Node* n) { - if (n->type == TB_REGION) { - return true; - } else if (n->type == TB_PROJ && (n->inputs[0]->type == TB_ROOT || n->inputs[0]->type == TB_BRANCH)) { - // Start's control proj or a branch target - return true; - } else { - return false; - } -} - -static bool cfg_underneath(TB_CFG* cfg, TB_Node* a, TB_BasicBlock* bb) { - // follow until we hit a terminator - for (;;) { - a = a->inputs[0]; - - ptrdiff_t search = nl_map_get(cfg->node_to_block, a); - if (search >= 0) { - return &cfg->node_to_block[search].v == bb; - } - } -} - -static bool is_mem_out_op(TB_Node* n) { - return n->dt.type == TB_MEMORY || (n->type >= TB_STORE && n->type <= TB_ATOMIC_CAS) || (n->type >= TB_CALL && n->type <= TB_TAILCALL) || n->type == TB_SPLITMEM || n->type == TB_MERGEMEM; -} - -static bool is_pinned(TB_Node* n) { - return (n->type >= TB_ROOT && n->type <= TB_SAFEPOINT_POLL) || n->type == TB_PROJ; -} - -static bool is_mem_in_op(TB_Node* n) { - return is_mem_out_op(n) || n->type == TB_SAFEPOINT_POLL || n->type == TB_LOAD; -} - -static bool is_mem_only_in_op(TB_Node* n) { - return n->type == TB_SAFEPOINT_POLL || n->type == TB_LOAD; -} - -static bool cfg_critical_edge(TB_Node* proj, TB_Node* n) { - assert(proj->type == TB_PROJ); - - // multi-user proj, this means it's basically a BB - if (proj->users->next != NULL || proj->users->n->type != TB_REGION) { - return true; - } - - assert(n->type == TB_BRANCH); - TB_Node* r = proj->users->n; - if (r->type == TB_REGION) { - FOR_USERS(u, r) { - if (u->n->type == TB_PHI && u->n->dt.type != TB_MEMORY) { - return true; - } - } - } - - return false; -} - -//////////////////////////////// -// CFG analysis -//////////////////////////////// -// if we see a branch projection, it may either be a BB itself -// or if it enters a REGION directly, then that region is the BB. -static TB_Node* cfg_next_bb_after_cproj(TB_Node* n) { - assert(n->type == TB_PROJ && n->inputs[0]->type == TB_BRANCH); - if (!cfg_critical_edge(n, n->inputs[0])) { - return n->users->n; - } else { - return n; - } -} - -static TB_Node* cfg_next_region_control(TB_Node* n) { - if (n->type != TB_REGION) { - FOR_USERS(u, n) { - if (u->n->type == TB_REGION && u->n->input_count == 1) { - return u->n; - } - } - } - - return n; -} - -static User* proj_with_index(TB_Node* n, int i) { - FOR_USERS(u, n) { - TB_NodeProj* p = TB_NODE_GET_EXTRA(u->n); - if (p->index == i) { - return u; - } - } - - return NULL; -} - -static User* cfg_next_user(TB_Node* n) { - FOR_USERS(u, n) { - if (cfg_is_control(u->n)) { - return u; - } - } - - return NULL; -} - -static bool cfg_basically_empty_only_mem_phis(TB_Node* n) { - if (n->type == TB_PROJ && n->users->next == NULL && n->users->n->type == TB_REGION) { - FOR_USERS(u, n) { - if (u->n->type == TB_PHI && u->n->dt.type != TB_MEMORY) { - return false; - } - } - - return true; - } - - return false; -} - -static bool cfg_has_phis(TB_Node* n) { - if (n->type != TB_REGION) { - return false; - } - - FOR_USERS(u, n) { - if (u->n->type == TB_PHI) { - return true; - } - } - - return false; -} - -static bool cfg_is_unreachable(TB_Node* n) { - FOR_USERS(u, n) { - if (u->n->type == TB_UNREACHABLE) { - return true; - } - } - - return false; -} - -static TB_Node* cfg_next_control0(TB_Node* n) { - FOR_USERS(u, n) { - if (u->slot == 0 && cfg_is_control(u->n)) { - return u->n; - } - } - - return NULL; -} - -static TB_Node* cfg_next_control(TB_Node* n) { - FOR_USERS(u, n) { - if (cfg_is_control(u->n)) { - return u->n; - } - } - - return NULL; -} - -static TB_Node* get_pred(TB_Node* n, int i) { - TB_Node* base = n; - n = n->inputs[i]; - - if (base->type == TB_REGION && n->type == TB_PROJ) { - TB_Node* parent = n->inputs[0]; - - // start or cprojs with multiple users (it's a BB) will just exit - if (parent->type == TB_ROOT || (!ctrl_out_as_cproj_but_not_branch(parent) && n->users->next != NULL)) { - return n; - } - n = parent; - } - - while (!cfg_is_bb_entry(n)) { - n = n->inputs[0]; - } - - return n; -} - -static TB_Node* cfg_get_pred(TB_CFG* cfg, TB_Node* n, int i) { - n = n->inputs[i]; - for (;;) { - ptrdiff_t search = nl_map_get(cfg->node_to_block, n); - if (search >= 0 || n->type == TB_REGION) { - return n; - } - - n = n->inputs[0]; - } -} - -static TB_Node* next_control(TB_Node* n) { - // unless it's a branch (aka a terminator), it'll have one successor - TB_Node* next = NULL; - FOR_USERS(u, n) { - TB_Node* succ = u->n; - - // we can't treat regions in the chain - if (succ->type == TB_REGION) break; - - // we've found the next step in control flow - if (cfg_is_control(succ)) { - return succ; - } - } - - return NULL; -} - -static TB_Node* get_block_begin(TB_Node* n) { - while (!cfg_is_bb_entry(n)) { - n = n->inputs[0]; - } - return n; -} - -static TB_BasicBlock* idom_bb(TB_Passes* p, TB_BasicBlock* bb) { - return bb->dom; -} - -// shorthand because we use it a lot -static TB_Node* idom(TB_CFG* cfg, TB_Node* n) { - if (cfg->node_to_block == NULL) return NULL; - - ptrdiff_t search = nl_map_get(cfg->node_to_block, n); - if (search < 0) { - return NULL; - } - - TB_BasicBlock* dom = cfg->node_to_block[search].v.dom; - return dom ? dom->start : NULL; -} - -static int dom_depth(TB_CFG* cfg, TB_Node* n) { - return nl_map_get_checked(cfg->node_to_block, n).dom_depth; -} - -static bool slow_dommy(TB_CFG* cfg, TB_Node* expected_dom, TB_Node* bb) { - while (bb != NULL && expected_dom != bb) { - TB_Node* new_bb = idom(cfg, bb); - if (new_bb == NULL || new_bb == bb) { - return false; - } - bb = new_bb; - } - - return true; -} - -extern thread_local TB_Arena* tmp_arena; - -void verify_tmp_arena(TB_Passes* p); - -// CFG -// pushes postorder walk into worklist items, also modifies the visited set. -TB_CFG tb_compute_rpo(TB_Function* f, TB_Passes* restrict p); -TB_CFG tb_compute_rpo2(TB_Function* f, Worklist* ws); -void tb_free_cfg(TB_CFG* cfg); -// postorder walk -> dominators -void tb_compute_dominators(TB_Function* f, TB_Passes* restrict p, TB_CFG cfg); -void tb_compute_dominators2(TB_Function* f, Worklist* ws, TB_CFG cfg); - -// Worklist API -void worklist_alloc(Worklist* restrict ws, size_t initial_cap); -void worklist_free(Worklist* restrict ws); -void worklist_clear(Worklist* restrict ws); -void worklist_clear_visited(Worklist* restrict ws); -bool worklist_test(Worklist* restrict ws, TB_Node* n); -bool worklist_test_n_set(Worklist* restrict ws, TB_Node* n); -void worklist_push(Worklist* restrict ws, TB_Node* restrict n); -int worklist_popcount(Worklist* ws); -TB_Node* worklist_pop(Worklist* ws); - -// Local scheduler -void greedy_scheduler(TB_Passes* p, TB_CFG* cfg, Worklist* ws, DynArray(PhiVal)* phi_vals, TB_BasicBlock* bb, TB_Node* end); -void tb_pass_schedule(TB_Passes* p, TB_CFG cfg, bool renumber); - -// makes arch-friendly IR -void tb_pass_legalize(TB_Passes* p, TB_Arch arch); - -Lattice* lattice_universe_get(TB_Passes* p, TB_Node* n); -LatticeTrifecta lattice_truthy(Lattice* l); diff --git a/vendor/tb/src/opt/peeps.h b/vendor/tb/src/opt/peeps.h deleted file mode 100644 index 98c835e9..00000000 --- a/vendor/tb/src/opt/peeps.h +++ /dev/null @@ -1,96 +0,0 @@ -// This is the table of all peephole rewrite rules, there's a few categories: -// -// * Idealize: replaces a node with a simplified form, this is run until exhaustion -// on a node (although in practice will warn past 5 trips, it's weird to -// even do more than 2 trips) -// -// * Identity: replace a node with it's direct inputs (one step). -// -// * ValueOf: constant propagation crap (works for pessimistic and optimistic crap out of the box) -// -// * GVN: if a node has some identical copy, it will be replaced with it. -// -// They're run in this order for every node and given each is well-formed (one step Church-Rosser) -// the number of rewrites performed should scale linearly with the size of the IR. -typedef TB_Node* (*NodeIdealize)(TB_Passes* restrict p, TB_Function* f, TB_Node* n); -typedef TB_Node* (*NodeIdentity)(TB_Passes* restrict p, TB_Function* f, TB_Node* n); -typedef Lattice* (*NodeValueOf)(TB_Passes* restrict p, TB_Node* n); - -enum { - NODE_IS_CTRL = 1, -}; - -typedef struct { - NodeIdealize idealize; - NodeIdentity identity; - NodeValueOf value; - uint32_t flags; -} NodeVtable; - -static const NodeVtable vtables[TB_NODE_TYPE_MAX] = { - // type ideal identity value - [TB_INTEGER_CONST] = { NULL, NULL, value_int }, - // memory - [TB_LOAD] = { ideal_load, identity_load, NULL }, - [TB_STORE] = { ideal_store, NULL, value_mem }, - [TB_MEMSET] = { ideal_memset, NULL, value_mem }, - [TB_MEMCPY] = { ideal_memcpy, NULL, value_mem }, - [TB_SPLITMEM] = { NULL, NULL, value_split_mem }, - [TB_MERGEMEM] = { ideal_merge_mem, NULL, value_merge_mem }, - // ptr values - [TB_LOCAL] = { NULL, NULL, value_ptr_vals }, - [TB_SYMBOL] = { NULL, NULL, value_ptr_vals }, - // pointer arithmetic - [TB_MEMBER_ACCESS] = { ideal_member_ptr, identity_member_ptr,NULL }, - [TB_ARRAY_ACCESS] = { ideal_array_ptr, NULL, NULL }, - // arithmetic - [TB_ADD] = { ideal_int_binop, identity_int_binop, value_arith }, - [TB_SUB] = { ideal_int_binop, identity_int_binop, value_arith }, - [TB_MUL] = { ideal_int_binop, identity_int_binop, value_arith }, - [TB_UDIV] = { ideal_int_div, identity_int_binop, NULL }, - [TB_SDIV] = { ideal_int_div, identity_int_binop, NULL }, - [TB_UMOD] = { ideal_int_mod, identity_int_binop, NULL }, - [TB_SMOD] = { ideal_int_mod, identity_int_binop, NULL }, - // comparisons - [TB_CMP_EQ] = { ideal_int_binop, identity_int_binop, value_cmp }, - [TB_CMP_NE] = { ideal_int_binop, identity_int_binop, value_cmp }, - [TB_CMP_SLT] = { ideal_int_binop, identity_int_binop, value_cmp }, - [TB_CMP_SLE] = { ideal_int_binop, identity_int_binop, value_cmp }, - [TB_CMP_ULT] = { ideal_int_binop, identity_int_binop, value_cmp }, - [TB_CMP_ULE] = { ideal_int_binop, identity_int_binop, value_cmp }, - // bitwise ops - [TB_AND] = { ideal_int_binop, identity_int_binop, value_bits }, - [TB_OR] = { ideal_int_binop, identity_int_binop, value_bits }, - [TB_XOR] = { ideal_int_binop, identity_int_binop, value_bits }, - // shift - [TB_SHL] = { ideal_int_binop, identity_int_binop, value_shift }, - [TB_SHR] = { ideal_int_binop, identity_int_binop, value_shift }, - [TB_SAR] = { ideal_int_binop, identity_int_binop, value_shift }, - // unary - [TB_NEG] = { NULL, NULL, value_unary }, - [TB_NOT] = { NULL, NULL, value_unary }, - // casts - [TB_BITCAST] = { ideal_bitcast, NULL, value_bitcast }, - [TB_TRUNCATE] = { ideal_truncate, NULL, value_trunc }, - [TB_ZERO_EXT] = { ideal_extension, NULL, value_zext }, - [TB_SIGN_EXT] = { ideal_extension, NULL, value_sext }, - // misc - [TB_LOOKUP] = { NULL, NULL, value_lookup }, - [TB_PROJ] = { NULL, NULL, value_proj }, - [TB_SELECT] = { ideal_select, NULL, value_select }, - [TB_PHI] = { ideal_phi, identity_phi, value_phi }, - // control flow - [TB_RETURN] = { ideal_return, NULL, value_ctrl, NODE_IS_CTRL }, - [TB_REGION] = { ideal_region, identity_region, value_region, NODE_IS_CTRL }, - [TB_NATURAL_LOOP] = { ideal_region, identity_region, value_region, NODE_IS_CTRL }, - [TB_AFFINE_LOOP] = { ideal_region, identity_region, value_region, NODE_IS_CTRL }, - [TB_BRANCH] = { ideal_branch, NULL, value_branch, NODE_IS_CTRL }, - [TB_SAFEPOINT_POLL] = { NULL, identity_safepoint, value_ctrl, NODE_IS_CTRL }, - [TB_CALL] = { ideal_libcall, NULL, value_call, NODE_IS_CTRL }, - [TB_TAILCALL] = { NULL, NULL, value_ctrl, NODE_IS_CTRL }, - [TB_SYSCALL] = { NULL, NULL, value_call, NODE_IS_CTRL }, - [TB_DEBUGBREAK] = { NULL, NULL, value_ctrl, NODE_IS_CTRL }, - [TB_TRAP] = { NULL, NULL, value_ctrl, NODE_IS_CTRL }, - [TB_UNREACHABLE] = { NULL, NULL, value_ctrl, NODE_IS_CTRL }, -}; - diff --git a/vendor/tb/src/opt/print.h b/vendor/tb/src/opt/print.h deleted file mode 100644 index 77bec7c1..00000000 --- a/vendor/tb/src/opt/print.h +++ /dev/null @@ -1,521 +0,0 @@ - -typedef struct { - TB_Passes* opt; - TB_Function* f; - TB_CFG cfg; - TB_Scheduler sched; -} PrinterCtx; - -static void print_type(TB_DataType dt) { - switch (dt.type) { - case TB_INT: { - if (dt.data == 0) printf("void"); - else printf("i%d", dt.data); - break; - } - case TB_PTR: { - if (dt.data == 0) printf("ptr"); - else printf("ptr%d", dt.data); - break; - } - case TB_FLOAT: { - if (dt.data == TB_FLT_32) printf("f32"); - if (dt.data == TB_FLT_64) printf("f64"); - break; - } - case TB_TUPLE: { - printf("tuple"); - break; - } - case TB_CONTROL: { - printf("ctrl"); - break; - } - case TB_MEMORY: { - printf("mem"); - break; - } - default: tb_todo(); - } -} - -static void print_type2(TB_DataType dt) { - #if 0 - printf("\x1b[96m"); - print_type(dt); - printf("\x1b[0m"); - #else - print_type(dt); - #endif -} - -static void print_ref_to_node(PrinterCtx* ctx, TB_Node* n, bool def) { - if (n == NULL) { - printf("_"); - } else if (n->type == TB_ROOT) { - printf("%s", ctx->f->super.name); - - if (def) { - printf("("); - TB_Node** params = ctx->f->params; - FOREACH_N(i, 1, 3 + ctx->f->param_count) { - if (i > 1) printf(", "); - - if (params[i] == NULL) { - printf("_"); - } else { - printf("v%u: ", params[i]->gvn); - print_type2(params[i]->dt); - } - } - printf(")"); - } - } else if (cfg_is_region(n)) { - TB_NodeRegion* r = TB_NODE_GET_EXTRA(n); - if (r->tag != NULL) { - printf(".%s", r->tag); - } else { - ptrdiff_t i = try_find_traversal_index(&ctx->cfg, n); - if (i >= 0) { - printf(".bb%zu", i); - } else { - printf("*DEAD*"); - } - } - - if (def) { - bool first = true; - printf("("); - FOR_USERS(u, n) { - if (u->n->type == TB_PHI) { - if (first) { - first = false; - } else { - printf(", "); - } - - printf("v%u: ", u->n->gvn); - print_type2(u->n->dt); - } - } - // printf(")"); - printf(") // v%u", n->gvn); - if (n->type == TB_NATURAL_LOOP) printf(" !natural"); - if (n->type == TB_AFFINE_LOOP) printf(" !affine"); - } - } else if (n->type == TB_FLOAT32_CONST) { - TB_NodeFloat32* f = TB_NODE_GET_EXTRA(n); - printf("%f", f->value); - } else if (n->type == TB_FLOAT64_CONST) { - TB_NodeFloat64* f = TB_NODE_GET_EXTRA(n); - printf("%f", f->value); - } else if (n->type == TB_SYMBOL) { - TB_Symbol* sym = TB_NODE_GET_EXTRA_T(n, TB_NodeSymbol)->sym; - if (sym->name[0]) { - printf("%s", sym->name); - } else { - printf("sym%p", sym); - } - } else if (n->type == TB_PROJ && n->dt.type == TB_CONTROL) { - if (n->inputs[0]->type == TB_ROOT) { - print_ref_to_node(ctx, n->inputs[0], def); - } else { - ptrdiff_t i = try_find_traversal_index(&ctx->cfg, n); - if (i >= 0) { - printf(".bb%zu", i); - if (def) { - // printf("()"); - printf("() // v%u", n->gvn); - } - } else { - printf("*DEAD*"); - } - } - } else if (n->type == TB_ZERO_EXT) { - printf("(zxt."); - print_type2(n->dt); - printf(" "); - print_ref_to_node(ctx, n->inputs[1], false); - printf(")"); - } else if (n->type == TB_SIGN_EXT) { - printf("(sxt."); - print_type2(n->dt); - printf(" "); - print_ref_to_node(ctx, n->inputs[1], false); - printf(")"); - } else if (n->type == TB_INTEGER_CONST) { - TB_NodeInt* num = TB_NODE_GET_EXTRA(n); - - if (num->value < 0xFFFF) { - int bits = n->dt.type == TB_PTR ? 64 : n->dt.data; - printf("%"PRId64, tb__sxt(num->value, bits, 64)); - } else { - printf("%#0"PRIx64, num->value); - } - } else { - printf("v%u", n->gvn); - } -} - -// deals with printing BB params -static void print_branch_edge(PrinterCtx* ctx, TB_Node* n, bool fallthru) { - TB_Node* target = fallthru ? cfg_next_control(n) : cfg_next_bb_after_cproj(n); - print_ref_to_node(ctx, target, false); - - // print phi args - printf("("); - if (cfg_is_region(target)) { - int phi_i = -1; - FOR_USERS(u, n) { - if (cfg_is_region(u->n)) { - phi_i = 1 + u->slot; - break; - } - } - - bool first = true; - FOR_USERS(u, target) { - if (u->n->type == TB_PHI) { - if (first) { - first = false; - } else { - printf(", "); - } - - assert(phi_i >= 0); - print_ref_to_node(ctx, u->n->inputs[phi_i], false); - } - } - } - printf(")"); -} - -static void print_bb(PrinterCtx* ctx, TB_Node* bb_start) { - print_ref_to_node(ctx, bb_start, true); - - if (ctx->opt->error_n == bb_start) { - printf("\x1b[31m <-- ERROR\x1b[0m"); - } - printf("\n"); - - TB_BasicBlock* bb = ctx->opt->scheduled[bb_start->gvn]; - Worklist* ws = &ctx->opt->worklist; - - #ifndef NDEBUG - TB_BasicBlock* expected = &nl_map_get_checked(ctx->cfg.node_to_block, bb_start); - assert(expected == bb); - #endif - - ctx->sched(ctx->opt, &ctx->cfg, ws, NULL, bb, bb->end); - - FOREACH_N(i, ctx->cfg.block_count, dyn_array_length(ws->items)) { - TB_Node* n = ws->items[i]; - - // skip these - if (n->type == TB_INTEGER_CONST || n->type == TB_FLOAT32_CONST || - n->type == TB_FLOAT64_CONST || n->type == TB_SYMBOL || - n->type == TB_SIGN_EXT || n->type == TB_ZERO_EXT || - n->type == TB_PROJ || n->type == TB_REGION || - n->type == TB_NATURAL_LOOP || n->type == TB_AFFINE_LOOP || - n->type == TB_NULL || n->type == TB_PHI) { - continue; - } - - TB_NodeLocation* v; - if (v = nl_table_get(&ctx->f->locations, n), v) { - printf(" # location %s:%d\n", v->file->path, v->line); - } - - switch (n->type) { - case TB_DEBUGBREAK: printf(" debugbreak"); break; - case TB_UNREACHABLE: printf(" unreachable"); break; - - case TB_BRANCH: { - TB_NodeBranch* br = TB_NODE_GET_EXTRA(n); - TB_ArenaSavepoint sp = tb_arena_save(tmp_arena); - TB_Node** restrict succ = tb_arena_alloc(tmp_arena, br->succ_count * sizeof(TB_Node**)); - - // fill successors - FOR_USERS(u, n) { - if (u->n->type == TB_PROJ) { - int index = TB_NODE_GET_EXTRA_T(u->n, TB_NodeProj)->index; - succ[index] = u->n; - } - } - - if (br->succ_count == 1) { - printf(" goto "); - print_branch_edge(ctx, succ[0], false); - } else if (br->succ_count == 2) { - int bits = n->inputs[1]->dt.type == TB_PTR ? 64 : n->inputs[1]->dt.data; - - printf(" if "); - FOREACH_N(i, 1, n->input_count) { - if (i != 1) printf(", "); - print_ref_to_node(ctx, n->inputs[i], false); - } - if (br->keys[0].key == 0) { - printf(" then "); - } else { - printf(" != %"PRId64" then ", tb__sxt(br->keys[0].key, bits, 64)); - } - print_branch_edge(ctx, succ[0], false); - printf(" else "); - print_branch_edge(ctx, succ[1], false); - } else { - printf(" br "); - FOREACH_N(i, 1, n->input_count) { - if (i != 1) printf(", "); - print_ref_to_node(ctx, n->inputs[i], false); - } - printf("%s=> {\n", n->input_count > 1 ? " " : ""); - - FOREACH_N(i, 0, br->succ_count) { - if (i != 0) printf(" %"PRId64": ", br->keys[i - 1].key); - else printf(" default: "); - - print_branch_edge(ctx, succ[i], false); - printf("\n"); - } - printf(" }"); - } - printf(" // v%u", n->gvn); - tb_arena_restore(tmp_arena, sp); - break; - } - - case TB_TRAP: { - printf(" trap"); - break; - } - - case TB_RETURN: { - printf(" end "); - FOREACH_N(i, 1, n->input_count) { - if (i != 1) printf(", "); - print_ref_to_node(ctx, n->inputs[i], false); - } - break; - } - - default: { - if (n->dt.type == TB_TUPLE) { - // print with multiple returns - TB_Node* projs[32] = { 0 }; - FOR_USERS(use, n) { - if (use->n->type == TB_PROJ) { - int index = TB_NODE_GET_EXTRA_T(use->n, TB_NodeProj)->index; - projs[index] = use->n; - } - } - - printf(" "); - - size_t first = projs[0] && projs[0]->dt.type == TB_CONTROL ? 1 : 0; - FOREACH_N(i, first, 32) { - if (projs[i] == NULL) break; - if (i > first) printf(", "); - printf("v%u", projs[i]->gvn); - } - printf(" = %s.(", tb_node_get_name(n)); - FOREACH_N(i, first, 32) { - if (projs[i] == NULL) break; - if (i > first) printf(", "); - print_type2(projs[i]->dt); - } - printf(")"); - } else { - // print as normal instruction - printf(" v%u = %s.", n->gvn, tb_node_get_name(n)); - - TB_DataType dt = n->dt; - if (n->type >= TB_CMP_EQ && n->type <= TB_CMP_FLE) { - dt = TB_NODE_GET_EXTRA_T(n, TB_NodeCompare)->cmp_dt; - } else if (n->type == TB_STORE) { - dt = n->inputs[3]->dt; - } - print_type2(dt); - } - printf(" "); - - size_t first = n->type == TB_PROJ ? 0 : 1; - FOREACH_N(i, first, n->input_count) { - if (i != first) printf(", "); - print_ref_to_node(ctx, n->inputs[i], false); - } - - // print extra data - switch (n->type) { - case TB_CMP_EQ: - case TB_CMP_NE: - case TB_CMP_ULT: - case TB_CMP_ULE: - case TB_CMP_SLT: - case TB_CMP_SLE: - case TB_CMP_FLT: - case TB_CMP_FLE: - case TB_SELECT: - case TB_BITCAST: - break; - - case TB_MEMBER_ACCESS: { - printf(", %"PRId64, TB_NODE_GET_EXTRA_T(n, TB_NodeMember)->offset); - break; - } - - case TB_ARRAY_ACCESS: { - printf(", %"PRId64, TB_NODE_GET_EXTRA_T(n, TB_NodeArray)->stride); - break; - } - - case TB_PROJ: { - printf(", %d", TB_NODE_GET_EXTRA_T(n, TB_NodeProj)->index); - break; - } - - case TB_AND: - case TB_OR: - case TB_XOR: - case TB_ADD: - case TB_SUB: - case TB_MUL: - case TB_SHL: - case TB_SHR: - case TB_ROL: - case TB_ROR: - case TB_SAR: - case TB_UDIV: - case TB_SDIV: - case TB_UMOD: - case TB_SMOD: - { - TB_NodeBinopInt* b = TB_NODE_GET_EXTRA(n); - if (b->ab & TB_ARITHMATIC_NSW) printf(" !nsw"); - if (b->ab & TB_ARITHMATIC_NUW) printf(" !nuw"); - break; - } - - case TB_LOAD: - case TB_STORE: - case TB_MEMSET: - case TB_MEMCPY: { - TB_NodeMemAccess* mem = TB_NODE_GET_EXTRA(n); - printf(" !align(%d)", mem->align); - break; - } - - case TB_ATOMIC_LOAD: - case TB_ATOMIC_XCHG: - case TB_ATOMIC_ADD: - case TB_ATOMIC_SUB: - case TB_ATOMIC_AND: - case TB_ATOMIC_XOR: - case TB_ATOMIC_OR: - case TB_ATOMIC_CAS: { - static const char* order_names[] = { - "relaxed", "consume", "acquire", - "release", "acqrel", "seqcst" - }; - - TB_NodeAtomic* atomic = TB_NODE_GET_EXTRA(n); - printf(" !order(%s)", order_names[atomic->order]); - if (n->type == TB_ATOMIC_CAS) { - printf(" !fail_order(%s)", order_names[atomic->order2]); - } - break; - } - - case TB_CALL: - case TB_TAILCALL: - case TB_SYSCALL: - case TB_SAFEPOINT_POLL: - case TB_MERGEMEM: - case TB_SPLITMEM: - break; - - case TB_LOCAL: { - TB_NodeLocal* l = TB_NODE_GET_EXTRA(n); - printf("!size(%u) !align(%u)", l->size, l->align); - if (l->type) { - printf(" !var(%s)", l->name); - } - break; - } - - case TB_LOOKUP: { - TB_NodeLookup* l = TB_NODE_GET_EXTRA(n); - - printf(" { default: %"PRId64, l->entries[0].val); - FOREACH_N(i, 1, l->entry_count) { - printf(", %"PRId64": %"PRId64, l->entries[i].key, l->entries[i].val); - } - printf("}"); - break; - } - - default: tb_assert(extra_bytes(n) == 0, "TODO"); - } - break; - } - } - - if (ctx->opt->error_n == n) { - printf("\x1b[31m <-- ERROR\x1b[0m"); - } - - printf("\n"); - } - - dyn_array_set_length(ws->items, ctx->cfg.block_count); - - if (!cfg_is_terminator(bb->end)) { - printf(" goto "); - print_branch_edge(ctx, bb->end, true); - printf("\n"); - } -} - -void tb_pass_print(TB_Passes* opt) { - TB_Function* f = opt->f; - cuikperf_region_start("print", NULL); - - Worklist old = opt->worklist; - Worklist tmp_ws = { 0 }; - worklist_alloc(&tmp_ws, f->node_count); - - PrinterCtx ctx = { opt, f }; - opt->worklist = tmp_ws; - ctx.cfg = tb_compute_rpo(f, opt); - - // does the IR printing need smart scheduling lol (yes... when we're testing things) - ctx.sched = greedy_scheduler; - - TB_ArenaSavepoint sp = tb_arena_save(tmp_arena); - - // schedule nodes - tb_pass_schedule(opt, ctx.cfg, false); - worklist_clear_visited(&opt->worklist); - - TB_Node* end_bb = NULL; - FOREACH_N(i, 0, ctx.cfg.block_count) { - TB_Node* end = nl_map_get_checked(ctx.cfg.node_to_block, opt->worklist.items[i]).end; - if (end->type == TB_RETURN) { - end_bb = opt->worklist.items[i]; - continue; - } - - print_bb(&ctx, opt->worklist.items[i]); - } - - if (end_bb != NULL) { - print_bb(&ctx, end_bb); - } - - tb_arena_restore(tmp_arena, sp); - worklist_free(&opt->worklist); - tb_free_cfg(&ctx.cfg); - opt->worklist = old; - opt->scheduled = NULL; - opt->error_n = NULL; - cuikperf_region_end(); -} diff --git a/vendor/tb/src/opt/print_c.h b/vendor/tb/src/opt/print_c.h deleted file mode 100644 index 5fd86fc8..00000000 --- a/vendor/tb/src/opt/print_c.h +++ /dev/null @@ -1,1672 +0,0 @@ - -#include -#include -#include - -typedef struct nl_buffer_t nl_buffer_t; - -nl_buffer_t *nl_buffer_new(); - -void nl_buffer_format(nl_buffer_t *buf, const char *fmt, ...) __attribute__ (( format( printf, 2, 3 ) )); -char *nl_buffer_get(nl_buffer_t *buf); - -typedef struct { - size_t gvn; - size_t label; - bool used: 1; -} CFmtFrame; - -typedef struct { - size_t gvn; - size_t label; - bool used: 1; -} CFmtFallNext; - -typedef struct { - size_t low; - size_t high; -} CFmtBlockRange; - -typedef struct { - const char *name; - TB_Passes* opt; - TB_Function* f; - TB_Module *module; - size_t a; - size_t num_labels; - NL_HashSet declared_types; - DynArray(CFmtFrame *) visited_blocks; - NL_Table block_ranges; - TB_CFG cfg; - TB_Scheduler sched; - NL_HashSet declared_vars; - nl_buffer_t *globals; - nl_buffer_t *pre; - nl_buffer_t *buf; - void *arena; - ptrdiff_t loop_goes_to; - int depth; -} CFmtState; - -static void c_fmt_ref_to_node(CFmtState* ctx, TB_Node* n); -static void c_fmt_typed_ref_to_node(CFmtState* ctx, TB_DataType dt, TB_Node* n); - -static void c_fmt_spaces(CFmtState *ctx) { - for (unsigned char i = 0; i < ctx->depth; i++) { - nl_buffer_format(ctx->buf, " "); - } -} - -static const char *c_fmt_type_name(TB_DataType dt) { - switch (dt.type) { - case TB_INT: { - if (dt.data == 0) return "void"; - if (dt.data == 1) return "char"; - if (dt.data <= 8) return "uint8_t"; - if (dt.data <= 16) return "uint16_t"; - if (dt.data <= 32) return "uint32_t"; - if (dt.data <= 64) return "uint64_t"; - else __builtin_trap(); - break; - } - case TB_PTR: { - if (dt.data == 0) return "void*"; - else tb_todo(); - break; - } - case TB_FLOAT: { - if (dt.data == TB_FLT_32) return "float"; - if (dt.data == TB_FLT_64) return "double"; - break; - } - case TB_TUPLE: { - tb_todo(); - break; - } - case TB_CONTROL: { - tb_todo(); - break; - } - case TB_MEMORY: { - tb_todo(); - break; - } - default: { - tb_todo(); - break; - } - } - return "void *"; -} - -static const char *c_fmt_type_name_signed(TB_DataType dt) { - switch (dt.type) { - case TB_INT: { - if (dt.data == 0) return "void"; - if (dt.data == 1) return "char"; - if (dt.data <= 8) return "int8_t"; - if (dt.data <= 16) return "int16_t"; - if (dt.data <= 32) return "int32_t"; - if (dt.data <= 64) return "int64_t"; - else __builtin_trap(); - break; - } - case TB_PTR: { - if (dt.data == 0) return "void*"; - else tb_todo(); - break; - } - case TB_FLOAT: { - if (dt.data == TB_FLT_32) return "float"; - if (dt.data == TB_FLT_64) return "double"; - break; - } - case TB_TUPLE: { - tb_todo(); - break; - } - case TB_CONTROL: { - tb_todo(); - break; - } - case TB_MEMORY: { - tb_todo(); - break; - } - default: { - tb_todo(); - break; - } - } - return "void *"; -} - -static void c_fmt_output(CFmtState* ctx, TB_Node* n) { - if (n->users != NULL) { - if (!nl_hashset_put(&ctx->declared_vars, n)) { - nl_buffer_format(ctx->pre, " %s v%u;\n", c_fmt_type_name(n->dt), n->gvn); - } - c_fmt_spaces(ctx); - nl_buffer_format(ctx->buf, "v%u = ", n->gvn); - } -} - -static bool c_fmt_will_inline(TB_Node *n) { - // if (n->type == TB_INTEGER_CONST || n->type == TB_FLOAT32_CONST || - // n->type == TB_FLOAT64_CONST || n->type == TB_SYMBOL || - // n->type == TB_SIGN_EXT || n->type == TB_ZERO_EXT || - // n->type == TB_PROJ || n->type == TB_REGION || - // n->type == TB_NULL || - // n->type == TB_PHI) { - // return true; - // } - if (n->type == TB_ROOT) { - return true; - } - if (n->type == TB_PROJ && n->dt.type == TB_CONTROL) { - return true; - } - if (n->type == TB_REGION) { - return true; - } - if (n->type == TB_FLOAT32_CONST) { - return true; - } - if (n->type == TB_FLOAT64_CONST) { - return true; - } - if (n->type == TB_SYMBOL) { - return true; - } - if (n->type == TB_ZERO_EXT) { - return true; - } - if (n->type == TB_SIGN_EXT) { - return true; - } - if (n->type == TB_INTEGER_CONST) { - return true; - } - size_t len = 0; - for (User *head = n->users; head != NULL; head = head->next) { - len += 1; - } - if (len <= 1) { - if (n->type == TB_SELECT) { - return true; - } - if (n->type == TB_SHL || n->type == TB_SHR || n->type == TB_SAR) { - return true; - } - if (n->type == TB_AND || n->type == TB_OR || n->type == TB_XOR) { - return true; - } - if (n->type == TB_ADD || n->type == TB_FADD) { - return true; - } - if (n->type == TB_SUB || n->type == TB_FSUB) { - return true; - } - if (n->type == TB_MUL || n->type == TB_FMUL) { - return true; - } - if (n->type == TB_SDIV || n->type == TB_UDIV || n->type == TB_FDIV) { - return true; - } - if (n->type == TB_CMP_EQ) { - return true; - } - if (n->type == TB_CMP_NE) { - return true; - } - if (n->type == TB_CMP_SLT || n->type == TB_CMP_ULT || n->type == TB_CMP_FLT) { - return true; - } - if (n->type == TB_CMP_SLE || n->type == TB_CMP_ULE || n->type == TB_CMP_FLE) { - return true; - } - } - return false; -} - -static void c_fmt_inline_node(CFmtState* ctx, TB_Node *n) { - if (n->type == TB_SELECT) { - TB_Node *cond = n->inputs[n->input_count-3]; - TB_Node *then = n->inputs[n->input_count-2]; - TB_Node *els = n->inputs[n->input_count-1]; - nl_buffer_format(ctx->buf, "("); - nl_buffer_format(ctx->buf, "(char) "); - c_fmt_ref_to_node(ctx, cond); - nl_buffer_format(ctx->buf, " ? "); - nl_buffer_format(ctx->buf, "(%s) ", c_fmt_type_name(n->dt)); - c_fmt_ref_to_node(ctx, then); - nl_buffer_format(ctx->buf, " : "); - nl_buffer_format(ctx->buf, "(%s) ", c_fmt_type_name(n->dt)); - c_fmt_ref_to_node(ctx, els); - nl_buffer_format(ctx->buf, ")"); - } else if (n->type == TB_SHL) { - TB_Node *lhs = n->inputs[n->input_count-2]; - TB_Node *rhs = n->inputs[n->input_count-1]; - nl_buffer_format(ctx->buf, "("); - nl_buffer_format(ctx->buf, "(%s) ", c_fmt_type_name(n->dt)); - c_fmt_ref_to_node(ctx, lhs); - nl_buffer_format(ctx->buf, " << "); - nl_buffer_format(ctx->buf, "(%s) ", c_fmt_type_name(n->dt)); - c_fmt_ref_to_node(ctx, rhs); - nl_buffer_format(ctx->buf, ")"); - } else if (n->type == TB_SHR || n->type == TB_SAR) { - TB_Node *lhs = n->inputs[n->input_count-2]; - TB_Node *rhs = n->inputs[n->input_count-1]; - nl_buffer_format(ctx->buf, "("); - nl_buffer_format(ctx->buf, "(%s) ", c_fmt_type_name(n->dt)); - c_fmt_ref_to_node(ctx, lhs); - nl_buffer_format(ctx->buf, " >> "); - nl_buffer_format(ctx->buf, "(%s) ", c_fmt_type_name(n->dt)); - c_fmt_ref_to_node(ctx, rhs); - nl_buffer_format(ctx->buf, ")"); - } else if (n->type == TB_AND) { - TB_Node *lhs = n->inputs[n->input_count-2]; - TB_Node *rhs = n->inputs[n->input_count-1]; - nl_buffer_format(ctx->buf, "("); - nl_buffer_format(ctx->buf, "(%s) ", c_fmt_type_name(n->dt)); - c_fmt_ref_to_node(ctx, lhs); - nl_buffer_format(ctx->buf, " & "); - nl_buffer_format(ctx->buf, "(%s) ", c_fmt_type_name(n->dt)); - c_fmt_ref_to_node(ctx, rhs); - nl_buffer_format(ctx->buf, ")"); - } else if (n->type == TB_OR) { - TB_Node *lhs = n->inputs[n->input_count-2]; - TB_Node *rhs = n->inputs[n->input_count-1]; - nl_buffer_format(ctx->buf, "("); - nl_buffer_format(ctx->buf, "(%s) ", c_fmt_type_name(n->dt)); - c_fmt_ref_to_node(ctx, lhs); - nl_buffer_format(ctx->buf, " | "); - nl_buffer_format(ctx->buf, "(%s) ", c_fmt_type_name(n->dt)); - c_fmt_ref_to_node(ctx, rhs); - nl_buffer_format(ctx->buf, ")"); - } else if (n->type == TB_XOR) { - TB_Node *lhs = n->inputs[n->input_count-2]; - TB_Node *rhs = n->inputs[n->input_count-1]; - nl_buffer_format(ctx->buf, "("); - nl_buffer_format(ctx->buf, "(%s) ", c_fmt_type_name(n->dt)); - c_fmt_ref_to_node(ctx, lhs); - nl_buffer_format(ctx->buf, " ^ "); - nl_buffer_format(ctx->buf, "(%s) ", c_fmt_type_name(n->dt)); - c_fmt_ref_to_node(ctx, rhs); - nl_buffer_format(ctx->buf, ")"); - } else if (n->type == TB_ADD || n->type == TB_FADD) { - TB_Node *lhs = n->inputs[n->input_count-2]; - TB_Node *rhs = n->inputs[n->input_count-1]; - nl_buffer_format(ctx->buf, "("); - nl_buffer_format(ctx->buf, "(%s) ", c_fmt_type_name(n->dt)); - c_fmt_ref_to_node(ctx, lhs); - nl_buffer_format(ctx->buf, " + "); - nl_buffer_format(ctx->buf, "(%s) ", c_fmt_type_name(n->dt)); - c_fmt_ref_to_node(ctx, rhs); - nl_buffer_format(ctx->buf, ")"); - } else if (n->type == TB_SUB || n->type == TB_FSUB) { - TB_Node *lhs = n->inputs[n->input_count-2]; - TB_Node *rhs = n->inputs[n->input_count-1]; - nl_buffer_format(ctx->buf, "("); - nl_buffer_format(ctx->buf, "(%s) ", c_fmt_type_name(n->dt)); - c_fmt_ref_to_node(ctx, lhs); - nl_buffer_format(ctx->buf, " - "); - nl_buffer_format(ctx->buf, "(%s) ", c_fmt_type_name(n->dt)); - c_fmt_ref_to_node(ctx, rhs); - nl_buffer_format(ctx->buf, ")"); - } else if (n->type == TB_MUL || n->type == TB_FMUL) { - TB_Node *lhs = n->inputs[n->input_count-2]; - TB_Node *rhs = n->inputs[n->input_count-1]; - nl_buffer_format(ctx->buf, "("); - nl_buffer_format(ctx->buf, "(%s) ", c_fmt_type_name(n->dt)); - c_fmt_ref_to_node(ctx, lhs); - nl_buffer_format(ctx->buf, " * "); - nl_buffer_format(ctx->buf, "(%s) ", c_fmt_type_name(n->dt)); - c_fmt_ref_to_node(ctx, rhs); - nl_buffer_format(ctx->buf, ")"); - } else if (n->type == TB_SDIV) { - TB_Node *lhs = n->inputs[n->input_count-2]; - TB_Node *rhs = n->inputs[n->input_count-1]; - nl_buffer_format(ctx->buf, "("); - nl_buffer_format(ctx->buf, "(%s) ", c_fmt_type_name_signed(n->dt)); - c_fmt_ref_to_node(ctx, lhs); - nl_buffer_format(ctx->buf, " / "); - nl_buffer_format(ctx->buf, "(%s) ", c_fmt_type_name_signed(n->dt)); - c_fmt_ref_to_node(ctx, rhs); - nl_buffer_format(ctx->buf, ")"); - } else if (n->type == TB_UDIV) { - TB_Node *lhs = n->inputs[n->input_count-2]; - TB_Node *rhs = n->inputs[n->input_count-1]; - nl_buffer_format(ctx->buf, "("); - nl_buffer_format(ctx->buf, "(%s) ", c_fmt_type_name(n->dt)); - c_fmt_ref_to_node(ctx, lhs); - nl_buffer_format(ctx->buf, " / "); - nl_buffer_format(ctx->buf, "(%s) ", c_fmt_type_name(n->dt)); - c_fmt_ref_to_node(ctx, rhs); - nl_buffer_format(ctx->buf, ")"); - } else if (n->type == TB_FDIV) { - TB_Node *lhs = n->inputs[n->input_count-2]; - TB_Node *rhs = n->inputs[n->input_count-1]; - nl_buffer_format(ctx->buf, "("); - nl_buffer_format(ctx->buf, "(%s) ", c_fmt_type_name_signed(n->dt)); - c_fmt_ref_to_node(ctx, lhs); - nl_buffer_format(ctx->buf, " / "); - nl_buffer_format(ctx->buf, "(%s) ", c_fmt_type_name_signed(n->dt)); - c_fmt_ref_to_node(ctx, rhs); - nl_buffer_format(ctx->buf, ")"); - } else if (n->type == TB_CMP_EQ) { - TB_Node *lhs = n->inputs[n->input_count-2]; - TB_Node *rhs = n->inputs[n->input_count-1]; - nl_buffer_format(ctx->buf, "("); - c_fmt_ref_to_node(ctx, lhs); - nl_buffer_format(ctx->buf, " == "); - nl_buffer_format(ctx->buf, "(%s) ", c_fmt_type_name_signed(lhs->dt)); - c_fmt_ref_to_node(ctx, rhs); - nl_buffer_format(ctx->buf, ")"); - } else if (n->type == TB_CMP_NE) { - TB_Node *lhs = n->inputs[n->input_count-2]; - TB_Node *rhs = n->inputs[n->input_count-1]; - nl_buffer_format(ctx->buf, "("); - c_fmt_ref_to_node(ctx, lhs); - nl_buffer_format(ctx->buf, " != "); - nl_buffer_format(ctx->buf, "(%s) ", c_fmt_type_name_signed(lhs->dt)); - c_fmt_ref_to_node(ctx, rhs); - nl_buffer_format(ctx->buf, ")"); - } else if (n->type == TB_CMP_SLT) { - TB_Node *lhs = n->inputs[n->input_count-2]; - TB_Node *rhs = n->inputs[n->input_count-1]; - nl_buffer_format(ctx->buf, "("); - c_fmt_ref_to_node(ctx, lhs); - nl_buffer_format(ctx->buf, " < "); - nl_buffer_format(ctx->buf, "(%s) ", c_fmt_type_name_signed(lhs->dt)); - c_fmt_ref_to_node(ctx, rhs); - nl_buffer_format(ctx->buf, ")"); - } else if (n->type == TB_CMP_SLE) { - TB_Node *lhs = n->inputs[n->input_count-2]; - TB_Node *rhs = n->inputs[n->input_count-1]; - nl_buffer_format(ctx->buf, "("); - c_fmt_ref_to_node(ctx, lhs); - nl_buffer_format(ctx->buf, " <= "); - nl_buffer_format(ctx->buf, "(%s) ", c_fmt_type_name_signed(lhs->dt)); - c_fmt_ref_to_node(ctx, rhs); - nl_buffer_format(ctx->buf, ")"); - } else if (n->type == TB_CMP_ULT) { - TB_Node *lhs = n->inputs[n->input_count-2]; - TB_Node *rhs = n->inputs[n->input_count-1]; - nl_buffer_format(ctx->buf, "("); - c_fmt_ref_to_node(ctx, lhs); - nl_buffer_format(ctx->buf, " < "); - nl_buffer_format(ctx->buf, "(%s) ", c_fmt_type_name_signed(lhs->dt)); - c_fmt_ref_to_node(ctx, rhs); - nl_buffer_format(ctx->buf, ")"); - } else if (n->type == TB_CMP_ULE) { - TB_Node *lhs = n->inputs[n->input_count-2]; - TB_Node *rhs = n->inputs[n->input_count-1]; - nl_buffer_format(ctx->buf, "("); - c_fmt_ref_to_node(ctx, lhs); - nl_buffer_format(ctx->buf, " <= "); - nl_buffer_format(ctx->buf, "(%s) ", c_fmt_type_name_signed(lhs->dt)); - c_fmt_ref_to_node(ctx, rhs); - nl_buffer_format(ctx->buf, ")"); - } else if (n->type == TB_CMP_FLT) { - TB_Node *lhs = n->inputs[n->input_count-2]; - TB_Node *rhs = n->inputs[n->input_count-1]; - nl_buffer_format(ctx->buf, "("); - c_fmt_ref_to_node(ctx, lhs); - nl_buffer_format(ctx->buf, " < "); - nl_buffer_format(ctx->buf, "(%s) ", c_fmt_type_name(lhs->dt)); - c_fmt_ref_to_node(ctx, rhs); - nl_buffer_format(ctx->buf, ")"); - } else if (n->type == TB_CMP_FLE) { - TB_Node *lhs = n->inputs[n->input_count-2]; - TB_Node *rhs = n->inputs[n->input_count-1]; - nl_buffer_format(ctx->buf, "("); - c_fmt_ref_to_node(ctx, lhs); - nl_buffer_format(ctx->buf, " <= "); - nl_buffer_format(ctx->buf, "(%s) ", c_fmt_type_name(lhs->dt)); - c_fmt_ref_to_node(ctx, rhs); - nl_buffer_format(ctx->buf, ")"); - } else if (n->type == TB_FLOAT32_CONST) { - TB_NodeFloat32* f = TB_NODE_GET_EXTRA(n); - nl_buffer_format(ctx->buf, "%f", f->value); - } else if (n->type == TB_FLOAT64_CONST) { - TB_NodeFloat64* f = TB_NODE_GET_EXTRA(n); - nl_buffer_format(ctx->buf, "%f", f->value); - } else if (n->type == TB_SYMBOL) { - TB_Symbol* sym = TB_NODE_GET_EXTRA_T(n, TB_NodeSymbol)->sym; - if (sym->name[0]) { - if (sym->tag == TB_SYMBOL_GLOBAL) { - nl_buffer_format(ctx->buf, "tb2c_sym%zu_%s", sym->symbol_id, sym->name); - } else { - nl_buffer_format(ctx->buf, "%s", sym->name); - } - } else if (ctx->module->is_jit) { - void *addr = sym->address; - nl_buffer_format(ctx->buf, "(void*)%p", TB_NODE_GET_EXTRA_T(n, TB_NodeSymbol)->sym->address); - } else { - switch((int) sym->tag) { - case TB_SYMBOL_EXTERNAL: { - nl_buffer_format(ctx->buf, "(void*)0"); - break; - } - case TB_SYMBOL_GLOBAL: { - TB_Global *g = (void *) sym; - - uint8_t *data = tb_platform_heap_alloc(g->size); - - memset(&data[g->pos], 0, g->size); - FOREACH_N(k, 0, g->obj_count) { - if (g->objects[k].type == TB_INIT_OBJ_REGION) { - assert(g->objects[k].offset + g->objects[k].region.size <= g->size); - memcpy(&data[g->pos + g->objects[k].offset], g->objects[k].region.ptr, g->objects[k].region.size); - } - } - - nl_buffer_format(ctx->buf, "\""); - FOREACH_N(k, 0, g->size) { - if (data[k] == 0 && k + 1 == g->size) { - break; - } - if (32 <= data[k] && data[k] <= 126) { - nl_buffer_format(ctx->buf, "%c", data[k]); - } else if (data[k] == '\t') { - nl_buffer_format(ctx->buf, "\\t"); - } else if (data[k] == '\r') { - nl_buffer_format(ctx->buf, "\\r"); - } else if (data[k] == '\n') { - nl_buffer_format(ctx->buf, "\\n"); - } else if (data[k] == ' ') { - nl_buffer_format(ctx->buf, " "); - } else { - nl_buffer_format(ctx->buf, "\\x%02x", data[k]); - } - } - nl_buffer_format(ctx->buf, "\""); - - tb_platform_heap_free(data); - break; - } - case TB_SYMBOL_FUNCTION: { - nl_buffer_format(ctx->buf, "(void *) 0"); - break; - } - } - } - } else if (n->type == TB_ZERO_EXT) { - c_fmt_ref_to_node(ctx, n->inputs[1]); - } else if (n->type == TB_SIGN_EXT) { - c_fmt_ref_to_node(ctx, n->inputs[1]); - } else if (n->type == TB_INTEGER_CONST) { - TB_NodeInt* num = TB_NODE_GET_EXTRA(n); - - nl_buffer_format(ctx->buf, "(%s)", c_fmt_type_name(n->dt)); - - if (num->value < 0xFFFF) { - nl_buffer_format(ctx->buf, "%"PRIi64"llu", num->value); - } else { - nl_buffer_format(ctx->buf, "%#0"PRIx64"llu", num->value); - } - } else { - assert(false && "wrong node type"); - } -} - -static void c_fmt_ref_to_node(CFmtState* ctx, TB_Node* n) { - if (c_fmt_will_inline(n)) { - c_fmt_inline_node(ctx, n); - } else { - nl_buffer_format(ctx->buf, "v%u", n->gvn); - return; - } -} - -static void c_fmt_typed_ref_to_node(CFmtState* ctx, TB_DataType dt, TB_Node *n) { - nl_buffer_format(ctx->buf, "(%s) ", c_fmt_type_name(dt)); - c_fmt_ref_to_node(ctx, n); -} - -CFmtBlockRange *c_fmt_get_block_range(CFmtState* ctx, TB_Node* n) { - if (nl_table_get(&ctx->block_ranges, n) == NULL) { - TB_BasicBlock* bb = ctx->opt->scheduled[n->gvn]; - Worklist* ws = &ctx->opt->worklist; - CFmtBlockRange *range = tb_platform_heap_alloc(sizeof(CFmtBlockRange)); - if (bb == NULL) { - range->low = dyn_array_length(ws->items); - range->high = dyn_array_length(ws->items); - } else { - size_t foreach_start = dyn_array_length(ws->items); - ctx->sched(ctx->opt, &ctx->cfg, ws, NULL, bb, bb->end); - size_t foreach_end = dyn_array_length(ws->items); - - range->low = foreach_start; - range->high = foreach_end; - } - nl_table_put(&ctx->block_ranges, n, range); - } - return nl_table_get(&ctx->block_ranges, n); -} - -TB_Node *c_fmt_only_return(CFmtState *ctx, TB_Node *target) { - Worklist* ws = &ctx->opt->worklist; - CFmtBlockRange *range = c_fmt_get_block_range(ctx, target); - - FOREACH_N(i, range->low, range->high) { - TB_Node* n = ws->items[i]; - - // skip these - if (c_fmt_will_inline(n) && n->type != TB_RETURN) { - continue; - } - - if (n->type == TB_NULL || n->type == TB_PHI || n->type == TB_PROJ || n->type == TB_CALLGRAPH) { - continue; - } - - if (n->type == TB_RETURN) { - return n; - } else { - return NULL; - } - } - return NULL; -} - -// deals with printing BB params -static void c_fmt_branch_edge(CFmtState* ctx, TB_Node* n, bool fallthru) { - TB_Node* target = fallthru ? cfg_next_control(n) : cfg_next_bb_after_cproj(n); - - if (target->type == TB_REGION) { - int phi_i = -1; - FOR_USERS(u, n) { - if (u->n->type == TB_REGION) { - phi_i = 1 + u->slot; - break; - } - } - - if (target->users != NULL) { - c_fmt_spaces(ctx); - nl_buffer_format(ctx->buf, "{\n"); - ctx->depth += 1; - size_t has_phi = 0; - FOR_USERS(u, target) { - if (u->n->type == TB_PHI) { - if (u->n->inputs[phi_i] != NULL) { - if (u->n->inputs[phi_i]->dt.type != TB_CONTROL && u->n->inputs[phi_i]->dt.type != TB_MEMORY) { - has_phi += 1; - } - } - } - } - ctx->a += has_phi; - size_t pos = 0; - FOR_USERS(u, target) { - if (u->n->type == TB_PHI) { - if (u->n->inputs[phi_i] != NULL) { - if (u->n->inputs[phi_i]->dt.type != TB_CONTROL && u->n->inputs[phi_i]->dt.type != TB_MEMORY) { - assert(phi_i >= 0); - if (pos != 0) { - c_fmt_spaces(ctx); - nl_buffer_format(ctx->buf, "%s a%zu = ", c_fmt_type_name(u->n->dt), ctx->a + pos); - c_fmt_ref_to_node(ctx, u->n->inputs[phi_i]); - nl_buffer_format(ctx->buf, ";\n"); - } - pos += 1; - } - } - } - } - pos = 0; - FOR_USERS(u, target) { - if (u->n->type == TB_PHI) { - if (u->n->inputs[phi_i] != NULL) { - if (u->n->inputs[phi_i]->dt.type != TB_CONTROL && u->n->inputs[phi_i]->dt.type != TB_MEMORY) { - assert(phi_i >= 0); - if (pos == 0) { - c_fmt_output(ctx, u->n); - c_fmt_ref_to_node(ctx, u->n->inputs[phi_i]); - nl_buffer_format(ctx->buf, ";\n"); - } else { - c_fmt_output(ctx, u->n); - nl_buffer_format(ctx->buf, "a%zu;\n", ctx->a + pos); - } - pos += 1; - } - } - } - } - ctx->depth -= 1; - c_fmt_spaces(ctx); - nl_buffer_format(ctx->buf, "}\n"); - } - } - - c_fmt_spaces(ctx); - nl_buffer_format(ctx->buf, "goto bb%u;\n", target->gvn); -} - -static void c_fmt_bb(CFmtState* ctx, TB_Node* bb_start) { - size_t declared_vars_length = dyn_array_length(&ctx->declared_vars); - - nl_buffer_format(ctx->buf, "bb%u:\n", bb_start->gvn); - - TB_BasicBlock* bb = ctx->opt->scheduled[bb_start->gvn]; - if (bb == NULL) { - return; - } - Worklist* ws = &ctx->opt->worklist; - - // #ifndef NDEBUG - // TB_BasicBlock* expected = &nl_map_get_checked(ctx->cfg.node_to_block, bb_start); - // assert(expected == bb); - // #endif - - CFmtBlockRange *range = c_fmt_get_block_range(ctx, bb_start); - - TB_Node* prev_effect = NULL; - FOREACH_N(i, range->low, range->high) { - TB_Node* n = ws->items[i]; - - // skip these - if (c_fmt_will_inline(n) && n->type != TB_ROOT) { - continue; - } - - if (n->type == TB_MERGEMEM || n->type == TB_SPLITMEM || n->type == TB_NULL || n->type == TB_PHI || n->type == TB_PROJ) { - continue; - } - - switch (n->type) { - case TB_DEAD: { - c_fmt_spaces(ctx); - nl_buffer_format(ctx->buf, "// dead\n"); - break; - } - - case TB_DEBUGBREAK: { - c_fmt_spaces(ctx); - nl_buffer_format(ctx->buf, "// debugbreak\n"); - break; - } - case TB_UNREACHABLE: { - c_fmt_spaces(ctx); - nl_buffer_format(ctx->buf, "// unreachable\n"); - break; - } - - case TB_BRANCH: { - TB_NodeBranch* br = TB_NODE_GET_EXTRA(n); - TB_Node** restrict succ = tb_arena_alloc(ctx->arena, br->succ_count * sizeof(TB_Node**)); - - size_t succ_count = 0; - - // fill successors - FOR_USERS(u, n) { - if (u->n->type == TB_PROJ) { - int index = TB_NODE_GET_EXTRA_T(u->n, TB_NodeProj)->index; - succ[index] = u->n; - succ_count += 1; - } - } - - if (succ_count == 1) { - c_fmt_branch_edge(ctx, succ[0], false); - } else if (succ_count == 2) { - if (br->keys[0].key == 0) { - c_fmt_spaces(ctx); - nl_buffer_format(ctx->buf, "if ("); - FOREACH_N(i, 1, n->input_count) { - if (i != 1) nl_buffer_format(ctx->buf, ", "); - c_fmt_ref_to_node(ctx, n->inputs[i]); - } - nl_buffer_format(ctx->buf, ") {\n"); - ctx->depth += 1; - c_fmt_branch_edge(ctx, succ[0], false); - ctx->depth -= 1; - c_fmt_spaces(ctx); - nl_buffer_format(ctx->buf, "} else {\n"); - ctx->depth += 1; - c_fmt_branch_edge(ctx, succ[1], false); - ctx->depth -= 1; - c_fmt_spaces(ctx); - nl_buffer_format(ctx->buf, "}\n"); - } else { - c_fmt_spaces(ctx); - nl_buffer_format(ctx->buf, "if ((uint64_t) "); - FOREACH_N(i, 1, n->input_count) { - if (i != 1) nl_buffer_format(ctx->buf, ", "); - c_fmt_ref_to_node(ctx, n->inputs[i]); - } - nl_buffer_format(ctx->buf, " == (uint64_t) %"PRIi64, br->keys[0].key); - nl_buffer_format(ctx->buf, ") {\n"); - ctx->depth += 1; - c_fmt_branch_edge(ctx, succ[1], false); - ctx->depth -= 1; - c_fmt_spaces(ctx); - nl_buffer_format(ctx->buf, "} else {\n"); - ctx->depth += 1; - c_fmt_branch_edge(ctx, succ[0], false); - ctx->depth -= 1; - c_fmt_spaces(ctx); - nl_buffer_format(ctx->buf, "}\n"); - } - } else { - c_fmt_spaces(ctx); - nl_buffer_format(ctx->buf, "switch ((uint64_t) "); - FOREACH_N(i, 1, n->input_count) { - if (i != 1) nl_buffer_format(ctx->buf, ", "); - c_fmt_ref_to_node(ctx, n->inputs[i]); - } - nl_buffer_format(ctx->buf, ") {\n"); - FOREACH_N(i, 1, succ_count) { - c_fmt_spaces(ctx); - nl_buffer_format(ctx->buf, "case %"PRIi64"llu:\n", br->keys[i-1].key); - ctx->depth += 1; - c_fmt_branch_edge(ctx, succ[i], false); - ctx->depth -= 1; - } - c_fmt_spaces(ctx); - nl_buffer_format(ctx->buf, "default:\n"); - ctx->depth += 1; - c_fmt_branch_edge(ctx, succ[0], false); - ctx->depth -= 1; - c_fmt_spaces(ctx); - nl_buffer_format(ctx->buf, "}\n"); - } - break; - } - - case TB_TRAP: { - c_fmt_spaces(ctx); - nl_buffer_format(ctx->buf, "throw new Error(\"trap\");\n"); - break; - } - - case TB_TAILCALL: { - TB_Node *func = n->inputs[2]; - - TB_FunctionPrototype *proto = ctx->f->prototype; - size_t nrets = proto->return_count; - TB_PrototypeParam *rets = &proto->params[proto->param_count]; - - if (!nl_hashset_put(&ctx->declared_types, n)) { - if (nrets == 0) { - nl_buffer_format(ctx->globals, "typedef void(*tb2c_%s_v%u_t)(", ctx->name, n->gvn); - } else if (nrets == 1) { - nl_buffer_format(ctx->globals, "typedef %s(*tb2c_%s_v%u_t)(", c_fmt_type_name(rets[0].dt), ctx->name, n->gvn); - } else { - nl_buffer_format(ctx->globals, "typedef struct {\n"); - size_t index = 0; - FOREACH_N(i, 0, nrets) { - nl_buffer_format(ctx->globals, " %s v%zu;\n", c_fmt_type_name(rets[i].dt), index); - index += 1; - } - nl_buffer_format(ctx->globals, "} tb2c_%s_v%u_ret_t;\n", ctx->name, n->gvn); - nl_buffer_format(ctx->globals, "typedef tb2c_%s_v%u_ret_t(*tb2c_%s_v%u_t)(", ctx->name, n->gvn, ctx->name, n->gvn); - } - bool first = true; - FOREACH_N(i, 3, n->input_count) { - if (n->inputs[i]->dt.type != TB_CONTROL && n->inputs[i]->dt.type != TB_MEMORY) { - if (!first) { - nl_buffer_format(ctx->globals, ", "); - } - nl_buffer_format(ctx->globals, "%s", c_fmt_type_name(n->inputs[i]->dt)); - first = false; - } - } - if (first) { - nl_buffer_format(ctx->globals, "void"); - } - nl_buffer_format(ctx->globals, ");\n"); - } - if (nrets == 0 || nrets == 1) { - c_fmt_spaces(ctx); - if (nrets == 1) { - nl_buffer_format(ctx->buf, "return (tb2c_%s_ret_t) ", ctx->name); - } - nl_buffer_format(ctx->buf, "((tb2c_%s_v%u_t) ", ctx->name, n->gvn); - c_fmt_ref_to_node(ctx, func); - nl_buffer_format(ctx->buf, ")"); - nl_buffer_format(ctx->buf, "("); - { - bool first = true; - FOREACH_N(i, 3, n->input_count) { - if (n->inputs[i]->dt.type != TB_CONTROL && n->inputs[i]->dt.type != TB_MEMORY) { - if (!first) { - nl_buffer_format(ctx->buf, ", "); - } - c_fmt_ref_to_node(ctx, n->inputs[i]); - first = false; - } - } - } - nl_buffer_format(ctx->buf, ");\n"); - if (nrets == 0) { - c_fmt_spaces(ctx); - nl_buffer_format(ctx->buf, "return;\n"); - } - } else { - c_fmt_spaces(ctx); - nl_buffer_format(ctx->buf, "{\n"); - ctx->depth += 1; - c_fmt_spaces(ctx); - nl_buffer_format(ctx->buf, "tb2c_%s_v%u_ret_t v%u_ret = ", ctx->name, n->gvn, n->gvn); - nl_buffer_format(ctx->buf, "((tb2c_%s_v%u_t) ", ctx->name, n->gvn); - c_fmt_ref_to_node(ctx, func); - nl_buffer_format(ctx->buf, ")("); - { - bool first = true; - FOREACH_N(i, 3, n->input_count) { - if (n->inputs[i]->dt.type != TB_CONTROL && n->inputs[i]->dt.type != TB_MEMORY) { - if (!first) { - nl_buffer_format(ctx->buf, ", "); - } - c_fmt_ref_to_node(ctx, n->inputs[i]); - first = false; - } - } - } - nl_buffer_format(ctx->buf, ");\n"); - c_fmt_spaces(ctx); - nl_buffer_format(ctx->buf, "tb2c_%s_ret_t ret;\n", ctx->name); - size_t index = 0; - FOREACH_N(i, 0, nrets) { - c_fmt_spaces(ctx); - nl_buffer_format(ctx->buf, "ret.v%zu = v%u_ret.v%zu;\n", index, n->gvn, index); - index += 1; - } - c_fmt_spaces(ctx); - nl_buffer_format(ctx->buf, "return (tb2c_%s_ret_t) ret;\n", ctx->name); - ctx->depth -= 1; - c_fmt_spaces(ctx); - nl_buffer_format(ctx->buf, "}\n"); - } - break; - } - - case TB_RETURN: { - size_t count = 0; - FOREACH_N(i, 3, n->input_count) { - count += 1; - } - if (count == 0) { - c_fmt_spaces(ctx); - nl_buffer_format(ctx->buf, "return;\n"); - } else if (count == 1) { - c_fmt_spaces(ctx); - nl_buffer_format(ctx->buf, "return "); - FOREACH_N(i, 3, n->input_count) { - c_fmt_ref_to_node(ctx, n->inputs[i]); - } - nl_buffer_format(ctx->buf, ";\n"); - } else { - c_fmt_spaces(ctx); - nl_buffer_format(ctx->buf, "{\n"); - c_fmt_spaces(ctx); - nl_buffer_format(ctx->buf, " tb2c_%s_ret_t ret;\n", ctx->name); - - size_t index = 0; - FOREACH_N(i, 3, n->input_count) { - c_fmt_spaces(ctx); - nl_buffer_format(ctx->buf, " ret.v%zu = ", index); - c_fmt_ref_to_node(ctx, n->inputs[i]); - nl_buffer_format(ctx->buf, ";\n"); - index += 1; - } - c_fmt_spaces(ctx); - nl_buffer_format(ctx->buf, " return ret;\n"); - c_fmt_spaces(ctx); - nl_buffer_format(ctx->buf, "}\n"); - } - break; - } - - case TB_NATURAL_LOOP: - case TB_AFFINE_LOOP: - case TB_CALLGRAPH: { - break; - } - - case TB_STORE: { - TB_Node *dest = n->inputs[n->input_count-2]; - TB_Node *src = n->inputs[n->input_count-1]; - c_fmt_spaces(ctx); - nl_buffer_format(ctx->buf, "*(%s*) ", c_fmt_type_name(src->dt)); - c_fmt_ref_to_node(ctx, dest); - nl_buffer_format(ctx->buf, " = "); - c_fmt_ref_to_node(ctx, src); - nl_buffer_format(ctx->buf, ";\n"); - break; - } - - case TB_NOT: { - TB_Node *src = n->inputs[n->input_count-1]; - c_fmt_output(ctx, n); - nl_buffer_format(ctx->buf, "~"); - c_fmt_ref_to_node(ctx, src); - nl_buffer_format(ctx->buf, ";\n"); - break; - } - - case TB_NEG: { - TB_Node *src = n->inputs[n->input_count-1]; - c_fmt_output(ctx, n); - nl_buffer_format(ctx->buf, "-"); - c_fmt_ref_to_node(ctx, src); - nl_buffer_format(ctx->buf, ";\n"); - break; - } - - case TB_LOAD: { - TB_Node *src = n->inputs[n->input_count-1]; - c_fmt_output(ctx, n); - nl_buffer_format(ctx->buf, "*(%s*) ", c_fmt_type_name(n->dt)); - c_fmt_ref_to_node(ctx, src); - nl_buffer_format(ctx->buf, ";\n"); - break; - } - - case TB_LOCAL: { - if (!nl_hashset_put(&ctx->declared_vars, n)) { - TB_NodeLocal* l = TB_NODE_GET_EXTRA(n); - nl_buffer_format(ctx->pre, " uint8_t v%u[0x%x];\n", n->gvn, l->size); - } - break; - } - - case TB_BITCAST: { - TB_Node *src = n->inputs[n->input_count-1]; - if (src->dt.type == TB_FLOAT || n->dt.type == TB_FLOAT) { - c_fmt_spaces(ctx); - nl_buffer_format(ctx->buf, "{\n"); - ctx->depth += 1; - c_fmt_spaces(ctx); - nl_buffer_format(ctx->buf, "union {%s src; %s dest;} tmp;\n", c_fmt_type_name(src->dt), c_fmt_type_name(n->dt)); - c_fmt_spaces(ctx); - nl_buffer_format(ctx->buf, "tmp.src = "); - c_fmt_ref_to_node(ctx, src); - nl_buffer_format(ctx->buf, ";\n"); - c_fmt_output(ctx, n); - nl_buffer_format(ctx->buf, "tmp.dest;\n"); - ctx->depth -= 1; - c_fmt_spaces(ctx); - nl_buffer_format(ctx->buf, "}\n"); - } else { - c_fmt_output(ctx, n); - nl_buffer_format(ctx->buf, "(%s) ", c_fmt_type_name(n->dt)); - c_fmt_ref_to_node(ctx, src); - nl_buffer_format(ctx->buf, ";\n"); - } - break; - } - - case TB_OR: { - TB_Node *lhs = n->inputs[n->input_count-2]; - TB_Node *rhs = n->inputs[n->input_count-1]; - c_fmt_output(ctx, n); - nl_buffer_format(ctx->buf, "(%s) ", c_fmt_type_name(n->dt)); - c_fmt_ref_to_node(ctx, lhs); - nl_buffer_format(ctx->buf, " | "); - nl_buffer_format(ctx->buf, "(%s) ", c_fmt_type_name(n->dt)); - c_fmt_ref_to_node(ctx, rhs); - nl_buffer_format(ctx->buf, ";\n"); - break; - } - - case TB_XOR: { - TB_Node *lhs = n->inputs[n->input_count-2]; - TB_Node *rhs = n->inputs[n->input_count-1]; - c_fmt_output(ctx, n); - nl_buffer_format(ctx->buf, "(%s) ", c_fmt_type_name(n->dt)); - c_fmt_ref_to_node(ctx, lhs); - nl_buffer_format(ctx->buf, " ^ "); - nl_buffer_format(ctx->buf, "(%s) ", c_fmt_type_name(n->dt)); - c_fmt_ref_to_node(ctx, rhs); - nl_buffer_format(ctx->buf, ";\n"); - break; - } - case TB_SHR: { - TB_Node *lhs = n->inputs[n->input_count-2]; - TB_Node *rhs = n->inputs[n->input_count-1]; - c_fmt_output(ctx, n); - nl_buffer_format(ctx->buf, "(%s) ", c_fmt_type_name(n->dt)); - c_fmt_ref_to_node(ctx, lhs); - nl_buffer_format(ctx->buf, " >> "); - nl_buffer_format(ctx->buf, "(%s) ", c_fmt_type_name(n->dt)); - c_fmt_ref_to_node(ctx, rhs); - nl_buffer_format(ctx->buf, ";\n"); - break; - } - case TB_SAR: { - TB_Node *lhs = n->inputs[n->input_count-2]; - TB_Node *rhs = n->inputs[n->input_count-1]; - c_fmt_output(ctx, n); - nl_buffer_format(ctx->buf, "(%s) ", c_fmt_type_name(n->dt)); - c_fmt_ref_to_node(ctx, lhs); - nl_buffer_format(ctx->buf, " >> "); - nl_buffer_format(ctx->buf, "(%s) ", c_fmt_type_name(n->dt)); - c_fmt_ref_to_node(ctx, rhs); - nl_buffer_format(ctx->buf, ";\n"); - break; - } - case TB_SHL: { - TB_Node *lhs = n->inputs[n->input_count-2]; - TB_Node *rhs = n->inputs[n->input_count-1]; - c_fmt_output(ctx, n); - nl_buffer_format(ctx->buf, "(%s) ", c_fmt_type_name(n->dt)); - c_fmt_ref_to_node(ctx, lhs); - nl_buffer_format(ctx->buf, " << "); - nl_buffer_format(ctx->buf, "(%s) ", c_fmt_type_name(n->dt)); - c_fmt_ref_to_node(ctx, rhs); - nl_buffer_format(ctx->buf, ";\n"); - break; - } - - case TB_AND: { - TB_Node *lhs = n->inputs[n->input_count-2]; - TB_Node *rhs = n->inputs[n->input_count-1]; - c_fmt_output(ctx, n); - nl_buffer_format(ctx->buf, "(%s) ", c_fmt_type_name(n->dt)); - c_fmt_ref_to_node(ctx, lhs); - nl_buffer_format(ctx->buf, " & "); - nl_buffer_format(ctx->buf, "(%s) ", c_fmt_type_name(n->dt)); - c_fmt_ref_to_node(ctx, rhs); - nl_buffer_format(ctx->buf, ";\n"); - break; - } - - case TB_FADD: - case TB_ADD: { - TB_Node *lhs = n->inputs[n->input_count-2]; - TB_Node *rhs = n->inputs[n->input_count-1]; - c_fmt_output(ctx, n); - c_fmt_typed_ref_to_node(ctx, lhs->dt, lhs); - nl_buffer_format(ctx->buf, " + "); - c_fmt_typed_ref_to_node(ctx, rhs->dt, rhs); - nl_buffer_format(ctx->buf, ";\n"); - break; - } - case TB_FSUB: - case TB_SUB: { - TB_Node *lhs = n->inputs[n->input_count-2]; - TB_Node *rhs = n->inputs[n->input_count-1]; - c_fmt_output(ctx, n); - c_fmt_typed_ref_to_node(ctx, lhs->dt, lhs); - nl_buffer_format(ctx->buf, " - "); - c_fmt_typed_ref_to_node(ctx, rhs->dt, rhs); - nl_buffer_format(ctx->buf, ";\n"); - break; - } - case TB_FMUL: - case TB_MUL: { - TB_Node *lhs = n->inputs[n->input_count-2]; - TB_Node *rhs = n->inputs[n->input_count-1]; - c_fmt_output(ctx, n); - c_fmt_typed_ref_to_node(ctx, lhs->dt, lhs); - nl_buffer_format(ctx->buf, " * "); - c_fmt_typed_ref_to_node(ctx, rhs->dt, rhs); - nl_buffer_format(ctx->buf, ";\n"); - break; - } - case TB_FDIV: { - TB_Node *lhs = n->inputs[n->input_count-2]; - TB_Node *rhs = n->inputs[n->input_count-1]; - c_fmt_output(ctx, n); - c_fmt_typed_ref_to_node(ctx, lhs->dt, lhs); - nl_buffer_format(ctx->buf, " / "); - c_fmt_typed_ref_to_node(ctx, rhs->dt, rhs); - nl_buffer_format(ctx->buf, ";\n"); - break; - } - case TB_SDIV: { - TB_Node *lhs = n->inputs[n->input_count-2]; - TB_Node *rhs = n->inputs[n->input_count-1]; - c_fmt_output(ctx, n); - nl_buffer_format(ctx->buf, "(%s) ", c_fmt_type_name_signed(n->dt)); - c_fmt_typed_ref_to_node(ctx, lhs->dt, lhs); - nl_buffer_format(ctx->buf, " / "); - nl_buffer_format(ctx->buf, "(%s) ", c_fmt_type_name_signed(n->dt)); - c_fmt_typed_ref_to_node(ctx, rhs->dt, rhs); - nl_buffer_format(ctx->buf, ";\n"); - break; - } - case TB_UDIV: { - TB_Node *lhs = n->inputs[n->input_count-2]; - TB_Node *rhs = n->inputs[n->input_count-1]; - c_fmt_output(ctx, n); - c_fmt_typed_ref_to_node(ctx, lhs->dt, lhs); - nl_buffer_format(ctx->buf, " / "); - c_fmt_typed_ref_to_node(ctx, rhs->dt, rhs); - nl_buffer_format(ctx->buf, ";\n"); - break; - } - case TB_SMOD: { - TB_Node *lhs = n->inputs[n->input_count-2]; - TB_Node *rhs = n->inputs[n->input_count-1]; - c_fmt_output(ctx, n); - nl_buffer_format(ctx->buf, "(%s) ", c_fmt_type_name_signed(n->dt)); - c_fmt_typed_ref_to_node(ctx, lhs->dt, lhs); - nl_buffer_format(ctx->buf, " %% "); - nl_buffer_format(ctx->buf, "(%s) ", c_fmt_type_name_signed(n->dt)); - c_fmt_typed_ref_to_node(ctx, rhs->dt, rhs); - nl_buffer_format(ctx->buf, ";\n"); - break; - } - case TB_UMOD: { - TB_Node *lhs = n->inputs[n->input_count-2]; - TB_Node *rhs = n->inputs[n->input_count-1]; - c_fmt_output(ctx, n); - c_fmt_typed_ref_to_node(ctx, lhs->dt, lhs); - nl_buffer_format(ctx->buf, " %% "); - c_fmt_typed_ref_to_node(ctx, rhs->dt, rhs); - nl_buffer_format(ctx->buf, ";\n"); - break; - } - - case TB_CMP_EQ: { - TB_Node *lhs = n->inputs[n->input_count-2]; - TB_Node *rhs = n->inputs[n->input_count-1]; - c_fmt_output(ctx, n); - c_fmt_typed_ref_to_node(ctx, lhs->dt, lhs); - nl_buffer_format(ctx->buf, " == "); - c_fmt_typed_ref_to_node(ctx, rhs->dt, rhs); - nl_buffer_format(ctx->buf, ";\n"); - break; - } - - case TB_CMP_NE: { - TB_Node *lhs = n->inputs[n->input_count-2]; - TB_Node *rhs = n->inputs[n->input_count-1]; - c_fmt_output(ctx, n); - c_fmt_typed_ref_to_node(ctx, lhs->dt, lhs); - nl_buffer_format(ctx->buf, " != "); - c_fmt_typed_ref_to_node(ctx, rhs->dt, rhs); - nl_buffer_format(ctx->buf, ";\n"); - break; - } - - case TB_POISON: { - break; - } - - case TB_CMP_FLT: { - TB_Node *lhs = n->inputs[n->input_count-2]; - TB_Node *rhs = n->inputs[n->input_count-1]; - c_fmt_output(ctx, n); - c_fmt_typed_ref_to_node(ctx, lhs->dt, lhs); - nl_buffer_format(ctx->buf, " < "); - c_fmt_typed_ref_to_node(ctx, rhs->dt, rhs); - nl_buffer_format(ctx->buf, ";\n"); - break; - } - - case TB_CMP_FLE: { - TB_Node *lhs = n->inputs[n->input_count-2]; - TB_Node *rhs = n->inputs[n->input_count-1]; - c_fmt_output(ctx, n); - c_fmt_typed_ref_to_node(ctx, lhs->dt, lhs); - nl_buffer_format(ctx->buf, " <= "); - c_fmt_typed_ref_to_node(ctx, rhs->dt, rhs); - nl_buffer_format(ctx->buf, ";\n"); - break; - } - - - case TB_CMP_SLT: { - TB_Node *lhs = n->inputs[n->input_count-2]; - TB_Node *rhs = n->inputs[n->input_count-1]; - c_fmt_output(ctx, n); - nl_buffer_format(ctx->buf, "(%s) ", c_fmt_type_name_signed(lhs->dt)); - c_fmt_typed_ref_to_node(ctx, lhs->dt, lhs); - nl_buffer_format(ctx->buf, " < "); - nl_buffer_format(ctx->buf, "(%s) ", c_fmt_type_name_signed(lhs->dt)); - c_fmt_typed_ref_to_node(ctx, rhs->dt, rhs); - nl_buffer_format(ctx->buf, ";\n"); - break; - } - - case TB_CMP_SLE: { - TB_Node *lhs = n->inputs[n->input_count-2]; - TB_Node *rhs = n->inputs[n->input_count-1]; - c_fmt_output(ctx, n); - nl_buffer_format(ctx->buf, "(%s) ", c_fmt_type_name_signed(lhs->dt)); - c_fmt_typed_ref_to_node(ctx, lhs->dt, lhs); - nl_buffer_format(ctx->buf, " <= "); - nl_buffer_format(ctx->buf, "(%s) ", c_fmt_type_name_signed(lhs->dt)); - c_fmt_typed_ref_to_node(ctx, rhs->dt, rhs); - nl_buffer_format(ctx->buf, ";\n"); - break; - } - - - case TB_CMP_ULT: { - TB_Node *lhs = n->inputs[n->input_count-2]; - TB_Node *rhs = n->inputs[n->input_count-1]; - c_fmt_output(ctx, n); - c_fmt_typed_ref_to_node(ctx, lhs->dt, lhs); - nl_buffer_format(ctx->buf, " < "); - c_fmt_typed_ref_to_node(ctx, rhs->dt, rhs); - nl_buffer_format(ctx->buf, ";\n"); - break; - } - - case TB_CMP_ULE: { - TB_Node *lhs = n->inputs[n->input_count-2]; - TB_Node *rhs = n->inputs[n->input_count-1]; - c_fmt_output(ctx, n); - c_fmt_ref_to_node(ctx, lhs); - nl_buffer_format(ctx->buf, " <= "); - nl_buffer_format(ctx->buf, "(%s) ", c_fmt_type_name(lhs->dt)); - c_fmt_ref_to_node(ctx, rhs); - nl_buffer_format(ctx->buf, ";\n"); - break; - } - - case TB_MEMBER_ACCESS: { - TB_Node *ptr = n->inputs[n->input_count-1]; - c_fmt_output(ctx, n); - nl_buffer_format(ctx->buf, "(void*) ((char *) "); - c_fmt_ref_to_node(ctx, ptr); - nl_buffer_format(ctx->buf, " + %"PRIi64, TB_NODE_GET_EXTRA_T(n, TB_NodeMember)->offset); - nl_buffer_format(ctx->buf, ");\n"); - break; - } - - - case TB_SELECT: { - TB_Node *cond = n->inputs[n->input_count-3]; - TB_Node *then = n->inputs[n->input_count-2]; - TB_Node *els = n->inputs[n->input_count-1]; - c_fmt_output(ctx, n); - c_fmt_ref_to_node(ctx, cond); - nl_buffer_format(ctx->buf, " ? "); - c_fmt_ref_to_node(ctx, then); - nl_buffer_format(ctx->buf, " : "); - c_fmt_ref_to_node(ctx, els); - nl_buffer_format(ctx->buf, ";\n"); - break; - } - - case TB_FLOAT2UINT: - case TB_FLOAT2INT: - case TB_UINT2FLOAT: - case TB_INT2FLOAT: - case TB_TRUNCATE: { - TB_Node *input = n->inputs[n->input_count-1]; - c_fmt_output(ctx, n); - nl_buffer_format(ctx->buf, "(%s) ", c_fmt_type_name(n->dt)); - c_fmt_ref_to_node(ctx, input); - nl_buffer_format(ctx->buf, ";\n"); - break; - } - - case TB_ARRAY_ACCESS: { - TB_Node *ptr = n->inputs[n->input_count-2]; - TB_Node *index = n->inputs[n->input_count-1]; - c_fmt_output(ctx, n); - nl_buffer_format(ctx->buf, "(void*) ((char *) "); - c_fmt_ref_to_node(ctx, ptr); - nl_buffer_format(ctx->buf, " + "); - c_fmt_ref_to_node(ctx, index); - nl_buffer_format(ctx->buf, " * %"PRIi64, TB_NODE_GET_EXTRA_T(n, TB_NodeArray)->stride); - nl_buffer_format(ctx->buf, ");\n"); - break; - } - case TB_CALL: { - TB_Node *func = n->inputs[2]; - - TB_Node* projs[4] = { 0 }; - FOR_USERS(use, n) { - if (use->n->type == TB_PROJ) { - int index = TB_NODE_GET_EXTRA_T(use->n, TB_NodeProj)->index; - projs[index] = use->n; - } - } - - if (projs[2] == NULL) { - nl_buffer_format(ctx->globals, "typedef void(*tb2c_%s_v%u_t)(", ctx->name, n->gvn); - } else if (projs[3] == NULL) { - nl_buffer_format(ctx->globals, "typedef %s(*tb2c_%s_v%u_t)(", c_fmt_type_name(projs[2]->dt), ctx->name, n->gvn); - } else { - nl_buffer_format(ctx->globals, "typedef struct {\n"); - FOREACH_N(i, 2, 4) { - if (projs[i] == NULL) break; - nl_buffer_format(ctx->globals, " %s v%u;\n", c_fmt_type_name(projs[i]->dt), projs[i]->gvn); - } - nl_buffer_format(ctx->globals, "} tb2c_%s_v%u_ret_t;\n", ctx->name, n->gvn); - nl_buffer_format(ctx->globals, "typedef tb2c_%s_v%u_ret_t(*tb2c_%s_v%u_t)(", ctx->name, n->gvn, ctx->name, n->gvn); - } - { - bool first = true; - FOREACH_N(i, 3, n->input_count) { - if (n->inputs[i]->dt.type != TB_CONTROL && n->inputs[i]->dt.type != TB_MEMORY) { - if (!first) { - nl_buffer_format(ctx->globals, ", "); - } - nl_buffer_format(ctx->globals, "%s", c_fmt_type_name(n->inputs[i]->dt)); - first = false; - } - } - if (first) { - nl_buffer_format(ctx->globals, "void"); - } - } - nl_buffer_format(ctx->globals, ");\n"); - if (projs[2] == NULL || projs[3] == NULL) { - if (projs[2] != NULL) { - c_fmt_output(ctx, projs[2]); - } else { - c_fmt_spaces(ctx); - } - nl_buffer_format(ctx->buf, "((tb2c_%s_v%u_t) ", ctx->name, n->gvn); - c_fmt_ref_to_node(ctx, func); - nl_buffer_format(ctx->buf, ")"); - nl_buffer_format(ctx->buf, "("); - { - bool first = true; - FOREACH_N(i, 3, n->input_count) { - if (n->inputs[i]->dt.type != TB_CONTROL && n->inputs[i]->dt.type != TB_MEMORY) { - if (!first) { - nl_buffer_format(ctx->buf, ", "); - } - c_fmt_ref_to_node(ctx, n->inputs[i]); - first = false; - } - } - } - nl_buffer_format(ctx->buf, ");\n"); - } else { - c_fmt_spaces(ctx); - nl_buffer_format(ctx->buf, "{\n"); - ctx->depth += 1; - c_fmt_spaces(ctx); - nl_buffer_format(ctx->buf, "tb2c_%s_v%u_ret_t ret = ", ctx->name, n->gvn); - nl_buffer_format(ctx->buf, "((tb2c_%s_v%u_t) ", ctx->name, n->gvn); - c_fmt_ref_to_node(ctx, func); - nl_buffer_format(ctx->buf, ")("); - { - bool first = true; - FOREACH_N(i, 3, n->input_count) { - if (n->inputs[i]->dt.type != TB_CONTROL && n->inputs[i]->dt.type != TB_MEMORY) { - if (!first) { - nl_buffer_format(ctx->buf, ", "); - } - c_fmt_ref_to_node(ctx, n->inputs[i]); - first = false; - } - } - } - nl_buffer_format(ctx->buf, ");\n"); - if (projs[2] != NULL) { - FOREACH_N(i, 2, 4) { - if (projs[i] == NULL) break; - c_fmt_output(ctx, projs[i]); - nl_buffer_format(ctx->buf, "ret.v%u;\n", projs[i]->gvn); - } - } - ctx->depth -= 1; - c_fmt_spaces(ctx); - nl_buffer_format(ctx->buf, "}\n"); - } - break; - } - - case TB_MEMCPY: { - TB_Node *dest = n->inputs[n->input_count-3]; - TB_Node *src = n->inputs[n->input_count-2]; - TB_Node *len = n->inputs[n->input_count-1]; - c_fmt_spaces(ctx); - nl_buffer_format(ctx->buf, "memcpy("); - c_fmt_ref_to_node(ctx, dest); - nl_buffer_format(ctx->buf, ", "); - c_fmt_ref_to_node(ctx, src); - nl_buffer_format(ctx->buf, ", "); - c_fmt_ref_to_node(ctx, len); - nl_buffer_format(ctx->buf, ");\n"); - break; - } - - case TB_MEMSET: { - TB_Node *a = n->inputs[n->input_count-3]; - TB_Node *b = n->inputs[n->input_count-2]; - TB_Node *c = n->inputs[n->input_count-1]; - c_fmt_spaces(ctx); - nl_buffer_format(ctx->buf, "memset("); - c_fmt_ref_to_node(ctx, a); - nl_buffer_format(ctx->buf, ", "); - c_fmt_ref_to_node(ctx, b); - nl_buffer_format(ctx->buf, ", "); - c_fmt_ref_to_node(ctx, c); - nl_buffer_format(ctx->buf, ");\n"); - break; - } - - default: { - fprintf(stderr, "internal unimplemented node type: %s\n", tb_node_get_name(n)); - fflush(stderr); - __builtin_trap(); - break; - } - } - } - - // dyn_array_set_length(ws->items, ctx->cfg.block_count); - - if (!cfg_is_terminator(bb->end)) { - c_fmt_branch_edge(ctx, bb->end, true); - } - - // nl_buffer_t *buf = ctx->buf; - // ctx->buf = old_buf; - // nl_buffer_format(ctx->buf, "%s", nl_buffer_get(buf)); -} - -TB_API char *tb_pass_c_prelude(TB_Module *mod) { - size_t n_private_syms = 0; - - void *arena = tb_arena_create(TB_ARENA_MEDIUM_CHUNK_SIZE); - - nl_buffer_t *buf = nl_buffer_new(); - - nl_buffer_format(buf, "typedef signed char int8_t;\n"); - nl_buffer_format(buf, "typedef unsigned char uint8_t;\n"); - nl_buffer_format(buf, "typedef signed short int16_t;\n"); - nl_buffer_format(buf, "typedef unsigned short uint16_t;\n"); - nl_buffer_format(buf, "typedef signed int int32_t;\n"); - nl_buffer_format(buf, "typedef unsigned int uint32_t;\n"); - if (sizeof(long) == 8) { - nl_buffer_format(buf, "typedef signed long int64_t;\n"); - nl_buffer_format(buf, "typedef unsigned long uint64_t;\n"); - } else { - nl_buffer_format(buf, "typedef signed long long int64_t;\n"); - nl_buffer_format(buf, "typedef unsigned long long uint64_t;\n"); - } - if (sizeof(size_t) == sizeof(uint64_t)) { - nl_buffer_format(buf, "typedef uint64_t size_t;\n"); - } - if (sizeof(size_t) == sizeof(uint32_t)) { - nl_buffer_format(buf, "typedef uint32_t size_t;\n"); - } - nl_buffer_format(buf, "void *memcpy(void *dest, const void *src, size_t n);\n"); - nl_buffer_format(buf, "void *memset(void *str, int c, size_t n);\n"); - nl_buffer_format(buf, "\n"); - NL_HashSet* syms = &tb_thread_info(mod)->symbols; - nl_hashset_for(sym_vp, syms) { - TB_Symbol *sym = *sym_vp; - if (sym->name == NULL ||sym->name[0] == '\0') { - continue; - } - switch ((int) sym->tag) { - case TB_SYMBOL_EXTERNAL: { - if (mod->is_jit) { - nl_buffer_format(buf, "static void *%s = (void *) 0x%zx;\n", sym->name, (size_t) sym->address); - } else { - nl_buffer_format(buf, "extern void %s(void);\n", sym->name); - } - break; - } - case TB_SYMBOL_GLOBAL: { - TB_Global *g = (void *) sym; - if (g->linkage == TB_LINKAGE_PRIVATE) { - nl_buffer_format(buf, "static "); - sym->symbol_id = ++n_private_syms; - } else { - sym->symbol_id = 0; - } - if (g->obj_count == 0) { - nl_buffer_format(buf, "uint8_t tb2c_sym%zu_%s[%zu];\n", (size_t) sym->symbol_id, sym->name, (size_t) g->size); - } else { - uint8_t *data_buf = tb_platform_heap_alloc(sizeof(uint8_t) * g->size); - memset(data_buf, 0, sizeof(uint8_t) * g->size); - FOREACH_N(k, 0, g->obj_count) { - if (g->objects[k].type == TB_INIT_OBJ_REGION) { - assert(g->objects[k].offset + g->objects[k].region.size <= g->size); - memcpy(&data_buf[g->pos + g->objects[k].offset], g->objects[k].region.ptr, g->objects[k].region.size); - } - } - nl_buffer_format(buf, "uint8_t tb2c_sym%zu_%s[%zu] = {\n", (size_t) sym->symbol_id, sym->name, (size_t) g->size); - for (size_t i = 0; i < g->size; i++) { - nl_buffer_format(buf, " 0x%02x,\n", data_buf[i]); - } - nl_buffer_format(buf, "};\n"); - } - // nl_buffer_format(buf, "// global: %s\n", sym->name); - break; - } - case TB_SYMBOL_FUNCTION: { - TB_Function *f = (void *) sym; - TB_FunctionPrototype* p = f->prototype; - if (p->return_count == 0) { - nl_buffer_format(buf, "typedef void tb2c_%s_ret_t;\n", sym->name); - } else if (p->return_count == 1) { - if (p->params[p->param_count].dt.type == TB_INT && p->params[p->param_count].dt.data == 32 && !strcmp(sym->name, "main")) { - nl_buffer_format(buf, "#define tb2c_%s_ret_t int\n", sym->name); - nl_buffer_format(buf, "#define main(...) main(int v4, char **v5)\n"); - } else { - nl_buffer_format(buf, "typedef %s tb2c_%s_ret_t;\n", c_fmt_type_name(p->params[p->param_count].dt), sym->name); - } - } else { - nl_buffer_format(buf, "typedef struct {\n"); - - size_t index = 0; - FOREACH_N(i, 0, p->return_count) { - nl_buffer_format(buf, " %s v%zu;\n", c_fmt_type_name(p->params[p->param_count + i].dt), index); - index += 1; - } - nl_buffer_format(buf, "} tb2c_%s_ret_t;\n", sym->name); - } - if (f->linkage == TB_LINKAGE_PRIVATE) { - nl_buffer_format(buf, "static "); - } - nl_buffer_format(buf, "tb2c_%s_ret_t %s(", sym->name, sym->name); - TB_Node** params = f->params; - size_t count = 0; - FOREACH_N(i, 3, 3 + f->param_count) { - if (params[i] != NULL && params[i]->dt.type != TB_MEMORY && params[i]->dt.type != TB_CONTROL && params[i]->dt.type != TB_TUPLE) { - if (count != 0) { - nl_buffer_format(buf, ", "); - } - nl_buffer_format(buf, "%s v%u", c_fmt_type_name(params[i]->dt), params[i]->gvn); - count += 1; - } - } - if (count == 0) { - nl_buffer_format(buf, "void"); - } - nl_buffer_format(buf, ");\n"); - break; - } - } - } - - char *ret = nl_buffer_get(buf); - - // tb_arena_destroy(arena); - - return ret; -} - -TB_API char *tb_pass_c_fmt(TB_Passes* opt) { - TB_Function* f = opt->f; - const char *name = f->super.name; - cuikperf_region_start("print_c", NULL); - - Worklist old = opt->worklist; - Worklist tmp_ws = { 0 }; - worklist_alloc(&tmp_ws, f->node_count); - - CFmtState ctx = { name, opt, f, f->super.module }; - - ctx.arena = tb_arena_create(TB_ARENA_MEDIUM_CHUNK_SIZE); - - ctx.globals = nl_buffer_new(); - - ctx.buf = nl_buffer_new(); - ctx.pre = nl_buffer_new(); - - // ctx.global_funcs = nl_hashset_alloc(ctx.module->compiled_function_count); - ctx.declared_types = nl_hashset_alloc(4); - ctx.visited_blocks = dyn_array_create(size_t, 8); - ctx.declared_vars = nl_hashset_alloc(16); - ctx.block_ranges = nl_table_alloc(ctx.cfg.block_count); - - // nl_hashset_clear(&ctx.visited_blocks); - - opt->worklist = tmp_ws; - ctx.cfg = tb_compute_rpo(f, opt); - - // does the IR printing need smart scheduling lol (yes... when we're testing things) - ctx.sched = greedy_scheduler; - - - // schedule nodes - tb_pass_schedule(opt, ctx.cfg, false); - worklist_clear_visited(&opt->worklist); - - // TB_Node* end_bb = NULL; - FOREACH_N(i, 0, ctx.cfg.block_count) { - if (opt->worklist.items[i] != NULL) { - ctx.depth += 1; - c_fmt_bb(&ctx, opt->worklist.items[i]); - ctx.depth -= 1; - } - } - - worklist_free(&opt->worklist); - // tb_free_cfg(&ctx.cfg); - opt->worklist = old; - opt->scheduled = NULL; - opt->error_n = NULL; - cuikperf_region_end(); - - nl_buffer_t *buf = nl_buffer_new(); - - nl_buffer_format(buf, "%s\n", nl_buffer_get(ctx.globals)); - nl_buffer_format(buf, "tb2c_%s_ret_t %s(", name, name); - TB_Node** params = f->params; - size_t count = 0; - FOREACH_N(i, 3, 3 + f->param_count) { - if (params[i] != NULL && params[i]->dt.type != TB_MEMORY && params[i]->dt.type != TB_CONTROL && params[i]->dt.type != TB_TUPLE) { - if (count != 0) { - nl_buffer_format(buf, ", "); - } - nl_buffer_format(buf, "%s v%u", c_fmt_type_name(params[i]->dt), params[i]->gvn); - count += 1; - } - } - if (count == 0) { - nl_buffer_format(buf, "void"); - } - nl_buffer_format(buf, ") {\n"); - nl_buffer_format(buf, "%s", nl_buffer_get(ctx.pre)); - nl_buffer_format(buf, "%s", nl_buffer_get(ctx.buf)); - nl_buffer_format(buf, "}\n"); - - dyn_array_destroy(ctx.visited_blocks); - - char *ret = nl_buffer_get(buf); - - // tb_arena_destroy(ctx.arena); - - return ret; -} diff --git a/vendor/tb/src/opt/print_dumb.h b/vendor/tb/src/opt/print_dumb.h deleted file mode 100644 index 0b05a25c..00000000 --- a/vendor/tb/src/opt/print_dumb.h +++ /dev/null @@ -1,105 +0,0 @@ - -static void dumb_print_node_internal(TB_Function* f, Lattice** types, TB_Node* n, uint64_t* visited) { - printf("%%%u: ", n->gvn); - if (types && types[n->gvn] != NULL && types[n->gvn] != &TOP_IN_THE_SKY) { - print_lattice(types[n->gvn], n->dt); - } else { - if (n->dt.type == TB_TUPLE) { - // print with multiple returns - TB_Node* projs[32] = { 0 }; - FOR_USERS(use, n) { - if (use->n->type == TB_PROJ) { - int index = TB_NODE_GET_EXTRA_T(use->n, TB_NodeProj)->index; - projs[index] = use->n; - } - } - - printf("{ "); - FOREACH_N(i, 0, 32) { - if (projs[i] == NULL) break; - if (i) printf(", "); - print_type(projs[i]->dt); - } - printf(" }"); - } else { - print_type(n->dt); - } - } - printf(" = %s ", tb_node_get_name(n)); - if (n->type == TB_STORE) { - print_type(n->inputs[3]->dt); - printf(" "); - } if (n->type == TB_SYMBOL) { - TB_Symbol* sym = TB_NODE_GET_EXTRA_T(n, TB_NodeSymbol)->sym; - if (sym->name[0]) { - printf("'%s' ", sym->name); - } else { - printf("%p ", sym); - } - } else if (n->type == TB_INTEGER_CONST) { - TB_NodeInt* num = TB_NODE_GET_EXTRA(n); - - if (num->value < 0xFFFF) { - int bits = n->dt.type == TB_PTR ? 64 : n->dt.data; - printf("%"PRId64" ", tb__sxt(num->value, bits, 64)); - } else { - printf("%#0"PRIx64" ", num->value); - } - } else if (n->type == TB_FLOAT32_CONST) { - TB_NodeFloat32* f = TB_NODE_GET_EXTRA(n); - printf("%f ", f->value); - } else if (n->type == TB_FLOAT64_CONST) { - TB_NodeFloat64* f = TB_NODE_GET_EXTRA(n); - printf("%f ", f->value); - } - printf("( "); - FOREACH_N(i, 0, n->input_count) { - TB_Node* in = n->inputs[i]; - if (in) { - if (in->type == TB_PROJ) { - int index = TB_NODE_GET_EXTRA_T(in, TB_NodeProj)->index; - in = in->inputs[0]; - - printf("%%%u.%d ", in->gvn, index); - } else { - printf("%%%u ", in->gvn); - } - } else { - printf("___ "); - } - } - printf(")\n"); -} - -static void dumb_print_node(TB_Function* f, Lattice** types, TB_Node* n, uint64_t* visited) { - if (visited[n->gvn / 64] & (1ull << (n->gvn % 64))) { - return; - } - visited[n->gvn / 64] |= (1ull << (n->gvn % 64)); - - FOREACH_REVERSE_N(i, 0, n->input_count) if (n->inputs[i]) { - TB_Node* in = n->inputs[i]; - if (in->type == TB_PROJ) { in = in->inputs[0]; } - - dumb_print_node(f, types, in, visited); - } - - dumb_print_node_internal(f, types, n, visited); -} - -void tb_dumb_print(TB_Function* f, TB_Passes* p) { - printf("=== DUMP %s ===\n", f->super.name); - - uint64_t* visited = tb_platform_heap_alloc(((f->node_count + 63) / 64) * sizeof(uint64_t)); - memset(visited, 0, ((f->node_count + 63) / 64) * sizeof(uint64_t)); - - TB_Node* root = f->root_node; - Lattice** types = p ? p->types : NULL; - - dumb_print_node_internal(f, types, root, visited); - visited[root->gvn / 64] |= (1ull << (root->gvn % 64)); - - FOREACH_N(i, 0, root->input_count) { - dumb_print_node(f, types, root->inputs[i], visited); - } -} diff --git a/vendor/tb/src/opt/properties.h b/vendor/tb/src/opt/properties.h deleted file mode 100644 index c3461099..00000000 --- a/vendor/tb/src/opt/properties.h +++ /dev/null @@ -1,89 +0,0 @@ -// Keeping track of all kinds of TB node properties -static bool is_associative(TB_NodeTypeEnum type) { - switch (type) { - case TB_ADD: case TB_MUL: - case TB_AND: case TB_XOR: case TB_OR: - return true; - - default: - return false; - } -} - -static bool is_commutative(TB_NodeTypeEnum type) { - switch (type) { - case TB_ADD: case TB_MUL: - case TB_AND: case TB_XOR: case TB_OR: - case TB_CMP_NE: case TB_CMP_EQ: - case TB_FADD: case TB_FMUL: - return true; - - default: - return false; - } -} - -static bool is_mem_access(TB_Node* n) { - switch (n->type) { - case TB_LOAD: - case TB_STORE: - case TB_MEMCPY: - case TB_MEMSET: - return true; - - default: - return false; - } -} - -static bool is_effect_tuple(TB_Node* n) { - switch (n->type) { - case TB_CALL: - case TB_SYSCALL: - case TB_TAILCALL: - case TB_READ: - case TB_WRITE: - case TB_INLINE_ASM: - return true; - - default: - return false; - } -} - -static bool cfg_is_region(TB_Node* n) { - return n->type >= TB_REGION && n->type <= TB_AFFINE_LOOP; -} - -static bool cfg_is_natural_loop(TB_Node* n) { - return n->type >= TB_NATURAL_LOOP && n->type <= TB_AFFINE_LOOP; -} - -static bool cfg_is_terminator(TB_Node* n) { - switch (n->type) { - case TB_BRANCH: - case TB_UNREACHABLE: - case TB_TRAP: - case TB_RETURN: - case TB_TAILCALL: - case TB_ROOT: - return true; - - default: - return false; - } -} - -static bool cfg_is_endpoint(TB_Node* n) { - switch (n->type) { - case TB_UNREACHABLE: - case TB_TRAP: - case TB_RETURN: - case TB_TAILCALL: - case TB_ROOT: - return true; - - default: - return false; - } -} diff --git a/vendor/tb/src/opt/scheduler.h b/vendor/tb/src/opt/scheduler.h deleted file mode 100644 index 5354d333..00000000 --- a/vendor/tb/src/opt/scheduler.h +++ /dev/null @@ -1,193 +0,0 @@ - -typedef struct SchedNode SchedNode; -struct SchedNode { - SchedNode* parent; - - TB_Node* n; - int index; - - int anti_i; - int anti_count; - User* antis[]; -}; - -typedef struct { - TB_Node* phi; - TB_Node* n; -} SchedPhi; - -static SchedNode* sched_make_node(TB_Arena* arena, SchedNode* parent, TB_Node* n) { - int anti_count = 0; - if (n->type == TB_MERGEMEM) { - anti_count = n->input_count - 2; - } else if (is_mem_out_op(n) && n->type != TB_PHI && n->type != TB_PROJ) { - anti_count = 1; - } - - SchedNode* s = tb_arena_alloc(arena, sizeof(SchedNode) + anti_count*sizeof(User*)); - *s = (SchedNode){ .parent = parent, .n = n, .index = 0, .anti_count = anti_count }; - - if (n->type == TB_MERGEMEM) { - FOREACH_N(i, 2, n->input_count) { - s->antis[i - 2] = n->inputs[i]->users; - } - } else if (anti_count == 1) { - s->antis[0] = n->inputs[1]->users; - } - - return s; -} - -static bool sched_in_bb(TB_Passes* passes, Worklist* ws, TB_BasicBlock* bb, TB_Node* n) { - return passes->scheduled[n->gvn] == bb && !worklist_test_n_set(ws, n); -} - -typedef struct { - int cap, count; - SchedPhi* arr; -} Phis; - -static void fill_phis(TB_Arena* arena, Phis* phis, TB_Node* succ, int phi_i) { - for (User* u = succ->users; u; u = u->next) { - if (u->n->type != TB_PHI) continue; - - // ensure cap (not very effective since it moves each time, that's ok it's rare) - if (phis->count == phis->cap) { - phis->cap *= 2; - - SchedPhi* new_phis = tb_arena_alloc(arena, phis->cap * sizeof(SchedPhi)); - memcpy(new_phis, phis->arr, phis->count * sizeof(SchedPhi)); - phis->arr = new_phis; - } - - phis->arr[phis->count++] = (SchedPhi){ .phi = u->n, .n = u->n->inputs[1 + phi_i] }; - } -} - -// basically just topological sort, no fancy shit -void greedy_scheduler(TB_Passes* passes, TB_CFG* cfg, Worklist* ws, DynArray(PhiVal)* phi_vals, TB_BasicBlock* bb, TB_Node* end) { - TB_Arena* arena = tmp_arena; - TB_ArenaSavepoint sp = tb_arena_save(arena); - - // find phis - int phi_curr = 0; - Phis phis = { .cap = 256, .arr = tb_arena_alloc(arena, 256 * sizeof(SchedPhi)) }; - - if (end->type == TB_BRANCH) { - for (User* u = end->users; u; u = u->next) { - if (u->n->type != TB_PROJ) continue; - - // we might have some memory phis over here if the projections aren't bbs - ptrdiff_t search = nl_map_get(cfg->node_to_block, u->n); - if (search >= 0) continue; - - User* succ = cfg_next_user(end); - if (cfg_is_region(succ->n)) { - fill_phis(arena, &phis, succ->n, succ->slot); - } - } - } else if (!cfg_is_endpoint(end)) { - User* succ = cfg_next_user(end); - if (cfg_is_region(succ->n)) { - fill_phis(arena, &phis, succ->n, succ->slot); - } - } - - SchedNode* top = sched_make_node(arena, NULL, end); - worklist_test_n_set(ws, end); - - // reserve projections for the top - TB_Node* start = bb->id == 0 ? passes->f->root_node : NULL; - if (start) { - FOR_USERS(use, start) { - if (use->n->type == TB_PROJ && !worklist_test_n_set(ws, use->n)) { - dyn_array_put(ws->items, use->n); - } - } - } - - size_t leftovers = 0; - size_t leftover_count = 1ull << bb->items.exp; - - while (top != NULL) retry: { - TB_Node* n = top->n; - - // resolve inputs first - if (n->type != TB_PHI && top->index < n->input_count) { - TB_Node* in = n->inputs[top->index++]; - if (in != NULL && sched_in_bb(passes, ws, bb, in)) { - top = sched_make_node(arena, top, in); - } - continue; - } - - // resolve anti-deps - if (top->anti_i < top->anti_count) { - if (top->antis[top->anti_i] != NULL) { - User* next = top->antis[top->anti_i]->next; - TB_Node* anti = top->antis[top->anti_i]->n; - int slot = top->antis[top->anti_i]->slot; - - if (anti != n && slot == 1 && sched_in_bb(passes, ws, bb, anti)) { - top = sched_make_node(arena, top, anti); - } - - top->antis[top->anti_i] = next; - if (next == NULL) { top->anti_i++; } - continue; - } - } - - // resolve phi edges & leftovers when we're at the endpoint - if (end == n) { - // skip non-phis - if (phi_curr < phis.count) { - TB_Node* phi = phis.arr[phi_curr].phi; - TB_Node* val = phis.arr[phi_curr].n; - phi_curr += 1; - - // reserve PHI space - if (phi_vals && val->dt.type != TB_MEMORY) { - PhiVal p; - p.phi = phi; - p.n = val; - p.dst = -1; - p.src = -1; - dyn_array_put(*phi_vals, p); - } - - if (sched_in_bb(passes, ws, bb, val)) { - top = sched_make_node(arena, top, val); - } - continue; - } - - // resolve leftover nodes placed here by GCM - while (leftovers < leftover_count && (bb->items.data[leftovers] == NULL || bb->items.data[leftovers] == NL_HASHSET_TOMB)) { - leftovers++; - } - - if (leftovers < leftover_count) { - if (!worklist_test_n_set(ws, bb->items.data[leftovers])) { - top = sched_make_node(arena, top, bb->items.data[leftovers]); - } - leftovers += 1; - continue; - } - } - - dyn_array_put(ws->items, n); - top = top->parent; - - // push outputs (projections, if they apply) - if (n->dt.type == TB_TUPLE && n->type != TB_BRANCH && n->type != TB_ROOT) { - for (User* use = n->users; use; use = use->next) { - if (use->n->type == TB_PROJ && !worklist_test_n_set(ws, use->n)) { - dyn_array_put(ws->items, use->n); - } - } - } - } - - tb_arena_restore(arena, sp); -} diff --git a/vendor/tb/src/opt/sroa.h b/vendor/tb/src/opt/sroa.h deleted file mode 100644 index 6ac8f193..00000000 --- a/vendor/tb/src/opt/sroa.h +++ /dev/null @@ -1,123 +0,0 @@ - -enum { SROA_LIMIT = 1024 }; - -typedef struct { - TB_Node* old_n; - int64_t offset; - TB_CharUnits size; - TB_DataType dt; -} AggregateConfig; - -static ptrdiff_t find_config(size_t config_count, AggregateConfig* configs, int64_t offset) { - FOREACH_N(i, 0, config_count) { - if (configs[i].offset == offset) return i; - } - - tb_unreachable(); - return -1; -} - -// -1 is a bad match -// -2 is no match, so we can add a new config -static ptrdiff_t compatible_with_configs(size_t config_count, AggregateConfig* configs, int64_t offset, TB_CharUnits size, TB_DataType dt) { - int64_t max = offset + size; - - FOREACH_N(i, 0, config_count) { - int64_t max2 = configs[i].offset + configs[i].size; - - if (offset >= configs[i].offset && max <= max2) { - // they overlap... but is it a clean overlap? - if (offset == configs[i].offset && max == max2 && TB_DATA_TYPE_EQUALS(dt, configs[i].dt)) { - return i; - } - - return -1; - } - } - - return -2; -} - -// false means failure to SROA -static bool add_configs(TB_Passes* p, User* use, TB_Node* base_address, size_t base_offset, size_t* config_count, AggregateConfig* configs, int pointer_size) { - for (; use; use = use->next) { - TB_Node* n = use->n; - - if (n->type == TB_MEMBER_ACCESS && use->slot == 1) { - // same rules, different offset - int64_t offset = TB_NODE_GET_EXTRA_T(n, TB_NodeMember)->offset; - if (!add_configs(p, n->users, base_address, base_offset + offset, config_count, configs, pointer_size)) { - return false; - } - continue; - } - - // we can only SROA if we know we're not using the - // address for anything but direct memory ops or TB_MEMBERs. - if (use->slot != 2) { - return false; - } - - // find direct memory op - if (n->type != TB_LOAD && n->type != TB_STORE) { - return false; - } - - TB_DataType dt = n->type == TB_LOAD ? n->dt : n->inputs[3]->dt; - TB_Node* address = n->inputs[2]; - int size = (bits_in_data_type(pointer_size, dt) + 7) / 8; - - // see if it's a compatible configuration - int match = compatible_with_configs(*config_count, configs, base_offset, size, dt); - if (match == -1) { - return false; - } else if (match == -2) { - // add new config - if (*config_count == SROA_LIMIT) { - return false; - } - configs[(*config_count)++] = (AggregateConfig){ address, base_offset, size, dt }; - } else if (configs[match].old_n != address) { - log_warn("%s: v%u SROA config matches but reaches so via a different node, please idealize nodes before mem2reg", p->f->super.name, address->gvn); - return false; - } - } - - return true; -} - -static size_t sroa_rewrite(TB_Passes* restrict p, int pointer_size, TB_Node* start, TB_Node* n) { - TB_ArenaSavepoint sp = tb_arena_save(tmp_arena); - - size_t config_count = 0; - AggregateConfig* configs = tb_arena_alloc(tmp_arena, SROA_LIMIT * sizeof(AggregateConfig)); - if (!add_configs(p, n->users, n, 0, &config_count, configs, pointer_size)) { - return 1; - } - - // split allocation into pieces - if (config_count > 1) { - DO_IF(TB_OPTDEBUG_SROA)(printf("sroa v%u => SROA to %zu pieces", n->gvn, config_count)); - - uint32_t alignment = TB_NODE_GET_EXTRA_T(n, TB_NodeLocal)->align; - FOREACH_N(i, 0, config_count) { - TB_Node* new_n = tb_alloc_node(p->f, TB_LOCAL, TB_TYPE_PTR, 1, sizeof(TB_NodeLocal)); - set_input(p->f, new_n, start, 0); - TB_NODE_SET_EXTRA(new_n, TB_NodeLocal, .size = configs[i].size, .align = alignment); - - // replace old pointer with new fancy - subsume_node(p->f, configs[i].old_n, new_n); - - // mark all users, there may be some fun new opts now - tb_pass_mark(p, new_n); - tb_pass_mark_users(p, new_n); - } - - // we marked the changes else where which is cheating the peephole - // but still doing all the progress it needs to. - tb_pass_mark_users(p, n); - } - - tb_arena_restore(tmp_arena, sp); - return config_count > 1 ? 1 + config_count : 1; -} diff --git a/vendor/tb/src/reg_alloc.h b/vendor/tb/src/reg_alloc.h deleted file mode 100644 index bfdee4c2..00000000 --- a/vendor/tb/src/reg_alloc.h +++ /dev/null @@ -1,970 +0,0 @@ -// This is just linear scan mostly inspired by: -// https://ssw.jku.at/Research/Papers/Wimmer04Master/Wimmer04Master.pdf -#ifdef NDEBUG -#define REG_ALLOC_LOG if (0) -#else -#define REG_ALLOC_LOG if (reg_alloc_log) -#endif - -typedef struct { - int start, end; -} LiveRange; - -typedef struct { - int pos; - - // this is used to denote folded reloads - // - // # r2 is spilled - // add r0, r1, r2 - // - // when we codegen, we don't need to allocate - // a register for r2 here. - // - // add r0, r1, [sp - 24] - enum { - USE_REG, - USE_MEM_OR_REG, - } kind; -} UsePos; - -typedef struct { - int pos; // 0 means not ready -} SpillSlot; - -struct LiveInterval { - int reg_class; - - TB_Node* n; - TB_X86_DataType dt; - - // results of regalloc - int assigned; - - // register num, -1 if the interval isn't a physical reg - int reg, hint; - - // each live interval reserves a spill slot in case they'll need it, it's - // shared across all splits of the interval (since the interval can't self - // intersect the reg is unique) - bool is_spill; - int split_kid; - SpillSlot* spill; - - // help speed up some of the main allocation loop - int active_range; - - // base interval tells us where to get our spill slot, we don't wanna - // make two separate spill slots. - int expected_spill; - LiveInterval* base; - - // we're gonna have so much memory to clean up... - DynArray(UsePos) uses; - - int range_cap, range_count; - LiveRange* ranges; -}; - -typedef DynArray(RegIndex) IntervalList; - -typedef struct { - DynArray(LiveInterval) intervals; - DynArray(RegIndex) inactive; - IntervalList unhandled; - Inst* first; - - int stack_usage; - - // time when the physical registers will be free again - int* free_pos; - int* block_pos; - - uint64_t callee_saved[CG_REGISTER_CLASSES]; - - Set active_set[CG_REGISTER_CLASSES]; - RegIndex active[CG_REGISTER_CLASSES][16]; - - Inst* cache; -} LSRA; - -static LiveRange* last_range(LiveInterval* i) { - return &i->ranges[dyn_array_length(i->ranges) - 1]; -} - -//////////////////////////////// -// Generate intervals -//////////////////////////////// -static void add_use_pos(LiveInterval* interval, int t, int kind) { - UsePos u = { t, kind }; - dyn_array_put(interval->uses, u); -} - -static void add_range(LiveInterval* interval, int start, int end) { - assert(start <= end); - assert(interval->range_count > 0); - - if (interval->ranges[interval->range_count - 1].start <= end) { - LiveRange* top = &interval->ranges[interval->range_count - 1]; - - // coalesce - top->start = TB_MIN(top->start, start); - top->end = TB_MAX(top->end, end); - } else { - if (interval->range_cap == interval->range_count) { - interval->range_cap *= 2; - interval->ranges = tb_platform_heap_realloc(interval->ranges, interval->range_cap * sizeof(LiveRange)); - } - - interval->ranges[interval->range_count++] = (LiveRange){ start, end }; - } -} - -static void reverse_bb_walk(LSRA* restrict ra, MachineBB* bb, Inst* inst) { - Inst* next = inst->next; - if (next && next->type != INST_LABEL) { - reverse_bb_walk(ra, bb, next); - } - - // mark outputs, inputs and temps - // - // TODO(NeGate): on x86 we can have one memory operand per instruction. - // we shouldn't force only register uses or else we'll make spilling more - // prominent. - RegIndex* ops = inst->operands; - bool dst_use_reg = inst->type == IMUL || inst->type == INST_ZERO || (inst->flags & (INST_MEM | INST_GLOBAL)); - - FOREACH_N(i, 0, inst->out_count) { - assert(*ops >= 0); - LiveInterval* interval = &ra->intervals[*ops++]; - - if (interval->range_count == 1) { - add_range(interval, inst->time, inst->time); - } else { - interval->ranges[interval->range_count - 1].start = inst->time; - } - - add_use_pos(interval, inst->time, dst_use_reg ? USE_REG : USE_OUT); - } - - int t = inst->type == MOV || inst->type == FP_MOV ? inst->time - 1 : inst->time; - int use = USE_REG; - - // reg<->reg ops can use one memory op, we'll prioritize that on the inputs side - if (!dst_use_reg && inst->type == MOV && (inst->flags & (INST_MEM | INST_GLOBAL)) == 0) { - use = USE_MEM_OR_REG; - } - - FOREACH_N(i, 0, inst->in_count) { - assert(*ops >= 0); - LiveInterval* interval = &ra->intervals[*ops++]; - - add_range(interval, bb->start, t); - add_use_pos(interval, t, use); - } - - // calls use the temporaries for clobbers - bool is_call = (inst->type == CALL || inst->type == SYSCALL); - FOREACH_N(i, 0, inst->tmp_count) { - assert(*ops >= 0); - LiveInterval* interval = &ra->intervals[*ops++]; - - add_range(interval, inst->time, inst->time + 1); - if (!is_call) { - add_use_pos(interval, inst->time, USE_REG); - } - } - - // safepoints don't care about memory or reg, it just needs to be available - FOREACH_N(i, 0, inst->save_count) { - assert(*ops >= 0); - LiveInterval* interval = &ra->intervals[*ops++]; - - add_range(interval, bb->start, t); - add_use_pos(interval, t, USE_MEM_OR_REG); - } -} - -static int range_intersect(LiveRange* a, LiveRange* b) { - if (b->start <= a->end && a->start <= b->end) { - return a->start > b->start ? a->start : b->start; - } else { - return -1; - } -} - -static int interval_intersect(LiveInterval* a, LiveInterval* b) { - FOREACH_REVERSE_N(i, 1, a->active_range+1) { - FOREACH_REVERSE_N(j, 1, b->active_range+1) { - int t = range_intersect(&a->ranges[i], &b->ranges[j]); - if (t >= 0) { - return t; - } - } - } - - return -1; -} - -#define FOREACH_SET(it, set) \ -FOREACH_N(_i, 0, ((set).capacity + 63) / 64) FOREACH_BIT(it, _i*64, (set).data[_i]) - -static int next_use(LSRA* restrict ra, LiveInterval* interval, int time) { - for (;;) { - FOREACH_N(i, 0, dyn_array_length(interval->uses)) { - if (interval->uses[i].pos > time) { - return interval->uses[i].pos; - } - } - - if (interval->split_kid >= 0) { - interval = &ra->intervals[interval->split_kid]; - continue; - } - - return INT_MAX; - } -} - -static LiveInterval* get_active(LSRA* restrict ra, int rc, int reg) { - if (!set_get(&ra->active_set[rc], reg)) { - return NULL; - } - - return &ra->intervals[ra->active[rc][reg]]; -} - -static void insert_split_move(LSRA* restrict ra, int t, int old_reg, int new_reg) { - Inst *prev, *inst; - TB_X86_DataType dt = ra->intervals[old_reg].dt; - - // invalidate - if (ra->cache->time >= t) { - ra->cache = ra->first; - } - - prev = ra->cache, inst = prev->next; - CUIK_TIMED_BLOCK("walk") { - while (inst != NULL) { - if (inst->time > t) { - ra->cache = prev; - break; - } - - prev = inst, inst = inst->next; - } - } - - // folded spill - if (inst && inst->type == MOV && inst->flags == 0 && inst->operands[0] == old_reg) { - inst->operands[0] = new_reg; - return; - } - - Inst* new_inst = tb_arena_alloc(tmp_arena, sizeof(Inst) + (2 * sizeof(RegIndex))); - *new_inst = (Inst){ .type = MOV, .flags = INST_SPILL, .dt = dt, .out_count = 1, 1 }; - new_inst->operands[0] = new_reg; - new_inst->operands[1] = old_reg; - new_inst->time = prev->time + 1; - new_inst->next = prev->next; - prev->next = new_inst; -} - -static int interval_start(LiveInterval* interval) { return interval->ranges[interval->range_count - 1].start; } -static int interval_end(LiveInterval* interval) { return interval->ranges[1].end; } - -static LiveInterval* split_interval_at(LSRA* restrict ra, LiveInterval* interval, int pos) { - // skip past previous intervals - while (interval->split_kid >= 0 && pos > interval_end(interval)) { - interval = &ra->intervals[interval->split_kid]; - } - - // assert(interval->reg >= 0 || pos <= interval_end(interval)); - return interval; -} - -static void allocate_spill_slot(LSRA* restrict ra, LiveInterval* interval) { - SpillSlot* spill = interval->spill; - assert(spill && "how do we not have a spill slot... we a fixed interval?"); - - if (spill->pos == 0) { - // allocate stack slot - int size = 8; - spill->pos = ra->stack_usage = align_up(ra->stack_usage + size, size); - } -} - -// any uses after `pos` after put into the new interval -static int split_intersecting(LSRA* restrict ra, int pos, LiveInterval* interval, bool is_spill) { - assert(interval->reg < 0); - cuikperf_region_start("split_intersecting", NULL); - - int ri = interval - ra->intervals; - LiveInterval it = *interval; - - assert(is_spill != interval->is_spill); - it.is_spill = is_spill; - - if (is_spill) { - allocate_spill_slot(ra, interval); - - int sp_offset = interval->spill->pos; - REG_ALLOC_LOG printf(" \x1b[33m# v%d: spill %s to [RBP - %d] at t=%d\x1b[0m\n", ri, reg_name(interval->reg_class, interval->assigned), sp_offset, pos); - } else { - int sp_offset = interval->spill->pos; - REG_ALLOC_LOG printf(" \x1b[33m# v%d: reload [RBP - %d] at t=%d\x1b[0m\n", ri, sp_offset, pos); - } - - // split lifetime - it.assigned = it.reg = -1; - it.uses = NULL; - it.split_kid = -1; - it.range_count = 0; - - assert(interval->split_kid < 0 && "cannot spill while spilled"); - int old_reg = interval - ra->intervals; - int new_reg = dyn_array_length(ra->intervals); - interval->split_kid = new_reg; - - dyn_array_put(ra->intervals, it); - interval = &ra->intervals[old_reg]; - - if (!is_spill) { - // since the split is starting at pos and pos is at the top of the - // unhandled list... we can push this to the top wit no problem - size_t i = 0, count = dyn_array_length(ra->unhandled); - for (; i < count; i++) { - if (pos > interval_start(&ra->intervals[ra->unhandled[i]])) break; - } - - // we know where to insert - dyn_array_put(ra->unhandled, 0); - memmove(&ra->unhandled[i + 1], &ra->unhandled[i], (count - i) * sizeof(RegIndex)); - ra->unhandled[i] = new_reg; - } - - // split uses - size_t use_count = dyn_array_length(interval->uses); - FOREACH_REVERSE_N(i, 0, use_count) { - size_t split_count = use_count - (i + 1); - if (interval->uses[i].pos > pos && split_count > 0) { - // split - DynArray(UsePos) uses = dyn_array_create(UsePos, split_count); - dyn_array_set_length(uses, split_count); - memcpy(uses, &interval->uses[i + 1], split_count * sizeof(UsePos)); - - dyn_array_set_length(interval->uses, i + 1); - it.uses = interval->uses; - interval->uses = uses; - break; - } - } - - // split ranges - size_t end = interval->range_count; - FOREACH_REVERSE_N(i, 1, end) { - LiveRange* range = &interval->ranges[i]; - if (range->end > pos) { - bool clean_split = pos < range->start; - - LiveRange old = interval->ranges[interval->active_range]; - - it.range_count = it.range_cap = i + 1; - it.active_range = it.range_count - 1; - it.ranges = interval->ranges; - - // move interval up, also insert INT_MAX and potentially - size_t start = it.range_count - !clean_split; - - interval->range_count = interval->range_cap = (end - start) + 1; - interval->ranges = tb_platform_heap_alloc(interval->range_count * sizeof(LiveRange)); - interval->active_range -= start - 1; - interval->ranges[0] = (LiveRange){ INT_MAX, INT_MAX }; - - FOREACH_N(j, start, end) { - assert(j - start + 1 < interval->range_count); - interval->ranges[j - start + 1] = it.ranges[j]; - } - - assert(interval->ranges[interval->active_range].start == old.start); - assert(interval->ranges[interval->active_range].end == old.end); - - if (range->start <= pos) { - interval->ranges[1].end = pos; - it.ranges[it.range_count - 1].start = pos; - } - break; - } - } - - // no ranges... weird but sure - if (it.range_count == 0) { - it.ranges = tb_platform_heap_alloc(1 * sizeof(LiveRange)); - it.range_count = 1; - it.range_cap = 1; - it.ranges[0] = (LiveRange){ INT_MAX, INT_MAX }; - } - - ra->intervals[new_reg] = it; - - // insert move (the control flow aware moves are inserted later) - insert_split_move(ra, pos, old_reg, new_reg); - - // reload before next use - if (is_spill) { - FOREACH_REVERSE_N(i, 0, dyn_array_length(it.uses)) { - if (it.uses[i].kind == USE_REG) { - // new split - split_intersecting(ra, it.uses[i].pos - 1, &ra->intervals[new_reg], false); - break; - } - } - } - - cuikperf_region_end(); - return new_reg; -} - -// returns -1 if no registers are available -static ptrdiff_t allocate_free_reg(LSRA* restrict ra, LiveInterval* interval) { - int rc = interval->reg_class; - - // callee saved will be biased to have nearer free positions to avoid incurring - // a spill on them early. - int half_free = 1 << 16; - FOREACH_N(i, 0, 16) { - ra->free_pos[i] = (ra->callee_saved[rc] & (1ull << i)) ? half_free : INT_MAX; - } - - // for each active reg, set the free pos to 0 - FOREACH_SET(i, ra->active_set[rc]) { - ra->free_pos[i] = 0; - } - - // for each inactive which intersects current - dyn_array_for(i, ra->inactive) { - LiveInterval* it = &ra->intervals[ra->inactive[i]]; - int fp = ra->free_pos[it->assigned]; - if (fp > 0) { - int p = interval_intersect(interval, it); - if (p >= 0 && p < fp) { - ra->free_pos[it->assigned] = p; - } - } - } - - if (rc == REG_CLASS_GPR) { - // reserved regs - ra->free_pos[RBP] = 0; - ra->free_pos[RSP] = 0; - } - - // try hint - int highest = -1; - int hint_reg = -1; - - if (interval->hint >= 0) { - LiveInterval* hint = &ra->intervals[interval->hint]; - assert(hint->reg_class == rc); - - // it's better in the long run to aggressively split - hint_reg = hint->assigned; - - if (interval_end(interval) < ra->free_pos[hint_reg]) { - highest = hint_reg; - } - /* else if (interval_start(interval) + 10 >= ra->free_pos[hint_reg]) { - REG_ALLOC_LOG printf(" # aggressive register splitting %s\n", reg_name(rc, hint_reg)); - highest = hint_reg; - }*/ - } - - // pick highest free pos - if (highest < 0) { - highest = 0; - FOREACH_N(i, 1, 16) if (ra->free_pos[i] > ra->free_pos[highest]) { - highest = i; - } - } - - int pos = ra->free_pos[highest]; - if (UNLIKELY(pos == 0)) { - // alloc failure - return -1; - } else { - if (UNLIKELY(ra->callee_saved[rc] & (1ull << highest))) { - ra->callee_saved[rc] &= ~(1ull << highest); - - REG_ALLOC_LOG printf(" # spill callee saved register %s\n", reg_name(rc, highest)); - - int size = rc ? 16 : 8; - int vreg = (rc ? FIRST_XMM : FIRST_GPR) + highest; - ra->stack_usage = align_up(ra->stack_usage + size, size); - - SpillSlot* s = TB_ARENA_ALLOC(tmp_arena, SpillSlot); - s->pos = ra->stack_usage; - - LiveInterval it = { - .is_spill = true, - .spill = s, - .dt = ra->intervals[vreg].dt, - .assigned = -1, - .reg = -1, - .split_kid = -1, - }; - - int old_reg = interval - ra->intervals; - int spill_slot = dyn_array_length(ra->intervals); - dyn_array_put(ra->intervals, it); - - // insert spill and reload - insert_split_move(ra, 0, vreg, spill_slot); - dyn_array_for(i, ra->epilogues) { - insert_split_move(ra, ra->epilogues[i] - 1, spill_slot, vreg); - } - - // adding to intervals might resized this - interval = &ra->intervals[old_reg]; - } - - if (interval_end(interval) <= pos) { - // we can steal it completely - REG_ALLOC_LOG printf(" # assign to %s", reg_name(rc, highest)); - - if (interval->hint >= 0) { - if (highest == hint_reg) { - REG_ALLOC_LOG printf(" (HINTED)\n"); - } else { - REG_ALLOC_LOG printf(" (FAILED HINT %s)\n", reg_name(rc, hint_reg)); - } - } else { - REG_ALLOC_LOG printf("\n"); - } - } else { - // TODO(NeGate): split current at optimal position before current - interval->assigned = highest; - split_intersecting(ra, pos - 1, interval, true); - } - - return highest; - } -} - -static ptrdiff_t allocate_blocked_reg(LSRA* restrict ra, LiveInterval* interval) { - int rc = interval->reg_class; - int* use_pos = ra->free_pos; - - FOREACH_N(i, 0, 16) ra->block_pos[i] = INT_MAX; - FOREACH_N(i, 0, 16) use_pos[i] = INT_MAX; - - // mark non-fixed intervals - int start = interval_start(interval); - FOREACH_SET(i, ra->active_set[rc]) { - LiveInterval* it = &ra->intervals[ra->active[rc][i]]; - if (it->reg_class == rc && it->reg < 0) { - use_pos[i] = next_use(ra, it, start); - } - } - - dyn_array_for(i, ra->inactive) { - LiveInterval* it = &ra->intervals[ra->inactive[i]]; - if (it->reg_class == rc && it->reg < 0) { - use_pos[i] = next_use(ra, it, start); - } - } - - // mark fixed intervals - FOREACH_SET(i, ra->active_set[rc]) { - LiveInterval* it = &ra->intervals[ra->active[rc][i]]; - if (it->reg_class == rc && it->reg >= 0) { - use_pos[i] = 0; - ra->block_pos[i] = 0; - } - } - - dyn_array_for(i, ra->inactive) { - LiveInterval* it = &ra->intervals[ra->inactive[i]]; - if (it->reg_class == rc && it->reg >= 0) { - int bp = ra->block_pos[it->assigned]; - if (bp > 0) { - int p = interval_intersect(interval, it); - if (p >= 0 && p < bp) { - ra->block_pos[it->assigned] = p; - } - } - } - } - - if (rc == REG_CLASS_GPR) { - // reserved regs - use_pos[RBP] = 0; - use_pos[RSP] = 0; - } - - // pick highest use pos - int highest = 0; - FOREACH_N(i, 1, 16) if (use_pos[i] > use_pos[highest]) { - highest = i; - } - - int pos = use_pos[highest]; - int first_use = INT_MAX; - if (dyn_array_length(interval->uses)) { - first_use = interval->uses[dyn_array_length(interval->uses) - 1].pos; - } - - bool spilled = false; - if (first_use > pos) { - // spill interval - allocate_spill_slot(ra, interval); - interval->is_spill = true; - - // split at optimal spot before first use that requires a register - FOREACH_REVERSE_N(i, 0, dyn_array_length(interval->uses)) { - if (interval->uses[i].pos >= pos && interval->uses[i].kind == USE_REG) { - split_intersecting(ra, interval->uses[i].pos - 1, interval, false); - break; - } - } - - spilled = true; - } else { - int start = interval_start(interval); - int split_pos = (start & ~1) - 1; - - // split active or inactive interval reg - LiveInterval* to_split = get_active(ra, rc, highest); - if (to_split != NULL) { - split_intersecting(ra, split_pos, to_split, true); - } - - // split any inactive interval for reg at the end of it's lifetime hole - dyn_array_for(i, ra->inactive) { - LiveInterval* it = &ra->intervals[ra->inactive[i]]; - LiveRange* r = &it->ranges[it->active_range]; - - if (it->reg_class == rc && it->assigned == highest && r->start <= pos+1 && pos <= r->end) { - split_intersecting(ra, split_pos, it, true); - } - } - } - - // split active reg if it intersects with fixed interval - LiveInterval* fix_interval = &ra->intervals[(rc ? FIRST_XMM : FIRST_GPR) + highest]; - if (dyn_array_length(fix_interval->ranges)) { - int p = interval_intersect(interval, fix_interval); - if (p >= 0) { - split_intersecting(ra, p, interval, true); - } - } - - return spilled ? -1 : highest; -} - -static void move_to_active(LSRA* restrict ra, LiveInterval* interval) { - int rc = interval->reg_class, reg = interval->assigned; - int ri = interval - ra->intervals; - - if (set_get(&ra->active_set[rc], reg)) { - tb_panic("v%d: interval v%d should never be forced out, we should've accomodated them in the first place", ri, ra->active[rc][reg]); - } - - assert(reg < 16); - set_put(&ra->active_set[rc], reg); - ra->active[rc][reg] = ri; -} - -// update active range to match where the position is currently -static bool update_interval(LSRA* restrict ra, LiveInterval* restrict interval, bool is_active, int time, int inactive_index) { - // get to the right range first - while (interval->ranges[interval->active_range].end <= time) { - assert(interval->active_range > 0); - interval->active_range -= 1; - } - - int ri = interval - ra->intervals; - int hole_end = interval->ranges[interval->active_range].start; - int active_end = interval->ranges[interval->active_range].end; - bool is_now_active = time >= hole_end; - - int rc = interval->reg_class; - int reg = interval->assigned; - - if (interval->active_range == 0) { // expired - if (is_active) { - REG_ALLOC_LOG printf(" # active %s has expired (v%d)\n", reg_name(rc, reg), ri); - set_remove(&ra->active_set[rc], reg); - } else { - REG_ALLOC_LOG printf(" # inactive %s has expired (v%d)\n", reg_name(rc, reg), ri); - dyn_array_remove(ra->inactive, inactive_index); - return true; - } - } else if (is_now_active != is_active) { // if we moved, change which list we're in - if (is_now_active) { // inactive -> active - REG_ALLOC_LOG printf(" # inactive %s is active again (until t=%d, v%d)\n", reg_name(rc, reg), active_end, ri); - - move_to_active(ra, interval); - dyn_array_remove(ra->inactive, inactive_index); - return true; - } else { // active -> inactive - REG_ALLOC_LOG printf(" # active %s is going quiet for now (until t=%d, v%d)\n", reg_name(rc, reg), hole_end, ri); - - set_remove(&ra->active_set[rc], reg); - dyn_array_put(ra->inactive, ri); - } - } - - return false; -} - -static void cuiksort_defs(LiveInterval* intervals, ptrdiff_t lo, ptrdiff_t hi, RegIndex* arr); -static int linear_scan(Ctx* restrict ctx, TB_Function* f, int stack_usage, DynArray(int) epilogues) { - LSRA ra = { .first = ctx->first, .cache = ctx->first, .intervals = ctx->intervals, .epilogues = epilogues, .stack_usage = stack_usage }; - - FOREACH_N(i, 0, CG_REGISTER_CLASSES) { - ra.active_set[i] = set_create_in_arena(tmp_arena, 16); - } - - // build intervals: - // we also track when uses happen to aid in splitting - MachineBBs mbbs = ctx->machine_bbs; - size_t interval_count = dyn_array_length(ra.intervals); - CUIK_TIMED_BLOCK("build intervals") { - FOREACH_REVERSE_N(i, 0, ctx->bb_count) { - TB_Node* bb = ctx->worklist.items[ctx->bb_order[i]]; - MachineBB* mbb = &nl_map_get_checked(mbbs, bb); - - int bb_start = mbb->start; - int bb_end = mbb->end + 2; - - // for anything that's live out, add the entire range - Set* live_out = &mbb->live_out; - FOREACH_N(j, 0, (interval_count + 63) / 64) { - uint64_t bits = live_out->data[j]; - if (bits == 0) continue; - - FOREACH_N(k, 0, 64) if (bits & (1ull << k)) { - add_range(&ra.intervals[j*64 + k], bb_start, bb_end); - } - } - - // for all instruction in BB (in reverse), add ranges - if (mbb->first) { - reverse_bb_walk(&ra, mbb, mbb->first); - } - } - } - - // we use every fixed interval at the very start to force them into - // the inactive set. - int fixed_count = 32; - FOREACH_N(i, 0, fixed_count) { - add_range(&ra.intervals[i], 0, 1); - } - - mark_callee_saved_constraints(ctx, ra.callee_saved); - - // generate unhandled interval list (sorted by starting point) - ra.unhandled = dyn_array_create(LiveInterval*, (interval_count * 4) / 3); - - SpillSlot* slots = tb_arena_alloc(tmp_arena, (interval_count - fixed_count) * sizeof(SpillSlot)); - FOREACH_N(i, 0, interval_count) { - if (i >= fixed_count) { - slots[i - fixed_count].pos = 0; - ra.intervals[i].spill = &slots[i - fixed_count]; - } - - ra.intervals[i].active_range = ra.intervals[i].range_count - 1; - dyn_array_put(ra.unhandled, i); - } - cuiksort_defs(ra.intervals, 0, interval_count - 1, ra.unhandled); - - // only need enough to store for the biggest register class - ra.free_pos = TB_ARENA_ARR_ALLOC(tmp_arena, 16, int); - ra.block_pos = TB_ARENA_ARR_ALLOC(tmp_arena, 16, int); - - // linear scan main loop - CUIK_TIMED_BLOCK("reg alloc") { - while (dyn_array_length(ra.unhandled)) { - RegIndex ri = dyn_array_pop(ra.unhandled); - LiveInterval* interval = &ra.intervals[ri]; - - int time = interval->ranges[interval->active_range].start; - assert(time != INT_MAX); - - int end = interval_end(interval); - if (interval->reg >= 0) { - REG_ALLOC_LOG printf(" # %-5s t=[%-4d - %4d)\n", reg_name(interval->reg_class, interval->reg), time, end); - } else if (interval->is_spill) { - REG_ALLOC_LOG { - printf(" # v%-4d t=[%-4d - %4d) SPILLED [RBP - %d]\n", ri, time, end, interval->spill->pos); - } - continue; - } else { - REG_ALLOC_LOG { - printf(" # v%-4d t=[%-4d - %4d) ", ri, time, end); - if (interval->n != NULL) { - print_node_sexpr(interval->n, 0); - } - printf("\n"); - } - } - - // expire intervals - FOREACH_N(rc, 0, CG_REGISTER_CLASSES) { - FOREACH_SET(reg, ra.active_set[rc]) { - RegIndex active_i = ra.active[rc][reg]; - update_interval(&ra, &ra.intervals[active_i], true, time, -1); - } - } - - int rc = interval->reg_class; - for (size_t i = 0; i < dyn_array_length(ra.inactive);) { - RegIndex inactive_i = ra.inactive[i]; - LiveInterval* it = &ra.intervals[inactive_i]; - - if (update_interval(&ra, it, false, time, i)) { - interval = &ra.intervals[ri]; // might've resized the intervals - continue; - } - - i++; - } - - ptrdiff_t reg = interval->reg; - if (reg < 0) { - // find register for virtual interval - if (reg < 0) { - reg = allocate_free_reg(&ra, interval); - interval = &ra.intervals[ri]; // might've resized the intervals - } - - // alloc failure - if (reg < 0) { - reg = allocate_blocked_reg(&ra, interval); - interval = &ra.intervals[ri]; // might've resized the intervals - - // tb_assert(reg >= 0, "regalloc failure"); - } - } - - // add to active set - if (reg >= 0) { - interval->assigned = reg; - move_to_active(&ra, interval); - } - - // display active set - REG_ALLOC_LOG { - printf(" \x1b[32m{ "); - FOREACH_N(rc, 0, CG_REGISTER_CLASSES) { - FOREACH_SET(reg, ra.active_set[rc]) { - int id = ra.active[rc][reg]; - - if (ra.intervals[id].reg >= 0) { - printf("%s ", reg_name(rc, ra.intervals[id].reg)); - } else { - printf("v%d:%s ", ra.active[rc][reg], reg_name(rc, reg)); - } - } - } - printf("}\x1b[0m\n"); - } - } - } - - // move resolver - CUIK_TIMED_BLOCK("move resolver") { - TB_Node** bbs = ctx->worklist.items; - int* bb_order = ctx->bb_order; - FOREACH_N(i, 0, ctx->bb_count) { - TB_Node* bb = bbs[bb_order[i]]; - MachineBB* mbb = &nl_map_get_checked(mbbs, bb); - TB_Node* end_node = mbb->end_node; - - for (User* u = end_node->users; u; u = u->next) { - if (cfg_is_control(u->n)) { - TB_Node* succ = end_node->type == TB_BRANCH ? cfg_next_bb_after_cproj(u->n) : u->n; - MachineBB* target = &nl_map_get_checked(mbbs, succ); - - // for all live-ins, we should check if we need to insert a move - FOREACH_SET(k, target->live_in) { - LiveInterval* interval = &ra.intervals[k]; - - // if the value changes across the edge, insert move - LiveInterval* start = split_interval_at(&ra, interval, mbb->end); - LiveInterval* end = split_interval_at(&ra, interval, target->start); - - if (start != end) { - if (start->spill > 0) { - assert(end->spill == start->spill && "TODO: both can't be spills yet"); - insert_split_move(&ra, target->start + 1, start - ra.intervals, end - ra.intervals); - } else { - insert_split_move(&ra, mbb->terminator - 1, start - ra.intervals, end - ra.intervals); - } - } - } - } - } - } - } - - // resolve all split interval references - CUIK_TIMED_BLOCK("split resolver") { - for (Inst* restrict inst = ra.first; inst; inst = inst->next) { - if (inst->flags & INST_SPILL) { - continue; - } - - int pos = inst->time; - FOREACH_N(i, 0, inst->out_count + inst->in_count + inst->tmp_count + inst->save_count) { - inst->operands[i] = split_interval_at(&ra, &ra.intervals[inst->operands[i]], pos) - ra.intervals; - } - } - } - - CUIK_TIMED_BLOCK("free intervals") { - dyn_array_for(i, ra.intervals) { - tb_platform_heap_free(ra.intervals[i].ranges); - dyn_array_destroy(ra.intervals[i].uses); - } - } - - ctx->intervals = ra.intervals; - return ra.stack_usage; -} - -//////////////////////////////// -// Sorting unhandled list -//////////////////////////////// -static size_t partition(LiveInterval* intervals, ptrdiff_t lo, ptrdiff_t hi, RegIndex* arr) { - int pivot = interval_start(&intervals[arr[(hi - lo) / 2 + lo]]); // middle - - ptrdiff_t i = lo - 1, j = hi + 1; - for (;;) { - // Move the left index to the right at least once and while the element at - // the left index is less than the pivot - do { i += 1; } while (interval_start(&intervals[arr[i]]) > pivot); - - // Move the right index to the left at least once and while the element at - // the right index is greater than the pivot - do { j -= 1; } while (interval_start(&intervals[arr[j]]) < pivot); - - // If the indices crossed, return - if (i >= j) return j; - - // Swap the elements at the left and right indices - SWAP(RegIndex, arr[i], arr[j]); - } -} - -static void cuiksort_defs(LiveInterval* intervals, ptrdiff_t lo, ptrdiff_t hi, RegIndex* arr) { - if (lo >= 0 && hi >= 0 && lo < hi) { - // get pivot - size_t p = partition(intervals, lo, hi, arr); - - // sort both sides - cuiksort_defs(intervals, lo, p, arr); - cuiksort_defs(intervals, p + 1, hi, arr); - } -} diff --git a/vendor/tb/src/set.h b/vendor/tb/src/set.h deleted file mode 100644 index 19e671f3..00000000 --- a/vendor/tb/src/set.h +++ /dev/null @@ -1,161 +0,0 @@ -#pragma once -#include "common.h" -#include "builtins.h" - -#define SET_INITIAL_CAP 32 - -typedef struct Set { - size_t capacity; - uint64_t* data; -} Set; - -static Set set_create_in_arena(TB_Arena* arena, size_t cap) { - void* ptr = tb_arena_alloc(arena, ((cap + 63) / 64) * sizeof(uint64_t)); - memset(ptr, 0, ((cap + 63) / 64) * sizeof(uint64_t)); - - return (Set){ - .capacity = cap, - .data = ptr, - }; -} - -static Set set_create(size_t cap) { - void* ptr = tb_platform_heap_alloc(((cap + 63) / 64) * sizeof(uint64_t)); - memset(ptr, 0, ((cap + 63) / 64) * sizeof(uint64_t)); - - return (Set){ - .capacity = cap, - .data = ptr, - }; -} - -static void set_free(Set* s) { - tb_platform_heap_free(s->data); -} - -static void set_clear(Set* s) { - memset(s->data, 0, ((s->capacity + 63) / 64) * sizeof(uint64_t)); -} - -// return true if changes -static bool set_union(Set* dst, Set* src) { - assert(dst->capacity >= src->capacity); - size_t n = (src->capacity + 63) / 64; - - uint64_t changes = 0; - FOREACH_N(i, 0, n) { - uint64_t old = dst->data[i]; - uint64_t new = old | src->data[i]; - - dst->data[i] = new; - changes |= (old ^ new); - } - return changes; -} - -static bool set_equals(Set* dst, Set* src) { - assert(dst->capacity == src->capacity); - size_t n = (dst->capacity + 63) / 64; - - FOREACH_N(i, 0, n) { - if (dst->data[i] != src->data[i]) return false; - } - - return true; -} - -static void set_copy(Set* dst, Set* src) { - assert(dst->capacity >= src->capacity); - memcpy(dst->data, src->data, ((src->capacity + 63) / 64) * sizeof(uint64_t)); -} - -static size_t set_popcount(Set* s) { - size_t n = (s->capacity + 63) / 64; - - size_t sum = 0; - FOREACH_N(i, 0, n) { - sum += tb_popcount64(s->data[i]); - } - - return sum; -} - -// Grabs a free slot -static ptrdiff_t set_pop_any(Set* s) { - size_t slots = (s->capacity + 63) / 64; - - for (size_t i = 0; i < slots; i++) { - if (s->data[i] == UINT64_MAX) continue; - - int index = s->data[i] != 0 ? tb_ffs64(~s->data[i]) - 1 : 0; - if (i*64 + index >= s->capacity) continue; - - s->data[i] |= (1ull << index); - return i*64 + index; - } - - return -1; -} - -static bool set_first_time(Set* s, size_t index) { - size_t slots = (s->capacity + 63) / 64; - lldiv_t d = lldiv(index, 64); - - if (d.quot >= slots) { - s->capacity = index * 2; - size_t new_slots = (s->capacity + 63) / 64; - - s->data = realloc(s->data, new_slots * sizeof(uint64_t)); - if (s->data == NULL) { - fprintf(stderr, "TB error: Set out of memory!"); - abort(); - } - - memset(s->data + slots, 0, (new_slots - slots) * sizeof(uint64_t)); - } - - if ((s->data[d.quot] & (1ull << d.rem)) == 0) { - s->data[d.quot] |= (1ull << d.rem); - return true; - } - - return false; -} - -static void set_put(Set* s, size_t index) { - size_t quot = index / 64; - size_t rem = index % 64; - - if (quot >= s->capacity) { - size_t old = s->capacity; - - s->capacity = quot * 2; - s->data = realloc(s->data, s->capacity * sizeof(uint64_t)); - if (s->data == NULL) { - fprintf(stderr, "TB error: Set out of memory!"); - abort(); - } - - memset(s->data + old, 0, (s->capacity - old) * sizeof(uint64_t)); - } - - s->data[quot] |= (1ull << rem); -} - -static void set_remove(Set* s, size_t index) { - size_t quot = index / 64; - size_t rem = index % 64; - if (quot < s->capacity) { - s->data[quot] &= ~(1ull << rem); - } -} - -static bool set_get(Set* s, size_t index) { - size_t quot = index / 64; - size_t rem = index % 64; - if (quot >= s->capacity) { - return false; - } - - return s->data[quot] & (1ull << rem); -} diff --git a/vendor/tb/src/symbols.c b/vendor/tb/src/symbols.c deleted file mode 100644 index d21e4d0f..00000000 --- a/vendor/tb/src/symbols.c +++ /dev/null @@ -1,75 +0,0 @@ -#include "tb_internal.h" -#include - -TB_SymbolIter tb_symbol_iter(TB_Module* mod) { - return (TB_SymbolIter){ .info = mod->first_info_in_module, .i = 0 }; -} - -TB_Symbol* tb_symbol_iter_next(TB_SymbolIter* iter) { - TB_ThreadInfo* info = iter->info; - while (info) { - size_t cap = 1ull << info->symbols.exp; - for (size_t i = iter->i; i < cap; i++) { - void* ptr = info->symbols.data[i]; - if (ptr == NULL) continue; - if (ptr == NL_HASHSET_TOMB) continue; - - iter->i = i + 1; - iter->info = info; - return (TB_Symbol*) ptr; - } - - info = atomic_load_explicit(&info->next_in_module, memory_order_relaxed); - } - - return NULL; -} - -void tb_module_kill_symbol(TB_Module* m, TB_Symbol* sym) { -} - -void tb_symbol_append(TB_Module* m, TB_Symbol* s) { - // append to thread's symbol table - TB_ThreadInfo* info = tb_thread_info(m); - if (info->symbols.data == NULL) { - info->symbols = nl_hashset_alloc(256); - } - - s->info = info; - nl_hashset_put(&info->symbols, s); - atomic_fetch_add(&m->symbol_count[s->tag], 1); -} - -TB_Symbol* tb_symbol_alloc(TB_Module* m, TB_SymbolTag tag, ptrdiff_t len, const char* name, size_t size) { - // TODO(NeGate): probably wanna have a custom heap for the symbol table - assert(tag != TB_SYMBOL_NONE); - TB_Symbol* s = tb_platform_heap_alloc(size); - memset(s, 0, size); - - s->tag = tag; - s->name = tb__arena_strdup(m, len, name); - s->module = m; - tb_symbol_append(m, s); - return s; -} - -TB_API bool tb_extern_resolve(TB_External* e, TB_Symbol* sym) { - TB_Symbol* expected = NULL; - if (atomic_compare_exchange_strong(&e->resolved, &expected, sym)) { - return true; - } else { - return false; - } -} - -TB_API TB_Function* tb_symbol_as_function(TB_Symbol* s) { - return s && s->tag == TB_SYMBOL_FUNCTION ? (TB_Function*) s : NULL; -} - -TB_API TB_External* tb_symbol_as_external(TB_Symbol* s) { - return s && s->tag == TB_SYMBOL_EXTERNAL ? (TB_External*) s : NULL; -} - -TB_API TB_Global* tb_symbol_as_global(TB_Symbol* s) { - return s && s->tag == TB_SYMBOL_GLOBAL ? (TB_Global*) s : NULL; -} diff --git a/vendor/tb/src/tb.c b/vendor/tb/src/tb.c deleted file mode 100644 index e5403be6..00000000 --- a/vendor/tb/src/tb.c +++ /dev/null @@ -1,690 +0,0 @@ -#include "tb_internal.h" -#include "host.h" -#include "opt/passes.h" - -static bool has_divrem(TB_Module* m) { - return m->target_arch != TB_ARCH_WASM32; -} - -static ICodeGen* tb__find_code_generator(TB_Module* m) { - // Place all the codegen interfaces down here - extern ICodeGen tb__x64_codegen; - extern ICodeGen tb__aarch64_codegen; - extern ICodeGen tb__mips32_codegen; - extern ICodeGen tb__mips64_codegen; - extern ICodeGen tb__wasm32_codegen; - - switch (m->target_arch) { - #ifdef TB_HAS_X64 - case TB_ARCH_X86_64: return &tb__x64_codegen; - #endif - - #ifdef TB_HAS_AARCH64 - case TB_ARCH_AARCH64: return &tb__aarch64_codegen; - #endif - - #ifdef TB_HAS_MIPS - case TB_ARCH_MIPS32: return &tb__mips32_codegen; - case TB_ARCH_MIPS64: return &tb__mips64_codegen; - #endif - - #ifdef TB_HAS_WASM - case TB_ARCH_WASM32: return &tb__wasm32_codegen; - #endif - - default: return NULL; - } -} - -TB_ThreadInfo* tb_thread_info(TB_Module* m) { - static thread_local TB_ThreadInfo* chain; - static thread_local mtx_t lock; - static thread_local bool init; - - if (!init) { - init = true; - mtx_init(&lock, mtx_plain); - } - - // there shouldn't really be contention here - mtx_lock(&lock); - - // almost always refers to one TB_ThreadInfo, but - // we can't assume the user has merely on TB_Module - // per thread. - TB_ThreadInfo* info = chain; - while (info != NULL) { - if (info->owner == m) { - goto done; - } - info = info->next; - } - - CUIK_TIMED_BLOCK("alloc thread info") { - info = tb_platform_heap_alloc(sizeof(TB_ThreadInfo)); - *info = (TB_ThreadInfo){ .owner = m, .chain = &chain, .lock = &lock }; - - // allocate memory for it - info->perm_arena = tb_arena_create(TB_ARENA_LARGE_CHUNK_SIZE); - info->tmp_arena = tb_arena_create(TB_ARENA_LARGE_CHUNK_SIZE); - - // thread local so it doesn't need to synchronize - info->next = chain; - if (chain != NULL) { - chain->prev = info; - } - chain = info; - - // link to the TB_Module* (we need to this to free later) - TB_ThreadInfo* old_top; - do { - old_top = atomic_load(&m->first_info_in_module); - info->next_in_module = old_top; - } while (!atomic_compare_exchange_strong(&m->first_info_in_module, &old_top, info)); - } - - done: - mtx_unlock(&lock); - return info; -} - -// we don't modify these strings -char* tb__arena_strdup(TB_Module* m, ptrdiff_t len, const char* src) { - if (len < 0) len = src ? strlen(src) : 0; - if (len == 0) return (char*) ""; - - char* newstr = tb_arena_alloc(get_permanent_arena(m), len + 1); - memcpy(newstr, src, len); - newstr[len] = 0; - return newstr; -} - -TB_Module* tb_module_create_for_host(bool is_jit) { - #if defined(TB_HOST_X86_64) - TB_Arch arch = TB_ARCH_X86_64; - #else - TB_Arch arch = TB_ARCH_UNKNOWN; - tb_panic("tb_module_create_for_host: cannot detect host platform"); - #endif - - #if defined(TB_HOST_WINDOWS) - TB_System sys = TB_SYSTEM_WINDOWS; - #elif defined(TB_HOST_OSX) - TB_System sys = TB_SYSTEM_MACOS; - #elif defined(TB_HOST_LINUX) - TB_System sys = TB_SYSTEM_LINUX; - #else - tb_panic("tb_module_create_for_host: cannot detect host platform"); - #endif - - return tb_module_create(arch, sys, is_jit); -} - -TB_ModuleSectionHandle tb_module_create_section(TB_Module* m, ptrdiff_t len, const char* name, TB_ModuleSectionFlags flags, TB_ComdatType comdat) { - size_t i = dyn_array_length(m->sections); - dyn_array_put_uninit(m->sections, 1); - - TB_ModuleSection* sec = &m->sections[i]; - *sec = (TB_ModuleSection){ - .name = tb__arena_strdup(m, len, name), - .flags = flags, - .comdat = { comdat }, - }; - return i; -} - -TB_Module* tb_module_create(TB_Arch arch, TB_System sys, bool is_jit) { - TB_Module* m = tb_platform_heap_alloc(sizeof(TB_Module)); - if (m == NULL) { - fprintf(stderr, "tb_module_create: Out of memory!\n"); - return NULL; - } - memset(m, 0, sizeof(TB_Module)); - - m->is_jit = is_jit; - - m->target_abi = (sys == TB_SYSTEM_WINDOWS) ? TB_ABI_WIN64 : TB_ABI_SYSTEMV; - m->target_arch = arch; - m->target_system = sys; - m->codegen = tb__find_code_generator(m); - - mtx_init(&m->lock, mtx_plain); - - // AOT uses sections to know where to organize things in an executable file, - // JIT does placement on the fly. - if (!is_jit) { - bool win = sys == TB_SYSTEM_WINDOWS; - tb_module_create_section(m, -1, ".text", TB_MODULE_SECTION_EXEC, TB_COMDAT_NONE); - tb_module_create_section(m, -1, ".data", TB_MODULE_SECTION_WRITE, TB_COMDAT_NONE); - tb_module_create_section(m, -1, win ? ".rdata" : ".rodata", 0, TB_COMDAT_NONE); - tb_module_create_section(m, -1, win ? ".tls$" : ".tls", TB_MODULE_SECTION_WRITE | TB_MODULE_SECTION_TLS, TB_COMDAT_NONE); - - if (win) { - m->chkstk_extern = (TB_Symbol*) tb_extern_create(m, -1, "__chkstk", TB_EXTERNAL_SO_LOCAL); - } - } else { - if (sys == TB_SYSTEM_WINDOWS) { - #ifdef _WIN32 - extern void __chkstk(void); - - // fill it with whatever MSVC/Clang gave us - m->chkstk_extern = (TB_Symbol*) tb_extern_create(m, -1, "__chkstk", TB_EXTERNAL_SO_LOCAL); - m->chkstk_extern->address = __chkstk; - #endif - } - } - - return m; -} - -TB_FunctionOutput* tb_pass_codegen(TB_Passes* p, TB_Arena* arena, const TB_FeatureSet* features, bool emit_asm) { - TB_Function* f = p->f; - TB_Module* m = f->super.module; - - TB_FunctionOutput* func_out = tb_arena_alloc(arena, sizeof(TB_FunctionOutput)); - *func_out = (TB_FunctionOutput){ .parent = f, .section = f->section, .linkage = f->linkage }; - m->codegen->compile_function(p, func_out, features, arena, emit_asm); - atomic_fetch_add(&m->compiled_function_count, 1); - - f->output = func_out; - return func_out; -} - -void tb_output_print_asm(TB_FunctionOutput* out, FILE* fp) { - if (fp == NULL) { - fp = stdout; - } - - TB_Assembly* a = tb_output_get_asm(out); - for (; a; a = a->next) { - fwrite(a->data, a->length, 1, fp); - } -} - -TB_Location* tb_output_get_locations(TB_FunctionOutput* out, size_t* out_count) { - *out_count = dyn_array_length(out->locations); - return &out->locations[0]; -} - -uint8_t* tb_output_get_code(TB_FunctionOutput* out, size_t* out_length) { - *out_length = out->code_size; - return out->code; -} - -TB_Assembly* tb_output_get_asm(TB_FunctionOutput* out) { - return out->asm_out; -} - -TB_Arena* tb_function_get_arena(TB_Function* f) { - return f->arena; -} - -void tb_module_destroy(TB_Module* m) { - // free thread info's arena - TB_ThreadInfo* info = atomic_load(&m->first_info_in_module); - while (info != NULL) { - TB_ThreadInfo* next = info->next_in_module; - - // free symbols - nl_hashset_free(info->symbols); - - tb_arena_destroy(info->tmp_arena); - tb_arena_destroy(info->perm_arena); - - // unlink, this needs to be synchronized in case another thread is - // accessing while we're freeing. - mtx_lock(info->lock); - if (info->prev == NULL) { - *info->chain = info->next; - } else { - info->prev->next = info->next; - } - mtx_unlock(info->lock); - - tb_platform_heap_free(info); - info = next; - } - - dyn_array_destroy(m->files); - tb_platform_heap_free(m); -} - -TB_SourceFile* tb_get_source_file(TB_Module* m, ptrdiff_t len, const char* path) { - mtx_lock(&m->lock); - - if (len <= 0) { - len = path ? strlen(path) : 0; - } - - NL_Slice key = { - .length = len, - .data = (const uint8_t*) path, - }; - - TB_SourceFile* file; - ptrdiff_t search = nl_map_get(m->files, key); - if (search < 0) { - file = tb_arena_alloc(get_permanent_arena(m), sizeof(TB_SourceFile) + key.length + 1); - file->id = -1; - file->len = key.length; - - memcpy(file->path, key.data, key.length); - key.data = file->path; - - nl_map_put(m->files, key, file); - } else { - file = m->files[search].v; - } - mtx_unlock(&m->lock); - return file; -} - -TB_FunctionPrototype* tb_prototype_create(TB_Module* m, TB_CallingConv cc, size_t param_count, const TB_PrototypeParam* params, size_t return_count, const TB_PrototypeParam* returns, bool has_varargs) { - size_t size = sizeof(TB_FunctionPrototype) + ((param_count + return_count) * sizeof(TB_PrototypeParam)); - TB_FunctionPrototype* p = tb_arena_alloc(get_permanent_arena(m), size); - - p->call_conv = cc; - p->param_count = param_count; - p->has_varargs = has_varargs; - if (param_count > 0) { - memcpy(p->params, params, param_count * sizeof(TB_PrototypeParam)); - } - if (return_count > 0 && !TB_IS_VOID_TYPE(returns[0].dt)) { - memcpy(p->params + param_count, returns, return_count * sizeof(TB_PrototypeParam)); - p->return_count = return_count; - } else { - p->return_count = 0; - } - return p; -} - -TB_Function* tb_function_create(TB_Module* m, ptrdiff_t len, const char* name, TB_Linkage linkage) { - TB_Function* f = (TB_Function*) tb_symbol_alloc(m, TB_SYMBOL_FUNCTION, len, name, sizeof(TB_Function)); - f->linkage = linkage; - return f; -} - -void tb_symbol_set_name(TB_Symbol* s, ptrdiff_t len, const char* name) { - s->name = tb__arena_strdup(s->module, len, name); -} - -const char* tb_symbol_get_name(TB_Symbol* s) { - return s->name; -} - -void tb_function_set_prototype(TB_Function* f, TB_ModuleSectionHandle section, TB_FunctionPrototype* p, TB_Arena* arena) { - assert(f->prototype == NULL); - size_t param_count = p->param_count; - - if (arena == NULL) { - f->arena = tb_arena_create(TB_ARENA_SMALL_CHUNK_SIZE); - } else { - f->arena = arena; - } - - f->gvn_nodes = nl_hashset_alloc(32); - f->locations = nl_table_alloc(16); - - f->section = section; - f->node_count = 0; - TB_Node* root = f->root_node = tb_alloc_node_dyn(f, TB_ROOT, TB_TYPE_TUPLE, 2, 4, 0); - - f->param_count = param_count; - f->params = tb_arena_alloc(f->arena, (3 + param_count) * sizeof(TB_Node*)); - - // fill in acceleration structure - f->params[0] = tb__make_proj(f, TB_TYPE_CONTROL, f->root_node, 0); - f->params[1] = tb__make_proj(f, TB_TYPE_MEMORY, f->root_node, 1); - f->params[2] = tb__make_proj(f, TB_TYPE_PTR, f->root_node, 2); - - // initial trace - f->trace.top_ctrl = f->trace.bot_ctrl = f->params[0]; - f->trace.mem = f->params[1]; - - // create parameter projections - TB_PrototypeParam* rets = TB_PROTOTYPE_RETURNS(p); - FOREACH_N(i, 0, param_count) { - TB_DataType dt = p->params[i].dt; - f->params[3+i] = tb__make_proj(f, dt, f->root_node, 3+i); - } - - // create callgraph node - TB_Node* callgraph = tb_alloc_node_dyn(f, TB_CALLGRAPH, TB_TYPE_VOID, 1, 8, sizeof(TB_NodeRegion)); - set_input(f, callgraph, root, 0); - set_input(f, root, callgraph, 0); - - // create return node - TB_Node* ret = f->ret_node = tb_alloc_node(f, TB_RETURN, TB_TYPE_CONTROL, 3 + p->return_count, 0); - set_input(f, root, ret, 1); - - // fill return crap - { - TB_Node* region = tb_alloc_node_dyn(f, TB_REGION, TB_TYPE_CONTROL, 0, 4, sizeof(TB_NodeRegion)); - TB_Node* mem_phi = tb_alloc_node_dyn(f, TB_PHI, TB_TYPE_MEMORY, 1, 5, 0); - set_input(f, mem_phi, region, 0); - - set_input(f, ret, region, 0); - set_input(f, ret, mem_phi, 1); - set_input(f, ret, f->params[2], 2); - - TB_PrototypeParam* returns = TB_PROTOTYPE_RETURNS(p); - FOREACH_N(i, 0, p->return_count) { - TB_Node* phi = tb_alloc_node_dyn(f, TB_PHI, returns[i].dt, 1, 5, 0); - set_input(f, phi, region, 0); - set_input(f, ret, phi, i + 3); - } - - TB_NODE_SET_EXTRA(region, TB_NodeRegion, .mem_in = mem_phi, .tag = "ret"); - } - - f->prototype = p; -} - -TB_FunctionPrototype* tb_function_get_prototype(TB_Function* f) { - return f->prototype; -} - -void* tb_global_add_region(TB_Module* m, TB_Global* g, size_t offset, size_t size) { - assert(offset == (uint32_t)offset); - assert(size == (uint32_t)size); - assert(g->obj_count + 1 <= g->obj_capacity); - - void* ptr = tb_platform_heap_alloc(size); - g->objects[g->obj_count++] = (TB_InitObj) { - .type = TB_INIT_OBJ_REGION, .offset = offset, .region = { .size = size, .ptr = ptr } - }; - - return ptr; -} - -void tb_global_add_symbol_reloc(TB_Module* m, TB_Global* g, size_t offset, TB_Symbol* symbol) { - assert(offset == (uint32_t) offset); - assert(g->obj_count + 1 <= g->obj_capacity); - assert(symbol != NULL); - - g->objects[g->obj_count++] = (TB_InitObj) { .type = TB_INIT_OBJ_RELOC, .offset = offset, .reloc = symbol }; -} - -TB_Global* tb_global_create(TB_Module* m, ptrdiff_t len, const char* name, TB_DebugType* dbg_type, TB_Linkage linkage) { - TB_Global* g = tb_arena_alloc(get_permanent_arena(m), sizeof(TB_Global)); - *g = (TB_Global){ - .super = { - .tag = TB_SYMBOL_GLOBAL, - .name = tb__arena_strdup(m, len, name), - .module = m, - }, - .dbg_type = dbg_type, - .linkage = linkage - }; - tb_symbol_append(m, (TB_Symbol*) g); - - return g; -} - -void tb_global_set_storage(TB_Module* m, TB_ModuleSectionHandle section, TB_Global* global, size_t size, size_t align, size_t max_objects) { - assert(size > 0 && align > 0 && tb_is_power_of_two(align)); - global->parent = section; - global->pos = 0; - global->size = size; - global->align = align; - global->obj_count = 0; - global->obj_capacity = max_objects; - global->objects = TB_ARENA_ARR_ALLOC(get_permanent_arena(m), max_objects, TB_InitObj); -} - -TB_Global* tb__small_data_intern(TB_Module* m, size_t len, const void* data) { - assert(len <= 16); - - // copy into SmallConst - SmallConst c = { .len = len }; - memcpy(c.data, data, c.len); - - mtx_lock(&m->lock); - ptrdiff_t search = nl_map_get(m->global_interns, c); - - TB_Global* g; - if (search >= 0) { - g = m->global_interns[search].v; - } else { - g = tb_global_create(m, 0, NULL, NULL, TB_LINKAGE_PRIVATE); - g->super.ordinal = *((uint64_t*) &c.data); - tb_global_set_storage(m, tb_module_get_rdata(m), g, len, len, 1); - - char* buffer = tb_global_add_region(m, g, 0, len); - memcpy(buffer, data, len); - nl_map_put(m->global_interns, c, g); - } - mtx_unlock(&m->lock); - return g; -} - -TB_Safepoint* tb_safepoint_get(TB_Function* f, uint32_t relative_ip) { - /*size_t left = 0; - size_t right = f->safepoint_count; - - uint32_t ip = relative_ip; - const TB_SafepointKey* keys = f->output->safepoints; - while (left < right) { - size_t middle = (left + right) / 2; - - if (keys[middle].ip == ip) return keys[left].sp; - if (keys[middle].ip < ip) left = middle + 1; - else right = middle; - }*/ - - return NULL; -} - -TB_ModuleSectionHandle tb_module_get_text(TB_Module* m) { return 0; } -TB_ModuleSectionHandle tb_module_get_data(TB_Module* m) { return 1; } -TB_ModuleSectionHandle tb_module_get_rdata(TB_Module* m) { return 2; } -TB_ModuleSectionHandle tb_module_get_tls(TB_Module* m) { return 3; } - -void tb_module_set_tls_index(TB_Module* m, ptrdiff_t len, const char* name) { - if (atomic_flag_test_and_set(&m->is_tls_defined)) { - m->tls_index_extern = (TB_Symbol*) tb_extern_create(m, len, name, TB_EXTERNAL_SO_LOCAL); - } -} - -void tb_symbol_bind_ptr(TB_Symbol* s, void* ptr) { - s->address = ptr; -} - -TB_ExternalType tb_extern_get_type(TB_External* e) { - return e->type; -} - -TB_External* tb_extern_create(TB_Module* m, ptrdiff_t len, const char* name, TB_ExternalType type) { - assert(name != NULL); - - TB_External* e = tb_arena_alloc(get_permanent_arena(m), sizeof(TB_External)); - *e = (TB_External){ - .super = { - .tag = TB_SYMBOL_EXTERNAL, - .name = tb__arena_strdup(m, len, name), - .module = m, - }, - .type = type, - }; - tb_symbol_append(m, (TB_Symbol*) e); - return e; -} - -// -// TLS - Thread local storage -// -// Certain backend elements require memory but we would prefer to avoid -// making any heap allocations when possible to there's a preallocated -// block per thread that can run TB. -// -void tb_free_thread_resources(void) { -} - -void tb_emit_symbol_patch(TB_FunctionOutput* func_out, TB_Symbol* target, size_t pos) { - TB_Module* m = func_out->parent->super.module; - TB_SymbolPatch* p = TB_ARENA_ALLOC(get_permanent_arena(m), TB_SymbolPatch); - - // function local, no need to synchronize - *p = (TB_SymbolPatch){ .target = target, .pos = pos }; - if (func_out->first_patch == NULL) { - func_out->first_patch = func_out->last_patch = p; - } else { - func_out->last_patch->next = p; - func_out->last_patch = p; - } - func_out->patch_count += 1; -} - -// EMITTER CODE: -// Simple linear allocation for the -// backend's to output code with. -void* tb_out_reserve(TB_Emitter* o, size_t count) { - if (o->count + count >= o->capacity) { - if (o->capacity == 0) { - o->capacity = 64; - } else { - o->capacity += count; - o->capacity *= 2; - } - - o->data = tb_platform_heap_realloc(o->data, o->capacity); - if (o->data == NULL) tb_todo(); - } - - return &o->data[o->count]; -} - -void tb_out_commit(TB_Emitter* o, size_t count) { - assert(o->count + count < o->capacity); - o->count += count; -} - -size_t tb_out_get_pos(TB_Emitter* o, void* p) { - return (uint8_t*)p - o->data; -} - -void* tb_out_grab(TB_Emitter* o, size_t count) { - void* p = tb_out_reserve(o, count); - o->count += count; - - return p; -} - -void* tb_out_get(TB_Emitter* o, size_t pos) { - assert(pos < o->count); - return &o->data[pos]; -} - -size_t tb_out_grab_i(TB_Emitter* o, size_t count) { - tb_out_reserve(o, count); - - size_t old = o->count; - o->count += count; - return old; -} - -void tb_out1b_UNSAFE(TB_Emitter* o, uint8_t i) { - assert(o->count + 1 < o->capacity); - - o->data[o->count] = i; - o->count += 1; -} - -void tb_out4b_UNSAFE(TB_Emitter* o, uint32_t i) { - tb_out_reserve(o, 4); - - *((uint32_t*)&o->data[o->count]) = i; - o->count += 4; -} - -void tb_out1b(TB_Emitter* o, uint8_t i) { - tb_out_reserve(o, 1); - - o->data[o->count] = i; - o->count += 1; -} - -void tb_out2b(TB_Emitter* o, uint16_t i) { - tb_out_reserve(o, 2); - - *((uint16_t*)&o->data[o->count]) = i; - o->count += 2; -} - -void tb_out4b(TB_Emitter* o, uint32_t i) { - tb_out_reserve(o, 4); - - *((uint32_t*)&o->data[o->count]) = i; - o->count += 4; -} - -void tb_patch1b(TB_Emitter* o, uint32_t pos, uint8_t i) { - *((uint8_t*)&o->data[pos]) = i; -} - -void tb_patch2b(TB_Emitter* o, uint32_t pos, uint16_t i) { - *((uint16_t*)&o->data[pos]) = i; -} - -void tb_patch4b(TB_Emitter* o, uint32_t pos, uint32_t i) { - assert(pos + 4 <= o->count); - *((uint32_t*)&o->data[pos]) = i; -} - -void tb_patch8b(TB_Emitter* o, uint32_t pos, uint64_t i) { - *((uint64_t*)&o->data[pos]) = i; -} - -uint8_t tb_get1b(TB_Emitter* o, uint32_t pos) { - return *((uint8_t*)&o->data[pos]); -} - -uint16_t tb_get2b(TB_Emitter* o, uint32_t pos) { - return *((uint16_t*)&o->data[pos]); -} - -uint32_t tb_get4b(TB_Emitter* o, uint32_t pos) { - return *((uint32_t*)&o->data[pos]); -} - -void tb_out8b(TB_Emitter* o, uint64_t i) { - tb_out_reserve(o, 8); - - *((uint64_t*)&o->data[o->count]) = i; - o->count += 8; -} - -void tb_out_zero(TB_Emitter* o, size_t len) { - tb_out_reserve(o, len); - memset(&o->data[o->count], 0, len); - o->count += len; -} - -size_t tb_outstr_nul_UNSAFE(TB_Emitter* o, const char* str) { - size_t start = o->count; - - for (; *str; str++) { - o->data[o->count++] = *str; - } - - o->data[o->count++] = 0; - return start; -} - -void tb_outstr_UNSAFE(TB_Emitter* o, const char* str) { - while (*str) o->data[o->count++] = *str++; -} - -size_t tb_outs(TB_Emitter* o, size_t len, const void* str) { - tb_out_reserve(o, len); - size_t start = o->count; - - memcpy(&o->data[o->count], str, len); - o->count += len; - return start; -} - -void tb_outs_UNSAFE(TB_Emitter* o, size_t len, const void* str) { - memcpy(&o->data[o->count], str, len); - o->count += len; -} diff --git a/vendor/tb/src/tb_builder.c b/vendor/tb/src/tb_builder.c deleted file mode 100644 index 0299aaf8..00000000 --- a/vendor/tb/src/tb_builder.c +++ /dev/null @@ -1,1057 +0,0 @@ -// IR BUILDER -// -// Handles generating the TB_Function IR via C functions. -// Note that these functions can perform certain simple -// optimizations while the generation happens to improve -// the machine code output or later analysis stages. -#include "tb_internal.h" - -static void add_input_late(TB_Function* f, TB_Node* n, TB_Node* in); - -TB_API void tb_inst_set_trace(TB_Function* f, TB_Trace trace) { f->trace = trace; } -TB_API TB_Trace tb_inst_get_trace(TB_Function* f) { return f->trace; } -TB_Node* tb_inst_get_control(TB_Function* f) { return f->trace.bot_ctrl; } -TB_API TB_Node* tb_inst_root_node(TB_Function* f) { return f->root_node; } - -static TB_Node* transfer_ctrl(TB_Function* f, TB_Node* n) { - TB_Node* prev = f->trace.bot_ctrl; - f->trace.bot_ctrl = n; - if (f->line_loc) { - nl_table_put(&f->locations, n, f->line_loc); - } - return prev; -} - -void tb_inst_set_control(TB_Function* f, TB_Node* control) { - if (control == NULL) { - f->trace.top_ctrl = NULL; - f->trace.bot_ctrl = NULL; - f->trace.mem = NULL; - } else { - assert(control->type == TB_REGION); - f->trace.top_ctrl = control; - f->trace.bot_ctrl = control; - f->trace.mem = TB_NODE_GET_EXTRA_T(control, TB_NodeRegion)->mem_in; - } -} - -TB_Node* tb_inst_region_mem_in(TB_Function* f, TB_Node* region) { - TB_NodeRegion* r = TB_NODE_GET_EXTRA(region); - return r->mem_in; -} - -TB_Trace tb_inst_trace_from_region(TB_Function* f, TB_Node* region) { - TB_NodeRegion* r = TB_NODE_GET_EXTRA(region); - return (TB_Trace){ region, region, r->mem_in }; -} - -static TB_Node* get_callgraph(TB_Function* f) { return f->root_node->inputs[0]; } -static TB_Node* peek_mem(TB_Function* f) { return f->trace.mem; } - -// adds memory effect to region -static TB_Node* append_mem(TB_Function* f, TB_Node* new_mem) { - TB_Node* old = f->trace.mem; - f->trace.mem = new_mem; - if (f->line_loc) { - nl_table_put(&f->locations, new_mem, f->line_loc); - } - return old; -} - -TB_Node* tb__make_proj(TB_Function* f, TB_DataType dt, TB_Node* src, int index) { - assert(src->dt.type == TB_TUPLE); - TB_Node* proj = tb_alloc_node(f, TB_PROJ, dt, 1, sizeof(TB_NodeProj)); - set_input(f, proj, src, 0); - TB_NODE_SET_EXTRA(proj, TB_NodeProj, .index = index); - return proj; -} - -bool tb_node_is_constant_int(TB_Function* f, TB_Node* n, uint64_t imm) { - return n->type == TB_INTEGER_CONST ? (TB_NODE_GET_EXTRA_T(n, TB_NodeInt)->value == imm) : false; -} - -bool tb_node_is_constant_non_zero(TB_Node* n) { - return n->type == TB_INTEGER_CONST ? (TB_NODE_GET_EXTRA_T(n, TB_NodeInt)->value != 0) : false; -} - -bool tb_node_is_constant_zero(TB_Node* n) { - return n->type == TB_INTEGER_CONST ? (TB_NODE_GET_EXTRA_T(n, TB_NodeInt)->value == 0) : false; -} - -void tb_function_attrib_variable(TB_Function* f, TB_Node* n, TB_Node* parent, ptrdiff_t len, const char* name, TB_DebugType* type) { - TB_NodeLocal* l = TB_NODE_GET_EXTRA(n); - l->name = tb__arena_strdup(f->super.module, len, name); - l->type = type; -} - -void tb_function_attrib_scope(TB_Function* f, TB_Node* n, TB_Node* parent) { -} - -void tb_inst_location(TB_Function* f, TB_SourceFile* file, int line, int column) { - TB_NodeLocation* loc = tb_arena_alloc(f->arena, sizeof(TB_NodeLocation)); - loc->file = file; - loc->line = line; - loc->column = column; - f->line_loc = loc; -} - -void tb_inst_set_exit_location(TB_Function* f, TB_SourceFile* file, int line, int column) { - TB_NodeLocation* loc = tb_arena_alloc(f->arena, sizeof(TB_NodeLocation)); - loc->file = file; - loc->line = line; - loc->column = column; - nl_table_put(&f->locations, f->root_node->inputs[0], loc); -} - -static void* alloc_from_node_arena(TB_Function* f, size_t necessary_size) { - // return malloc(necessary_size); - return tb_arena_alloc(f->arena, necessary_size); -} - -TB_Node* tb_alloc_node_dyn(TB_Function* f, int type, TB_DataType dt, int input_count, int input_cap, size_t extra) { - assert(input_count < UINT16_MAX && "too many inputs!"); - - TB_Node* n = alloc_from_node_arena(f, sizeof(TB_Node) + extra); - n->type = type; - n->input_cap = input_cap; - n->input_count = input_count; - n->dt = dt; - n->gvn = f->node_count++; - n->users = NULL; - - if (input_cap > 0) { - n->inputs = alloc_from_node_arena(f, input_cap * sizeof(TB_Node*)); - memset(n->inputs, 0, input_count * sizeof(TB_Node*)); - } else { - n->inputs = NULL; - } - - if (extra > 0) { - memset(n->extra, 0, extra); - } - - return n; -} - -TB_Node* tb_alloc_node(TB_Function* f, int type, TB_DataType dt, int input_count, size_t extra) { - return tb_alloc_node_dyn(f, type, dt, input_count, input_count, extra); -} - -static TB_Node* tb_bin_arith(TB_Function* f, int type, TB_ArithmeticBehavior arith_behavior, TB_Node* a, TB_Node* b) { - assert(TB_DATA_TYPE_EQUALS(a->dt, b->dt)); - - TB_Node* n = tb_alloc_node(f, type, a->dt, 3, sizeof(TB_NodeBinopInt)); - set_input(f, n, a, 1); - set_input(f, n, b, 2); - TB_NODE_SET_EXTRA(n, TB_NodeBinopInt, .ab = arith_behavior); - return tb_pass_gvn_node(f, n); -} - -static TB_Node* tb_bin_farith(TB_Function* f, int type, TB_Node* a, TB_Node* b) { - assert(TB_DATA_TYPE_EQUALS(a->dt, b->dt)); - - TB_Node* n = tb_alloc_node(f, type, a->dt, 3, 0); - set_input(f, n, a, 1); - set_input(f, n, b, 2); - return tb_pass_gvn_node(f, n); -} - -static TB_Node* tb_unary(TB_Function* f, int type, TB_DataType dt, TB_Node* src) { - TB_Node* n = tb_alloc_node(f, type, dt, 2, 0); - set_input(f, n, src, 1); - return tb_pass_gvn_node(f, n); -} - -TB_Node* tb_inst_trunc(TB_Function* f, TB_Node* src, TB_DataType dt) { - return tb_unary(f, TB_TRUNCATE, dt, src); -} - -TB_Node* tb_inst_int2ptr(TB_Function* f, TB_Node* src) { - return tb_unary(f, TB_BITCAST, TB_TYPE_PTR, src); -} - -TB_Node* tb_inst_ptr2int(TB_Function* f, TB_Node* src, TB_DataType dt) { - return tb_unary(f, TB_BITCAST, dt, src); -} - -TB_Node* tb_inst_int2float(TB_Function* f, TB_Node* src, TB_DataType dt, bool is_signed) { - assert(dt.type == TB_FLOAT); - assert(src->dt.type == TB_INT); - - if (src->type == TB_INTEGER_CONST) { - uint64_t y = TB_NODE_GET_EXTRA_T(src, TB_NodeInt)->value; - if (is_signed) { - y = tb__sxt(y, src->dt.data, 64); - } - - if (dt.data == TB_FLT_32) { - float x; - if (is_signed) x = (int64_t) y; - else x = (uint64_t) y; - - return tb_inst_float32(f, x); - } else if (dt.data == TB_FLT_64) { - double x; - if (is_signed) x = (int64_t) y; - else x = (uint64_t) y; - - return tb_inst_float64(f, x); - } - } - - return tb_unary(f, is_signed ? TB_INT2FLOAT : TB_UINT2FLOAT, dt, src); -} - -TB_Node* tb_inst_float2int(TB_Function* f, TB_Node* src, TB_DataType dt, bool is_signed) { - return tb_unary(f, is_signed ? TB_FLOAT2INT : TB_FLOAT2UINT, dt, src); -} - -TB_Node* tb_inst_fpxt(TB_Function* f, TB_Node* src, TB_DataType dt) { - return tb_unary(f, TB_FLOAT_EXT, dt, src); -} - -TB_Node* tb_inst_sxt(TB_Function* f, TB_Node* src, TB_DataType dt) { - if (src->type == TB_INTEGER_CONST) { - uint64_t y = TB_NODE_GET_EXTRA_T(src, TB_NodeInt)->value; - y = tb__sxt(y, src->dt.data, 64); - return tb_inst_uint(f, dt, y); - } - - return tb_unary(f, TB_SIGN_EXT, dt, src); -} - -TB_Node* tb_inst_zxt(TB_Function* f, TB_Node* src, TB_DataType dt) { - if (src->type == TB_INTEGER_CONST) { - uint64_t y = TB_NODE_GET_EXTRA_T(src, TB_NodeInt)->value; - return tb_inst_uint(f, dt, y); - } - - return tb_unary(f, TB_ZERO_EXT, dt, src); -} - -TB_Node* tb_inst_bitcast(TB_Function* f, TB_Node* src, TB_DataType dt) { - return tb_unary(f, TB_BITCAST, dt, src); -} - -TB_Node* tb_inst_param(TB_Function* f, int param_id) { - assert(param_id < f->param_count); - return f->params[3 + param_id]; -} - -void tb_get_data_type_size(TB_Module* mod, TB_DataType dt, size_t* size, size_t* align) { - const ICodeGen* restrict code_gen = tb__find_code_generator(mod); - code_gen->get_data_type_size(dt, size, align); -} - -void tb_inst_unreachable(TB_Function* f) { - TB_Node* n = tb_alloc_node(f, TB_UNREACHABLE, TB_TYPE_CONTROL, 1, 0); - set_input(f, n, transfer_ctrl(f, n), 0); - add_input_late(f, f->root_node, n); - f->trace.bot_ctrl = NULL; -} - -void tb_inst_debugbreak(TB_Function* f) { - TB_Node* n = tb_alloc_node(f, TB_DEBUGBREAK, TB_TYPE_CONTROL, 1, 0); - set_input(f, n, transfer_ctrl(f, n), 0); -} - -void tb_inst_trap(TB_Function* f) { - TB_Node* n = tb_alloc_node(f, TB_TRAP, TB_TYPE_CONTROL, 1, 0); - set_input(f, n, transfer_ctrl(f, n), 0); - add_input_late(f, f->root_node, n); - f->trace.bot_ctrl = NULL; -} - -TB_Node* tb_inst_local(TB_Function* f, TB_CharUnits size, TB_CharUnits alignment) { - assert(size > 0); - assert(alignment > 0 && tb_is_power_of_two(alignment)); - - // insert in the entry block - TB_Node* n = tb_alloc_node(f, TB_LOCAL, TB_TYPE_PTR, 1, sizeof(TB_NodeLocal)); - set_input(f, n, f->root_node, 0); - TB_NODE_SET_EXTRA(n, TB_NodeLocal, .size = size, .align = alignment); - return tb_pass_gvn_node(f, n); -} - -void tb_inst_safepoint_poll(TB_Function* f, void* tag, TB_Node* addr, int input_count, TB_Node** inputs) { - TB_Node* n = tb_alloc_node(f, TB_SAFEPOINT_POLL, TB_TYPE_CONTROL, 3 + input_count, sizeof(TB_NodeSafepoint)); - set_input(f, n, transfer_ctrl(f, n), 0); - set_input(f, n, peek_mem(f), 1); - set_input(f, n, addr, 2); - FOREACH_N(i, 0, input_count) { - set_input(f, n, inputs[i], i + 3); - } - TB_NODE_SET_EXTRA(n, TB_NodeSafepoint, tag); -} - -TB_Node* tb_inst_load(TB_Function* f, TB_DataType dt, TB_Node* addr, TB_CharUnits alignment, bool is_volatile) { - assert(addr); - - if (is_volatile) { - TB_Node* n = tb_alloc_node(f, TB_READ, TB_TYPE_TUPLE, 3, 0); - set_input(f, n, f->trace.bot_ctrl, 0); - set_input(f, n, peek_mem(f), 1); - set_input(f, n, addr, 2); - append_mem(f, tb__make_proj(f, TB_TYPE_MEMORY, n, 0)); - return tb__make_proj(f, dt, n, 1); - } else { - TB_Node* n = tb_alloc_node(f, TB_LOAD, dt, 3, sizeof(TB_NodeMemAccess)); - set_input(f, n, f->trace.bot_ctrl, 0); - set_input(f, n, peek_mem(f), 1); - set_input(f, n, addr, 2); - TB_NODE_SET_EXTRA(n, TB_NodeMemAccess, .align = alignment); - return tb_pass_gvn_node(f, n); - } -} - -void tb_inst_store(TB_Function* f, TB_DataType dt, TB_Node* addr, TB_Node* val, uint32_t alignment, bool is_volatile) { - assert(TB_DATA_TYPE_EQUALS(dt, val->dt)); - - TB_Node* n; - if (is_volatile) { - n = tb_alloc_node(f, TB_WRITE, TB_TYPE_MEMORY, 4, 0); - } else { - n = tb_alloc_node(f, TB_STORE, TB_TYPE_MEMORY, 4, sizeof(TB_NodeMemAccess)); - TB_NODE_SET_EXTRA(n, TB_NodeMemAccess, .align = alignment); - } - set_input(f, n, f->trace.bot_ctrl, 0); - set_input(f, n, append_mem(f, n), 1); - set_input(f, n, addr, 2); - set_input(f, n, val, 3); -} - -void tb_inst_memset(TB_Function* f, TB_Node* addr, TB_Node* val, TB_Node* size, TB_CharUnits align) { - assert(TB_IS_POINTER_TYPE(addr->dt)); - assert(TB_IS_INTEGER_TYPE(val->dt) && val->dt.data == 8); - - TB_Node* n = tb_alloc_node(f, TB_MEMSET, TB_TYPE_MEMORY, 5, sizeof(TB_NodeMemAccess)); - set_input(f, n, f->trace.bot_ctrl, 0); - set_input(f, n, append_mem(f, n), 1); - set_input(f, n, addr, 2); - set_input(f, n, val, 3); - set_input(f, n, size, 4); - TB_NODE_SET_EXTRA(n, TB_NodeMemAccess, .align = align); -} - -void tb_inst_memcpy(TB_Function* f, TB_Node* addr, TB_Node* val, TB_Node* size, TB_CharUnits align) { - assert(TB_IS_POINTER_TYPE(addr->dt)); - assert(TB_IS_POINTER_TYPE(val->dt)); - - TB_Node* n = tb_alloc_node(f, TB_MEMCPY, TB_TYPE_MEMORY, 5, sizeof(TB_NodeMemAccess)); - set_input(f, n, f->trace.bot_ctrl, 0); - set_input(f, n, append_mem(f, n), 1); - set_input(f, n, addr, 2); - set_input(f, n, val, 3); - set_input(f, n, size, 4); - TB_NODE_SET_EXTRA(n, TB_NodeMemAccess, .align = align); -} - -void tb_inst_memzero(TB_Function* f, TB_Node* dst, TB_Node* count, TB_CharUnits align) { - tb_inst_memset(f, dst, tb_inst_uint(f, TB_TYPE_I8, 0), count, align); -} - -TB_Node* tb_inst_bool(TB_Function* f, bool imm) { - TB_Node* n = tb_alloc_node(f, TB_INTEGER_CONST, TB_TYPE_BOOL, 1, sizeof(TB_NodeInt)); - set_input(f, n, f->root_node, 0); - TB_NODE_SET_EXTRA(n, TB_NodeInt, .value = imm); - return tb_pass_gvn_node(f, n); -} - -TB_Node* tb_inst_uint(TB_Function* f, TB_DataType dt, uint64_t imm) { - assert(TB_IS_POINTER_TYPE(dt) || TB_IS_INTEGER_TYPE(dt)); - - if (dt.type == TB_INT && dt.data < 64) { - uint64_t mask = ~UINT64_C(0) >> (64 - dt.data); - imm &= mask; - } - - TB_Node* n = tb_alloc_node(f, TB_INTEGER_CONST, dt, 1, sizeof(TB_NodeInt)); - set_input(f, n, f->root_node, 0); - TB_NODE_SET_EXTRA(n, TB_NodeInt, .value = imm); - return tb_pass_gvn_node(f, n); -} - -TB_Node* tb_inst_sint(TB_Function* f, TB_DataType dt, int64_t imm) { - assert(TB_IS_POINTER_TYPE(dt) || (TB_IS_INTEGER_TYPE(dt) && (dt.data <= 64))); - - TB_Node* n = tb_alloc_node(f, TB_INTEGER_CONST, dt, 1, sizeof(TB_NodeInt)); - set_input(f, n, f->root_node, 0); - TB_NODE_SET_EXTRA(n, TB_NodeInt, .value = imm); - return tb_pass_gvn_node(f, n); -} - -TB_Node* tb_inst_float32(TB_Function* f, float imm) { - TB_Node* n = tb_alloc_node(f, TB_FLOAT32_CONST, TB_TYPE_F32, 1, sizeof(TB_NodeFloat32)); - set_input(f, n, f->root_node, 0); - TB_NODE_SET_EXTRA(n, TB_NodeFloat32, .value = imm); - return tb_pass_gvn_node(f, n); -} - -TB_Node* tb_inst_float64(TB_Function* f, double imm) { - TB_Node* n = tb_alloc_node(f, TB_FLOAT64_CONST, TB_TYPE_F64, 1, sizeof(TB_NodeFloat64)); - set_input(f, n, f->root_node, 0); - TB_NODE_SET_EXTRA(n, TB_NodeFloat64, .value = imm); - return tb_pass_gvn_node(f, n); -} - -TB_Node* tb_inst_string(TB_Function* f, size_t len, const char* str) { - TB_Global* dummy = tb_global_create(f->super.module, 0, NULL, NULL, TB_LINKAGE_PRIVATE); - tb_global_set_storage(f->super.module, tb_module_get_rdata(f->super.module), dummy, len, 1, 1); - - char* dst = tb_global_add_region(f->super.module, dummy, 0, len); - memcpy(dst, str, len); - - return tb_inst_get_symbol_address(f, (TB_Symbol*) dummy); -} - -TB_Node* tb_inst_cstring(TB_Function* f, const char* str) { - return tb_inst_string(f, strlen(str) + 1, str); -} - -TB_Node* tb_inst_array_access(TB_Function* f, TB_Node* base, TB_Node* index, int64_t stride) { - TB_Node* n = tb_alloc_node(f, TB_ARRAY_ACCESS, TB_TYPE_PTR, 3, sizeof(TB_NodeArray)); - set_input(f, n, base, 1); - set_input(f, n, index, 2); - TB_NODE_SET_EXTRA(n, TB_NodeArray, .stride = stride); - return tb_pass_gvn_node(f, n); -} - -TB_Node* tb_inst_member_access(TB_Function* f, TB_Node* base, int64_t offset) { - if (offset == 0) { - return base; - } - - TB_Node* n = tb_alloc_node(f, TB_MEMBER_ACCESS, TB_TYPE_PTR, 2, sizeof(TB_NodeMember)); - set_input(f, n, base, 1); - TB_NODE_SET_EXTRA(n, TB_NodeMember, .offset = offset); - return tb_pass_gvn_node(f, n); -} - -TB_Node* tb_inst_get_symbol_address(TB_Function* f, TB_Symbol* target) { - assert(target != NULL); - - TB_Node* n = tb_alloc_node(f, TB_SYMBOL, TB_TYPE_PTR, 1, sizeof(TB_NodeSymbol)); - set_input(f, n, f->root_node, 0); - TB_NODE_SET_EXTRA(n, TB_NodeSymbol, .sym = target); - return tb_pass_gvn_node(f, n); -} - -TB_Node* tb_inst_syscall(TB_Function* f, TB_DataType dt, TB_Node* syscall_num, size_t param_count, TB_Node** params) { - TB_Node* n = tb_alloc_node(f, TB_SYSCALL, TB_TYPE_TUPLE, 3 + param_count, sizeof(TB_NodeCall) + sizeof(TB_Node*[3])); - set_input(f, n, syscall_num, 2); - FOREACH_N(i, 0, param_count) { - set_input(f, n, params[i], i + 3); - } - - // control proj - TB_Node* cproj = tb__make_proj(f, TB_TYPE_CONTROL, n, 0); - set_input(f, n, transfer_ctrl(f, cproj), 0); - - // memory proj - TB_Node* mproj = tb__make_proj(f, TB_TYPE_MEMORY, n, 1); - set_input(f, n, append_mem(f, mproj), 1); - - // return value - TB_Node* dproj = tb__make_proj(f, dt, n, 2); - - TB_NodeCall* c = TB_NODE_GET_EXTRA(n); - c->proj_count = 3; - c->proto = NULL; - c->projs[0] = cproj; - c->projs[1] = mproj; - c->projs[2] = dproj; - return dproj; -} - -TB_MultiOutput tb_inst_call(TB_Function* f, TB_FunctionPrototype* proto, TB_Node* target, size_t param_count, TB_Node** params) { - size_t proj_count = 2 + (proto->return_count > 1 ? proto->return_count : 1); - - TB_Node* n = tb_alloc_node(f, TB_CALL, TB_TYPE_TUPLE, 3 + param_count, sizeof(TB_NodeCall) + (sizeof(TB_Node*)*proj_count)); - set_input(f, n, target, 2); - FOREACH_N(i, 0, param_count) { - set_input(f, n, params[i], i + 3); - } - - TB_NodeCall* c = TB_NODE_GET_EXTRA(n); - c->proj_count = proj_count; - c->proto = proto; - - // control proj - TB_Node* cproj = tb__make_proj(f, TB_TYPE_CONTROL, n, 0); - set_input(f, n, transfer_ctrl(f, cproj), 0); - - // memory proj - TB_Node* mproj = tb__make_proj(f, TB_TYPE_MEMORY, n, 1); - set_input(f, n, append_mem(f, mproj), 1); - - // create data projections - TB_PrototypeParam* rets = TB_PROTOTYPE_RETURNS(proto); - FOREACH_N(i, 0, proto->return_count) { - c->projs[i + 2] = tb__make_proj(f, rets[i].dt, n, i + 2); - } - - // we'll slot a NULL so it's easy to tell when it's empty - if (proto->return_count == 0) { - c->projs[2] = NULL; - } - - c->projs[0] = cproj; - c->projs[1] = mproj; - - add_input_late(f, get_callgraph(f), n); - - if (proto->return_count == 1) { - return (TB_MultiOutput){ .count = proto->return_count, .single = c->projs[2] }; - } else { - return (TB_MultiOutput){ .count = proto->return_count, .multiple = c->projs + 2 }; - } -} - -void tb_inst_tailcall(TB_Function* f, TB_FunctionPrototype* proto, TB_Node* target, size_t param_count, TB_Node** params) { - TB_Node* n = tb_alloc_node(f, TB_TAILCALL, TB_TYPE_CONTROL, 3 + param_count, sizeof(TB_NodeTailcall)); - set_input(f, n, transfer_ctrl(f, n), 0); - set_input(f, n, peek_mem(f), 1); - set_input(f, n, target, 2); - FOREACH_N(i, 0, param_count) { - set_input(f, n, params[i], i + 3); - } - - TB_NodeTailcall* c = TB_NODE_GET_EXTRA(n); - c->proto = proto; - - add_input_late(f, get_callgraph(f), n); - add_input_late(f, f->root_node, n); -} - -TB_Node* tb_inst_poison(TB_Function* f, TB_DataType dt) { - TB_Node* n = tb_alloc_node(f, TB_POISON, dt, 1, 0); - set_input(f, n, f->root_node, 0); - return tb_pass_gvn_node(f, n); -} - -TB_Node* tb_inst_not(TB_Function* f, TB_Node* src) { - TB_Node* n = tb_alloc_node(f, TB_NOT, src->dt, 2, 0); - set_input(f, n, src, 1); - return tb_pass_gvn_node(f, n); -} - -TB_Node* tb_inst_bswap(TB_Function* f, TB_Node* src) { - TB_Node* n = tb_alloc_node(f, TB_BSWAP, src->dt, 2, 0); - set_input(f, n, src, 1); - return tb_pass_gvn_node(f, n); -} - -TB_Node* tb_inst_clz(TB_Function* f, TB_Node* src) { - assert(TB_IS_INTEGER_TYPE(src->dt)); - TB_Node* n = tb_alloc_node(f, TB_CLZ, TB_TYPE_I32, 2, 0); - set_input(f, n, src, 1); - return tb_pass_gvn_node(f, n); -} - -TB_Node* tb_inst_ctz(TB_Function* f, TB_Node* src) { - assert(TB_IS_INTEGER_TYPE(src->dt)); - TB_Node* n = tb_alloc_node(f, TB_CTZ, TB_TYPE_I32, 2, 0); - set_input(f, n, src, 1); - return tb_pass_gvn_node(f, n); -} - -TB_Node* tb_inst_popcount(TB_Function* f, TB_Node* src) { - assert(TB_IS_INTEGER_TYPE(src->dt)); - TB_Node* n = tb_alloc_node(f, TB_POPCNT, TB_TYPE_I32, 2, 0); - set_input(f, n, src, 1); - return tb_pass_gvn_node(f, n); -} - -TB_Node* tb_inst_neg(TB_Function* f, TB_Node* src) { - TB_DataType dt = src->dt; - if (src->type == TB_INTEGER_CONST) { - uint64_t x = TB_NODE_GET_EXTRA_T(src, TB_NodeInt)->value; - uint64_t mask = ~UINT64_C(0) >> (64 - dt.data); - - // two's complement negate is just invert and add 1 - uint64_t negated = (~x + 1) & mask; - return tb_inst_sint(f, dt, negated); - } else if (src->type == TB_FLOAT32_CONST) { - float x = TB_NODE_GET_EXTRA_T(src, TB_NodeFloat32)->value; - return tb_inst_float32(f, -x); - } else if (src->type == TB_FLOAT64_CONST) { - double x = TB_NODE_GET_EXTRA_T(src, TB_NodeFloat64)->value; - return tb_inst_float64(f, -x); - } else { - return tb_unary(f, TB_NEG, src->dt, src); - } -} - -TB_Node* tb_inst_select(TB_Function* f, TB_Node* cond, TB_Node* a, TB_Node* b) { - assert(TB_DATA_TYPE_EQUALS(a->dt, b->dt)); - - TB_Node* n = tb_alloc_node(f, TB_SELECT, a->dt, 4, 0); - set_input(f, n, cond, 1); - set_input(f, n, a, 2); - set_input(f, n, b, 3); - return tb_pass_gvn_node(f, n); -} - -TB_Node* tb_inst_and(TB_Function* f, TB_Node* a, TB_Node* b) { - // bitwise operators can't wrap - return tb_bin_arith(f, TB_AND, 0, a, b); -} - -TB_Node* tb_inst_or(TB_Function* f, TB_Node* a, TB_Node* b) { - assert(TB_DATA_TYPE_EQUALS(a->dt, b->dt)); - return tb_bin_arith(f, TB_OR, 0, a, b); -} - -TB_Node* tb_inst_xor(TB_Function* f, TB_Node* a, TB_Node* b) { - // bitwise operators can't wrap - return tb_bin_arith(f, TB_XOR, 0, a, b); -} - -TB_Node* tb_inst_add(TB_Function* f, TB_Node* a, TB_Node* b, TB_ArithmeticBehavior arith_behavior) { - return tb_bin_arith(f, TB_ADD, arith_behavior, a, b); -} - -TB_Node* tb_inst_sub(TB_Function* f, TB_Node* a, TB_Node* b, TB_ArithmeticBehavior arith_behavior) { - return tb_bin_arith(f, TB_SUB, arith_behavior, a, b); -} - -TB_Node* tb_inst_mul(TB_Function* f, TB_Node* a, TB_Node* b, TB_ArithmeticBehavior arith_behavior) { - return tb_bin_arith(f, TB_MUL, arith_behavior, a, b); -} - -TB_Node* tb_inst_div(TB_Function* f, TB_Node* a, TB_Node* b, bool signedness) { - TB_Node* peek = b->type == TB_SIGN_EXT ? b->inputs[1] : b; - if (peek->type == TB_INTEGER_CONST) { - TB_NodeInt* i = TB_NODE_GET_EXTRA(peek); - uint64_t log2 = tb_ffs(i->value) - 1; - if (i->value == UINT64_C(1) << log2) { - return tb_bin_arith(f, TB_SHR, 0, a, tb_inst_uint(f, a->dt, log2)); - } - } - - // division can't wrap or overflow - return tb_bin_arith(f, signedness ? TB_SDIV : TB_UDIV, 0, a, b); -} - -TB_Node* tb_inst_mod(TB_Function* f, TB_Node* a, TB_Node* b, bool signedness) { - // modulo can't wrap or overflow - return tb_bin_arith(f, signedness ? TB_SMOD : TB_UMOD, 0, a, b); -} - -TB_Node* tb_inst_shl(TB_Function* f, TB_Node* a, TB_Node* b, TB_ArithmeticBehavior arith_behavior) { - return tb_bin_arith(f, TB_SHL, arith_behavior, a, b); -} - -TB_Node* tb_inst_rol(TB_Function* f, TB_Node* a, TB_Node* b) { - return tb_bin_arith(f, TB_ROL, 0, a, b); -} - -TB_Node* tb_inst_ror(TB_Function* f, TB_Node* a, TB_Node* b) { - return tb_bin_arith(f, TB_ROR, 0, a, b); -} - -//////////////////////////////// -// Atomics -//////////////////////////////// -static TB_Node* atomic_op(TB_Function* f, int op, TB_DataType dt, TB_Node* addr, TB_Node* src, TB_MemoryOrder order) { - TB_Node* n = tb_alloc_node(f, op, TB_TYPE_TUPLE, src ? 4 : 3, sizeof(TB_NodeAtomic)); - set_input(f, n, f->trace.bot_ctrl, 0); - set_input(f, n, addr, 2); - if (src) { - set_input(f, n, src, 3); - } - - TB_Node* mproj = tb__make_proj(f, TB_TYPE_MEMORY, n, 0); - TB_Node* dproj = tb__make_proj(f, dt, n, 1); - TB_NODE_SET_EXTRA(n, TB_NodeAtomic, .order = order, .order2 = TB_MEM_ORDER_SEQ_CST); - - // memory proj - set_input(f, n, append_mem(f, mproj), 1); - return dproj; -} - -TB_Node* tb_inst_atomic_load(TB_Function* f, TB_Node* addr, TB_DataType dt, TB_MemoryOrder order) { - return atomic_op(f, TB_ATOMIC_LOAD, dt, addr, NULL, order); -} - -TB_Node* tb_inst_atomic_xchg(TB_Function* f, TB_Node* addr, TB_Node* src, TB_MemoryOrder order) { - assert(src); - return atomic_op(f, TB_ATOMIC_XCHG, src->dt, addr, src, order); -} - -TB_Node* tb_inst_atomic_add(TB_Function* f, TB_Node* addr, TB_Node* src, TB_MemoryOrder order) { - assert(src); - return atomic_op(f, TB_ATOMIC_ADD, src->dt, addr, src, order); -} - -TB_Node* tb_inst_atomic_sub(TB_Function* f, TB_Node* addr, TB_Node* src, TB_MemoryOrder order) { - assert(src); - return atomic_op(f, TB_ATOMIC_SUB, src->dt, addr, src, order); -} - -TB_Node* tb_inst_atomic_and(TB_Function* f, TB_Node* addr, TB_Node* src, TB_MemoryOrder order) { - assert(src); - return atomic_op(f, TB_ATOMIC_AND, src->dt, addr, src, order); -} - -TB_Node* tb_inst_atomic_xor(TB_Function* f, TB_Node* addr, TB_Node* src, TB_MemoryOrder order) { - assert(src); - return atomic_op(f, TB_ATOMIC_XOR, src->dt, addr, src, order); -} - -TB_Node* tb_inst_atomic_or(TB_Function* f, TB_Node* addr, TB_Node* src, TB_MemoryOrder order) { - assert(src); - return atomic_op(f, TB_ATOMIC_OR, src->dt, addr, src, order); -} - -TB_Node* tb_inst_atomic_cmpxchg(TB_Function* f, TB_Node* addr, TB_Node* expected, TB_Node* desired, TB_MemoryOrder succ, TB_MemoryOrder fail) { - assert(TB_DATA_TYPE_EQUALS(desired->dt, expected->dt)); - TB_DataType dt = desired->dt; - - TB_Node* n = tb_alloc_node(f, TB_ATOMIC_CAS, TB_TYPE_TUPLE, 5, sizeof(TB_NodeAtomic)); - set_input(f, n, f->trace.bot_ctrl, 0); - set_input(f, n, addr, 2); - set_input(f, n, expected, 3); - set_input(f, n, desired, 4); - - TB_Node* mproj = tb__make_proj(f, TB_TYPE_MEMORY, n, 0); - TB_Node* dproj = tb__make_proj(f, dt, n, 1); - TB_NODE_SET_EXTRA(n, TB_NodeAtomic, .order = succ, .order2 = fail); - - // memory proj - set_input(f, n, append_mem(f, mproj), 1); - return dproj; -} - -// TODO(NeGate): Maybe i should split the bitshift operations into a separate kind of -// operator that has different arithmatic behaviors, maybe like trap on a large shift amount -TB_Node* tb_inst_sar(TB_Function* f, TB_Node* a, TB_Node* b) { - // shift right can't wrap or overflow - return tb_bin_arith(f, TB_SAR, 0, a, b); -} - -TB_Node* tb_inst_shr(TB_Function* f, TB_Node* a, TB_Node* b) { - // shift right can't wrap or overflow - return tb_bin_arith(f, TB_SHR, 0, a, b); -} - -TB_Node* tb_inst_fadd(TB_Function* f, TB_Node* a, TB_Node* b) { - return tb_bin_farith(f, TB_FADD, a, b); -} - -TB_Node* tb_inst_fsub(TB_Function* f, TB_Node* a, TB_Node* b) { - return tb_bin_farith(f, TB_FSUB, a, b); -} - -TB_Node* tb_inst_fmul(TB_Function* f, TB_Node* a, TB_Node* b) { - return tb_bin_farith(f, TB_FMUL, a, b); -} - -TB_Node* tb_inst_fdiv(TB_Function* f, TB_Node* a, TB_Node* b) { - return tb_bin_farith(f, TB_FDIV, a, b); -} - -TB_Node* tb_inst_va_start(TB_Function* f, TB_Node* a) { - assert(a->type == TB_LOCAL); - - TB_Node* n = tb_alloc_node(f, TB_VA_START, TB_TYPE_PTR, 2, 0); - set_input(f, n, a, 1); - return tb_pass_gvn_node(f, n); -} - -TB_Node* tb_inst_x86_ldmxcsr(TB_Function* f, TB_Node* a) { - assert(a->dt.type == TB_INT && a->dt.data == 32); - - TB_Node* n = tb_alloc_node(f, TB_X86INTRIN_LDMXCSR, TB_TYPE_I32, 2, 0); - set_input(f, n, a, 1); - return tb_pass_gvn_node(f, n); -} - -TB_Node* tb_inst_cycle_counter(TB_Function* f) { - TB_Node* n = tb_alloc_node(f, TB_CYCLE_COUNTER, TB_TYPE_I64, 1, 0); - set_input(f, n, f->trace.bot_ctrl, 0); - return tb_pass_gvn_node(f, n); -} - -TB_Node* tb_inst_prefetch(TB_Function* f, TB_Node* addr, int level) { - TB_Node* n = tb_alloc_node(f, TB_PREFETCH, TB_TYPE_MEMORY, 2, sizeof(TB_NodePrefetch)); - set_input(f, n, addr, 1); - TB_NODE_SET_EXTRA(n, TB_NodePrefetch, .level = level); - return tb_pass_gvn_node(f, n); -} - -TB_Node* tb_inst_x86_stmxcsr(TB_Function* f) { - TB_Node* n = tb_alloc_node(f, TB_X86INTRIN_STMXCSR, TB_TYPE_I32, 1, 0); - set_input(f, n, f->trace.bot_ctrl, 0); - return tb_pass_gvn_node(f, n); -} - -TB_Node* tb_inst_x86_sqrt(TB_Function* f, TB_Node* a) { - return tb_unary(f, TB_X86INTRIN_SQRT, a->dt, a); -} - -TB_Node* tb_inst_x86_rsqrt(TB_Function* f, TB_Node* a) { - return tb_unary(f, TB_X86INTRIN_RSQRT, a->dt, a); -} - -TB_Node* tb_inst_cmp(TB_Function* f, TB_NodeType type, TB_Node* a, TB_Node* b) { - assert(TB_DATA_TYPE_EQUALS(a->dt, b->dt)); - - TB_Node* n = tb_alloc_node(f, type, TB_TYPE_BOOL, 3, sizeof(TB_NodeCompare)); - set_input(f, n, a, 1); - set_input(f, n, b, 2); - TB_NODE_SET_EXTRA(n, TB_NodeCompare, .cmp_dt = a->dt); - return tb_pass_gvn_node(f, n); -} - -TB_Node* tb_inst_cmp_eq(TB_Function* f, TB_Node* a, TB_Node* b) { - return tb_inst_cmp(f, TB_CMP_EQ, a, b); -} - -TB_Node* tb_inst_cmp_ne(TB_Function* f, TB_Node* a, TB_Node* b) { - return tb_inst_cmp(f, TB_CMP_NE, a, b); -} - -TB_Node* tb_inst_cmp_ilt(TB_Function* f, TB_Node* a, TB_Node* b, bool signedness) { - return tb_inst_cmp(f, signedness ? TB_CMP_SLT : TB_CMP_ULT, a, b); -} - -TB_Node* tb_inst_cmp_ile(TB_Function* f, TB_Node* a, TB_Node* b, bool signedness) { - return tb_inst_cmp(f, signedness ? TB_CMP_SLE : TB_CMP_ULE, a, b); -} - -TB_Node* tb_inst_cmp_igt(TB_Function* f, TB_Node* a, TB_Node* b, bool signedness) { - return tb_inst_cmp(f, signedness ? TB_CMP_SLT : TB_CMP_ULT, b, a); -} - -TB_Node* tb_inst_cmp_ige(TB_Function* f, TB_Node* a, TB_Node* b, bool signedness) { - return tb_inst_cmp(f, signedness ? TB_CMP_SLE : TB_CMP_ULE, b, a); -} - -TB_Node* tb_inst_cmp_flt(TB_Function* f, TB_Node* a, TB_Node* b) { - return tb_inst_cmp(f, TB_CMP_FLT, a, b); -} - -TB_Node* tb_inst_cmp_fle(TB_Function* f, TB_Node* a, TB_Node* b) { - return tb_inst_cmp(f, TB_CMP_FLE, a, b); -} - -TB_Node* tb_inst_cmp_fgt(TB_Function* f, TB_Node* a, TB_Node* b) { - return tb_inst_cmp(f, TB_CMP_FLT, b, a); -} - -TB_Node* tb_inst_cmp_fge(TB_Function* f, TB_Node* a, TB_Node* b) { - return tb_inst_cmp(f, TB_CMP_FLE, b, a); -} - -TB_Node* tb_inst_incomplete_phi(TB_Function* f, TB_DataType dt, TB_Node* region, size_t preds) { - TB_Node* n = tb_alloc_node(f, TB_PHI, dt, 1 + preds, 0); - set_input(f, n, region, 0); - return n; -} - -bool tb_inst_add_phi_operand(TB_Function* f, TB_Node* phi, TB_Node* region, TB_Node* val) { - assert(region->type != TB_REGION && "umm... im expecting a region not whatever that was"); - TB_Node* phi_region = phi->inputs[0]; - - // the slot to fill is based on the predecessor list of the region - FOREACH_N(i, 0, phi_region->input_count) { - TB_Node* pred = phi_region->inputs[i]; - while (pred->type != TB_REGION) pred = pred->inputs[0]; - - if (pred == region) { - set_input(f, phi, val, i + 1); - return true; - } - } - - return false; -} - -TB_Node* tb_inst_phi2(TB_Function* f, TB_Node* region, TB_Node* a, TB_Node* b) { - assert(TB_DATA_TYPE_EQUALS(a->dt, b->dt)); - - TB_Node* n = tb_alloc_node_dyn(f, TB_PHI, a->dt, 3, 3, 0); - set_input(f, n, region, 0); - set_input(f, n, a, 1); - set_input(f, n, b, 2); - return n; -} - -TB_API TB_Node* tb_inst_region(TB_Function* f) { - return tb_inst_new_trace(f).top_ctrl; -} - -TB_API TB_Trace tb_inst_new_trace(TB_Function* f) { - TB_Node* n = tb_alloc_node_dyn(f, TB_REGION, TB_TYPE_CONTROL, 0, 4, sizeof(TB_NodeRegion)); - - TB_Node* phi = tb_alloc_node_dyn(f, TB_PHI, TB_TYPE_MEMORY, 1, 5, 0); - set_input(f, phi, n, 0); - TB_NODE_SET_EXTRA(n, TB_NodeRegion, .mem_in = phi); - return (TB_Trace){ n, n, phi }; -} - -void tb_inst_set_region_name(TB_Function* f, TB_Node* n, ptrdiff_t len, const char* name) { - if (len < 0) len = strlen(name); - - TB_NodeRegion* r = TB_NODE_GET_EXTRA(n); - - char* newstr = alloc_from_node_arena(f, len + 1); - memcpy(newstr, name, len + 1); - r->tag = newstr; -} - -// this has to move things which is not nice... -static void add_input_late(TB_Function* f, TB_Node* n, TB_Node* in) { - assert(n->type == TB_REGION || n->type == TB_PHI || n->type == TB_ROOT || n->type == TB_CALLGRAPH || n->type == TB_MERGEMEM); - - if (n->input_count >= n->input_cap) { - size_t new_cap = n->input_count * 2; - TB_Node** new_inputs = alloc_from_node_arena(f, new_cap * sizeof(TB_Node*)); - if (n->inputs != NULL) { - memcpy(new_inputs, n->inputs, n->input_count * sizeof(TB_Node*)); - } - - n->inputs = new_inputs; - n->input_cap = new_cap; - } - - n->inputs[n->input_count] = in; - add_user(f, n, in, n->input_count, NULL); - n->input_count += 1; -} - -static void add_memory_edge(TB_Function* f, TB_Node* n, TB_Node* mem_state, TB_Node* target) { - assert(target->type == TB_REGION); - TB_NodeRegion* r = TB_NODE_GET_EXTRA(target); - assert(r->mem_in && r->mem_in->type == TB_PHI); - add_input_late(f, r->mem_in, mem_state); -} - -void tb_inst_goto(TB_Function* f, TB_Node* target) { - TB_Node* mem_state = peek_mem(f); - - // there's no need for a branch if the path isn't diverging. - TB_Node* n = f->trace.bot_ctrl; - f->trace.bot_ctrl = NULL; - - // just add the edge directly. - assert(n->dt.type == TB_CONTROL); - add_input_late(f, target, n); - add_memory_edge(f, n, mem_state, target); -} - -TB_Node* tb_inst_if(TB_Function* f, TB_Node* cond, TB_Node* if_true, TB_Node* if_false) { - TB_Node* mem_state = peek_mem(f); - - // generate control projections - TB_Node* n = tb_alloc_node(f, TB_BRANCH, TB_TYPE_TUPLE, 2, sizeof(TB_NodeBranch) + sizeof(TB_BranchKey)); - set_input(f, n, transfer_ctrl(f, NULL), 0); - set_input(f, n, cond, 1); - - FOREACH_N(i, 0, 2) { - TB_Node* target = i ? if_false : if_true; - - TB_Node* cproj = tb__make_proj(f, TB_TYPE_CONTROL, n, i); - add_input_late(f, target, cproj); - add_memory_edge(f, n, mem_state, target); - } - - TB_NodeBranch* br = TB_NODE_GET_EXTRA(n); - br->total_hits = 100; - br->succ_count = 2; - br->keys[0].key = 0; - br->keys[0].taken = 50; - return n; -} - -TB_Node* tb_inst_if2(TB_Function* f, TB_Node* cond, TB_Node* projs[2]) { - TB_Node* mem_state = peek_mem(f); - - // generate control projections - TB_Node* n = tb_alloc_node(f, TB_BRANCH, TB_TYPE_TUPLE, 2, sizeof(TB_NodeBranch) + sizeof(TB_BranchKey)); - set_input(f, n, transfer_ctrl(f, NULL), 0); - set_input(f, n, cond, 1); - - FOREACH_N(i, 0, 2) { - projs[i] = tb__make_proj(f, TB_TYPE_CONTROL, n, i); - } - - TB_NodeBranch* br = TB_NODE_GET_EXTRA(n); - br->total_hits = 100; - br->succ_count = 2; - br->keys[0].key = 0; - br->keys[0].taken = 50; - return n; -} - -// n is a TB_BRANCH with two successors, taken is the number of times it's true -void tb_inst_set_branch_freq(TB_Function* f, TB_Node* n, int total_hits, int taken) { - TB_NodeBranch* br = TB_NODE_GET_EXTRA(n); - assert(br->succ_count == 2 && "only works on if-branches"); - br->total_hits = total_hits; - br->keys[0].taken = total_hits - taken; -} - -TB_Node* tb_inst_branch(TB_Function* f, TB_DataType dt, TB_Node* key, TB_Node* default_label, size_t entry_count, const TB_SwitchEntry* entries) { - TB_Node* mem_state = peek_mem(f); - - // generate control projections - TB_Node* n = tb_alloc_node(f, TB_BRANCH, TB_TYPE_TUPLE, 2, sizeof(TB_NodeBranch) + (sizeof(TB_BranchKey) * entry_count)); - set_input(f, n, transfer_ctrl(f, NULL), 0); - set_input(f, n, key, 1); - - FOREACH_N(i, 0, 1 + entry_count) { - TB_Node* target = i ? entries[i - 1].value : default_label; - - TB_Node* cproj = tb__make_proj(f, TB_TYPE_CONTROL, n, i); - add_input_late(f, target, cproj); - add_memory_edge(f, n, mem_state, target); - } - - TB_NodeBranch* br = TB_NODE_GET_EXTRA(n); - br->total_hits = (1 + entry_count) * 10; - br->succ_count = 1 + entry_count; - FOREACH_N(i, 0, entry_count) { - br->keys[i].key = entries[i].key; - br->keys[i].taken = 10; - } - return n; -} - -void tb_inst_ret(TB_Function* f, size_t count, TB_Node** values) { - TB_Node* mem_state = peek_mem(f); - - // allocate return node - TB_Node* ret = f->ret_node; - TB_Node* ctrl = ret->inputs[0]; - assert(ctrl->type == TB_REGION); - - // add to PHIs - assert(ret->input_count >= 3 + count); - add_input_late(f, ret->inputs[1], mem_state); - - size_t i = 3; - for (; i < count + 3; i++) { - assert(ret->inputs[i]->dt.raw == values[i - 3]->dt.raw && "datatype mismatch"); - add_input_late(f, ret->inputs[i], values[i - 3]); - } - - size_t phi_count = ret->input_count; - for (; i < phi_count; i++) { - // put poison in the leftovers? - log_warn("%s: ir: generated poison due to inconsistent number of returned values", f->super.name); - - TB_Node* poison = tb_alloc_node(f, TB_POISON, ret->inputs[i]->dt, 1, 0); - set_input(f, poison, f->ret_node, 0); - - poison = tb__gvn(f, poison, 0); - add_input_late(f, ret->inputs[i], poison); - } - - // basically just tb_inst_goto without the memory PHI (we did it earlier) - TB_Node* n = f->trace.bot_ctrl; - f->trace.bot_ctrl = NULL; - - add_input_late(f, ctrl, n); -} diff --git a/vendor/tb/src/tb_internal.h b/vendor/tb/src/tb_internal.h deleted file mode 100644 index 599f8c8b..00000000 --- a/vendor/tb/src/tb_internal.h +++ /dev/null @@ -1,589 +0,0 @@ -#pragma once - -// Windows likes it's secure functions, i kinda do too -// but only sometimes and this isn't one of them -#if defined(_WIN32) && !defined(_CRT_SECURE_NO_WARNINGS) -#define _CRT_SECURE_NO_WARNINGS -#endif - -#include "tb.h" -#include "tb_formats.h" - -#include -#include -#include - -#if defined(_MSC_VER) && !defined(__clang__) -#include -#define thread_local __declspec(thread) -#define alignas(x) __declspec(align(x)) -#else -#define thread_local _Thread_local -#endif - -#ifndef _WIN32 -// NOTE(NeGate): I love how we assume that if it's not windows -// its just posix, these are the only options i guess -#include -#include -#include -#endif - -#include "tb_platform.h" -#include "dyn_array.h" -#include "builtins.h" - -#define NL_HASH_MAP_INLINE -#include -#include - -#include - -#define FOREACH_N(it, start, end) \ -for (ptrdiff_t it = (start), end__ = (end); it < end__; ++it) - -#define FOREACH_REVERSE_N(it, start, end) \ -for (ptrdiff_t it = (end), start__ = (start); (it--) > start__;) - -#define FOREACH_BIT(it, start, bits) \ -for (uint64_t _bits_ = (bits), it = (start); _bits_; _bits_ >>= 1, ++it) if (_bits_ & 1) - -#define TB_MIN(x, y) ((x) < (y) ? (x) : (y)) -#define TB_MAX(x, y) ((x) > (y) ? (x) : (y)) - -#define PP_ARG0(a, ...) a -#define PP_AFTER0(a, ...) __VA_ARGS__ - -#define PP_ID(...) __VA_ARGS__ -#define PP_VOID(...) -#define PP_JOIN(a, b) a b - -#ifndef CONCAT -#define CONCAT_(x, y) x ## y -#define CONCAT(x, y) CONCAT_(x, y) -#endif - -#include -#include "set.h" - -#include -#include - -typedef struct TB_Emitter { - size_t capacity, count; - uint8_t* data; -} TB_Emitter; - -#define TB_DATA_TYPE_EQUALS(a, b) ((a).raw == (b).raw) - -// i love my linked lists don't i? -typedef struct TB_SymbolPatch TB_SymbolPatch; -struct TB_SymbolPatch { - TB_SymbolPatch* next; - uint32_t pos; - bool internal; // handled already by the code gen's emit_call_patches - TB_Symbol* target; -}; - -struct TB_External { - TB_Symbol super; - TB_ExternalType type; - - void* thunk; // JIT will cache a thunk here because it's helpful - - // if non-NULL, the external was resolved - _Atomic(TB_Symbol*) resolved; -}; - -typedef struct TB_InitObj { - enum { - TB_INIT_OBJ_REGION, - TB_INIT_OBJ_RELOC, - } type; - TB_CharUnits offset; - union { - struct { - TB_CharUnits size; - const void* ptr; - } region; - - TB_Symbol* reloc; - }; -} TB_InitObj; - -struct TB_Global { - TB_Symbol super; - TB_Linkage linkage; - TB_ModuleSectionHandle parent; - - // layout stuff - void* address; // JIT-only - uint32_t pos; - TB_CharUnits size, align; - - // debug info - TB_DebugType* dbg_type; - - // contents - uint32_t obj_count, obj_capacity; - TB_InitObj* objects; -}; - -struct TB_DebugType { - enum { - TB_DEBUG_TYPE_VOID, - TB_DEBUG_TYPE_BOOL, - - TB_DEBUG_TYPE_UINT, - TB_DEBUG_TYPE_INT, - TB_DEBUG_TYPE_FLOAT, - - TB_DEBUG_TYPE_ARRAY, - TB_DEBUG_TYPE_POINTER, - - // special types - TB_DEBUG_TYPE_ALIAS, - TB_DEBUG_TYPE_FIELD, - - // aggregates - // TODO(NeGate): apparently codeview has cool vector and matrix types... yea - TB_DEBUG_TYPE_STRUCT, - TB_DEBUG_TYPE_UNION, - - TB_DEBUG_TYPE_FUNCTION, - } tag; - - // debug-info target specific data - union { - struct { - uint16_t type_id; - uint16_t type_id_fwd; // used by records to manage forward decls - }; - }; - - // tag specific - union { - int int_bits; - TB_FloatFormat float_fmt; - TB_DebugType* ptr_to; - struct { - TB_DebugType* base; - size_t count; - } array; - struct { - size_t len; - char* name; - TB_DebugType* type; - } alias; - struct { - size_t len; - char* name; - TB_CharUnits offset; - TB_DebugType* type; - } field; - struct TB_DebugTypeRecord { - size_t len; - char* tag; - TB_CharUnits size, align; - - size_t count; - TB_DebugType** members; - } record; - struct TB_DebugTypeFunc { - TB_CallingConv cc; - bool has_varargs; - - size_t param_count, return_count; - TB_DebugType** params; - TB_DebugType** returns; - } func; - }; -}; - -// TODO(NeGate): support complex variable descriptions -// currently we only support stack relative -typedef struct { - int32_t offset; -} TB_DebugValue; - -typedef struct TB_StackSlot { - const char* name; - TB_DebugType* type; - TB_DebugValue storage; -} TB_StackSlot; - -typedef struct TB_Comdat { - TB_ComdatType type; - uint32_t reloc_count; -} TB_Comdat; - -typedef struct { - uint32_t ip; // relative to the function body. - TB_Safepoint* sp; -} TB_SafepointKey; - -typedef struct COFF_UnwindInfo COFF_UnwindInfo; -typedef struct ICodeGen ICodeGen; - -typedef struct TB_FunctionOutput { - TB_Function* parent; - TB_ModuleSectionHandle section; - - TB_Linkage linkage; - - uint64_t ordinal; - uint8_t prologue_length; - uint8_t epilogue_length; - uint8_t nop_pads; - - TB_Assembly* asm_out; - uint64_t stack_usage; - - uint8_t* code; - size_t code_pos; // relative to the export-specific text section - size_t code_size; - - // export-specific - uint32_t wasm_type; - uint32_t unwind_info; - uint32_t unwind_size; - - DynArray(TB_StackSlot) stack_slots; - - // Part of the debug info - DynArray(TB_Location) locations; - - // Relocations - uint32_t patch_pos; - uint32_t patch_count; - - TB_SymbolPatch* first_patch; - TB_SymbolPatch* last_patch; -} TB_FunctionOutput; - -struct TB_Function { - TB_Symbol super; - TB_ModuleSectionHandle section; - TB_Linkage linkage; - - TB_DebugType* dbg_type; - TB_FunctionPrototype* prototype; - - // raw parameters - size_t param_count; - TB_Node** params; - - // IR allocation - TB_Arena* arena; - size_t node_count; - - TB_Node* ret_node; // ir building crap - - TB_Node* root_node; - TB_Trace trace; - - TB_NodeLocation* line_loc; - NL_Table locations; // TB_Node* -> TB_NodeLocation* - - // how we track duplicates for GVN, it's possible to run while optimizing - // the IR so we optionally do it. - NL_HashSet gvn_nodes; - - // Compilation output - union { - void* compiled_pos; - size_t compiled_symbol_id; - }; - - TB_FunctionOutput* output; -}; - -struct TB_ModuleSection { - char* name; - TB_LinkerSectionPiece* piece; - - int section_num; - TB_ModuleSectionFlags flags; - - TB_Comdat comdat; - - // export-specific - uint32_t export_flags; - uint32_t name_pos; - COFF_UnwindInfo* unwind; - - // this isn't computed until export time - uint32_t raw_data_pos; - uint32_t total_size; - uint32_t reloc_count; - uint32_t reloc_pos; - - DynArray(TB_Global*) globals; - DynArray(TB_FunctionOutput*) funcs; -}; - -typedef struct { - int len; - char data[16]; -} SmallConst; - -// only next_in_module is ever mutated on multiple threads (when first attached) -struct TB_ThreadInfo { - void* owner; - _Atomic(TB_ThreadInfo*) next_in_module; - - TB_ThreadInfo* prev; - TB_ThreadInfo* next; - - mtx_t* lock; - - // used for moving the start of the - // linked list forward. - TB_ThreadInfo** chain; - - TB_Arena* perm_arena; - TB_Arena* tmp_arena; - - // live symbols (globals, functions and externals) - // we'll be iterating these during object/executable - // export to get all the symbols compiled. - NL_HashSet symbols; -}; - -typedef struct { - size_t count; - TB_External** data; -} ExportList; - -struct TB_Module { - bool is_jit; - bool visited; // used by the linker - ICodeGen* codegen; - - atomic_flag is_tls_defined; - - // we have a global lock since the arena can be accessed - // from any thread. - mtx_t lock; - - // thread info - _Atomic(TB_ThreadInfo*) first_info_in_module; - - // small constants are interned because they - // come up a lot. - NL_Map(SmallConst, TB_Global*) global_interns; - - TB_ABI target_abi; - TB_Arch target_arch; - TB_System target_system; - TB_FeatureSet features; - ExportList exports; - - // This is a hack for windows since they've got this idea - // of a _tls_index - TB_Symbol* tls_index_extern; - TB_Symbol* chkstk_extern; - - _Atomic uint32_t uses_chkstk; - _Atomic uint32_t compiled_function_count; - _Atomic uint32_t symbol_count[TB_SYMBOL_MAX]; - - // needs to be locked with 'TB_Module.lock' - NL_Strmap(TB_SourceFile*) files; - - // unused by the JIT - DynArray(TB_ModuleSection) sections; - - // windows specific lol - TB_LinkerSectionPiece* xdata; -}; - -typedef struct { - size_t length; - TB_ObjectSection* data; -} TB_SectionGroup; - -typedef struct { - uint32_t used; - uint8_t data[]; -} TB_TemporaryStorage; - -struct ICodeGen { - // what does CHAR_BIT mean on said platform - int minimum_addressable_size, pointer_size; - - void (*get_data_type_size)(TB_DataType dt, size_t* out_size, size_t* out_align); - - // return the number of non-local patches - size_t (*emit_call_patches)(TB_Module* restrict m, TB_FunctionOutput* out_f); - - // NULLable if doesn't apply - void (*emit_win64eh_unwind_info)(TB_Emitter* e, TB_FunctionOutput* out_f, uint64_t stack_usage); - - void (*compile_function)(TB_Passes* p, TB_FunctionOutput* restrict func_out, const TB_FeatureSet* features, TB_Arena* arena, bool emit_asm); -}; - -// All debug formats i know of boil down to adding some extra sections to the object file -typedef struct { - const char* name; - - bool (*supported_target)(TB_Module* m); - int (*number_of_debug_sections)(TB_Module* m); - - // functions are laid out linearly based on their function IDs and - // thus function_sym_start tells you what the starting point is in the symbol table - TB_SectionGroup (*generate_debug_info)(TB_Module* m, TB_Arena* arena); -} IDebugFormat; - -#define TB_FITS_INTO(T,x) ((x) == (T)(x)) - -// tb_todo means it's something we fill in later -// tb_unreachable means it's logically impossible to reach -// tb_assume means we assume some expression cannot be false -// -// in debug builds these are all checked and tb_todo is some sort of trap -#if defined(_MSC_VER) && !defined(__clang__) -#if TB_DEBUG_BUILD -#define tb_todo() (assert(0 && "TODO"), __assume(0)) -#define tb_unreachable() (assert(0), __assume(0), 0) -#else -#define tb_todo() abort() -#define tb_unreachable() (__assume(0), 0) -#endif -#else -#if TB_DEBUG_BUILD -#define tb_todo() __builtin_trap() -#define tb_unreachable() (assert(0), 0) -#else -#define tb_todo() __builtin_trap() -#define tb_unreachable() (__builtin_unreachable(), 0) -#endif -#endif - -#ifndef NDEBUG -#define tb_assert(condition, ...) ((condition) ? 0 : (fprintf(stderr, __FILE__ ":" STR(__LINE__) ": assertion failed: " #condition "\n "), fprintf(stderr, __VA_ARGS__), abort(), 0)) -#else -#define tb_assert(condition, ...) (0) -#endif - -#if defined(_WIN32) && !defined(__GNUC__) -#define tb_panic(...) \ -do { \ - printf(__VA_ARGS__); \ - __fastfail(FAST_FAIL_FATAL_APP_EXIT); \ -} while (0) -#else -#define tb_panic(...) \ -do { \ - printf(__VA_ARGS__); \ - abort(); \ -} while (0) -#endif - -#ifndef COUNTOF -#define COUNTOF(...) (sizeof(__VA_ARGS__) / sizeof((__VA_ARGS__)[0])) -#endif - -TB_ThreadInfo* tb_thread_info(TB_Module* m); - -void* tb_out_reserve(TB_Emitter* o, size_t count); -void tb_out_commit(TB_Emitter* o, size_t count); - -// reserves & commits -void* tb_out_grab(TB_Emitter* o, size_t count); -size_t tb_out_grab_i(TB_Emitter* o, size_t count); -size_t tb_out_get_pos(TB_Emitter* o, void* p); - -// Adds null terminator onto the end and returns the starting position of the string -size_t tb_outstr_nul_UNSAFE(TB_Emitter* o, const char* str); - -void tb_out1b_UNSAFE(TB_Emitter* o, uint8_t i); -void tb_out4b_UNSAFE(TB_Emitter* o, uint32_t i); -void tb_outstr_UNSAFE(TB_Emitter* o, const char* str); -void tb_outs_UNSAFE(TB_Emitter* o, size_t len, const void* str); -size_t tb_outs(TB_Emitter* o, size_t len, const void* str); -void* tb_out_get(TB_Emitter* o, size_t pos); - -// fills region with zeros -void tb_out_zero(TB_Emitter* o, size_t len); - -void tb_out1b(TB_Emitter* o, uint8_t i); -void tb_out2b(TB_Emitter* o, uint16_t i); -void tb_out4b(TB_Emitter* o, uint32_t i); -void tb_out8b(TB_Emitter* o, uint64_t i); -void tb_patch1b(TB_Emitter* o, uint32_t pos, uint8_t i); -void tb_patch2b(TB_Emitter* o, uint32_t pos, uint16_t i); -void tb_patch4b(TB_Emitter* o, uint32_t pos, uint32_t i); -void tb_patch8b(TB_Emitter* o, uint32_t pos, uint64_t i); - -uint8_t tb_get1b(TB_Emitter* o, uint32_t pos); -uint16_t tb_get2b(TB_Emitter* o, uint32_t pos); -uint32_t tb_get4b(TB_Emitter* o, uint32_t pos); - -inline static uint64_t align_up(uint64_t a, uint64_t b) { - return a + (b - (a % b)) % b; -} - -// NOTE(NeGate): Considers 0 as a power of two -inline static bool tb_is_power_of_two(uint64_t x) { - return (x & (x - 1)) == 0; -} - -//////////////////////////////// -// HELPER FUNCTIONS -//////////////////////////////// -#ifdef _MSC_VER -#define TB_LIKELY(x) (!!(x)) -#define TB_UNLIKELY(x) (!!(x)) -#else -#define TB_LIKELY(x) __builtin_expect(!!(x), 1) -#define TB_UNLIKELY(x) __builtin_expect(!!(x), 0) -#endif - -TB_Node* tb_alloc_node_dyn(TB_Function* f, int type, TB_DataType dt, int input_count, int input_cap, size_t extra); -TB_Node* tb_alloc_node(TB_Function* f, int type, TB_DataType dt, int input_count, size_t extra); -TB_Node* tb__make_proj(TB_Function* f, TB_DataType dt, TB_Node* src, int index); - -ExportList tb_module_layout_sections(TB_Module* m); - -//////////////////////////////// -// EXPORTER HELPER -//////////////////////////////// -size_t tb_helper_write_section(TB_Module* m, size_t write_pos, TB_ModuleSection* section, uint8_t* output, uint32_t pos); -size_t tb__layout_relocations(TB_Module* m, DynArray(TB_ModuleSection) sections, const ICodeGen* restrict code_gen, size_t output_size, size_t reloc_size); - -TB_ExportChunk* tb_export_make_chunk(TB_Arena* arena, size_t size); -void tb_export_append_chunk(TB_ExportBuffer* buffer, TB_ExportChunk* c); - -//////////////////////////////// -// ANALYSIS -//////////////////////////////// -void set_input(TB_Function* f, TB_Node* n, TB_Node* in, int slot); -void add_user(TB_Function* f, TB_Node* n, TB_Node* in, int slot, User* recycled); -void print_node_sexpr(TB_Node* n, int depth); - -TB_Symbol* tb_symbol_alloc(TB_Module* m, TB_SymbolTag tag, ptrdiff_t len, const char* name, size_t size); -void tb_symbol_append(TB_Module* m, TB_Symbol* s); - -void tb_emit_symbol_patch(TB_FunctionOutput* func_out, TB_Symbol* target, size_t pos); -TB_Global* tb__small_data_intern(TB_Module* m, size_t len, const void* data); - -// out_bytes needs at least 16 bytes -void tb__md5sum(uint8_t* out_bytes, uint8_t* initial_msg, size_t initial_len); - -uint64_t tb__sxt(uint64_t src, uint64_t src_bits, uint64_t dst_bits); - -char* tb__arena_strdup(TB_Module* m, ptrdiff_t len, const char* src); -TB_Node* tb__gvn(TB_Function* f, TB_Node* n, size_t extra); - -static bool is_same_location(TB_Location* a, TB_Location* b) { - return a->file == b->file && a->line == b->line && a->column == b->column; -} - -static TB_Arena* get_temporary_arena(TB_Module* key) { - return tb_thread_info(key)->tmp_arena; -} - -static TB_Arena* get_permanent_arena(TB_Module* key) { - return tb_thread_info(key)->perm_arena; -} diff --git a/vendor/tb/src/tb_platform.h b/vendor/tb/src/tb_platform.h deleted file mode 100644 index 3d633df2..00000000 --- a/vendor/tb/src/tb_platform.h +++ /dev/null @@ -1,30 +0,0 @@ -// If you're trying to port TB on to a new platform you'll need to fill in these -// functions with their correct behavior. -#pragma once -#include - -#if defined(TB_USE_MIMALLOC) -#include -#define tb_platform_heap_alloc(size) mi_malloc(size) -#define tb_platform_heap_realloc(ptr, size) mi_realloc(ptr, size) -#define tb_platform_heap_free(ptr) mi_free(ptr) -#else -#define tb_platform_heap_alloc(size) malloc(size) -#define tb_platform_heap_free(ptr) free(ptr) -#define tb_platform_heap_realloc(ptr, size) realloc(ptr, size) -#endif - -//////////////////////////////// -// Virtual memory management -//////////////////////////////// -typedef enum { - TB_PAGE_RO, - TB_PAGE_RW, - TB_PAGE_RX, - TB_PAGE_RXW, -} TB_MemProtect; - -// This is used for JIT compiler pages or any large scale memory allocations. -void* tb_platform_valloc(size_t size); -void tb_platform_vfree(void* ptr, size_t size); -bool tb_platform_vprotect(void* ptr, size_t size, TB_MemProtect prot); diff --git a/vendor/tb/src/wasm/wasm_target.c b/vendor/tb/src/wasm/wasm_target.c deleted file mode 100644 index 07da7db1..00000000 --- a/vendor/tb/src/wasm/wasm_target.c +++ /dev/null @@ -1,864 +0,0 @@ -#ifdef TB_HAS_WASM -#include "../tb_internal.h" -#include "../emitter.h" -#include "../opt/passes.h" - -typedef struct { - int id; // where in the locals it went, -1 for unallocated - int uses; -} ValueDesc; - -// control flow element -typedef struct WasmElem WasmElem; -struct WasmElem { - enum { - WASM_BLOCK, - WASM_BLOCK2, // body goes after _then case - WASM_BR, - WASM_IF, - WASM_LOOP - } tag; - - bool refs; - uint16_t depth; - - TB_Node* body; - WasmElem* _then; - WasmElem* _else; // only relevant for WASM_IF -}; - -typedef struct DomTree DomTree; -struct DomTree { - int id; - WasmElem* elem; - TB_Node *start, *end; - DynArray(DomTree*) kids; -}; - -typedef struct Ctx { - TB_Passes* p; - TB_CGEmitter emit; - - TB_Module* module; - TB_Function* f; - TB_FeatureSet features; - - TB_CFG cfg; - size_t block_count; - - DomTree* doms; - - DynArray(DomTree*) filtered; - DynArray(PhiVal) phi_vals; - - int stack_n; - TB_Node* stack[64]; - - DynArray(uint8_t) local_desc; - ValueDesc* locals; -} Ctx; - -static void indent(int depth) { while (depth--) printf(" "); } - -// uleb128 encode -static void emit_uint(Ctx* ctx, uint64_t x) { - do { - uint32_t lo = x & 0x7F; - x >>= 7; - if (x) { - lo |= 0x80; - } - EMIT1(&ctx->emit, lo); - } while (x); -} - -WasmElem* new_elem(int tag, TB_Node* body) { - WasmElem* e = tb_arena_alloc(tmp_arena, sizeof(WasmElem)); - *e = (WasmElem){ .tag = tag, .body = body }; - return e; -} - -static uint8_t get_wasm_type(TB_DataType dt) { - switch (dt.type) { - case TB_INT: { - if (dt.data <= 8) return 0x7F; - if (dt.data <= 16) return 0x7F; - if (dt.data <= 32) return 0x7F; - if (dt.data <= 64) return 0x7E; - break; - } - case TB_FLOAT: { - if (dt.data == TB_FLT_32) return 0x7D; - if (dt.data == TB_FLT_64) return 0x7C; - break; - } - case TB_PTR: return 0x7F; - } - - assert(0 && "TODO"); - return 0; -} - -// uleb128 encode -static void patch_uint(uint8_t* p, uint64_t x) { - for (int i = 0; i < 4; i++) { - uint32_t lo = x & 0x7F; - x >>= 7; - if (i < 3) { - lo |= 0x80; - } - *p++ = lo; - } -} - -static int use_count(TB_Node* n) { - int c = 0; - FOR_USERS(u, n) { c += 1; } - return c; -} - -static int spill_tos(Ctx* ctx, TB_Node* n) { - // if there was something on top, we probably wanna spill it later - ValueDesc* spilled = &ctx->locals[n->gvn]; - if (spilled->id < 0) { - dyn_array_put(ctx->local_desc, get_wasm_type(n->dt)); - spilled->id = dyn_array_length(ctx->local_desc) - 1; - } - return spilled->id; -} - -static void push_val(Ctx* ctx, TB_Node* n) { - spill_tos(ctx, n); - ctx->stack[ctx->stack_n++] = n; -} - -static void compile_bb(Ctx* ctx, TB_Node* bb_start, int depth) { - TB_BasicBlock* bb = ctx->p->scheduled[bb_start->gvn]; - Worklist* ws = &ctx->p->worklist; - - #ifndef NDEBUG - TB_BasicBlock* expected = &nl_map_get_checked(ctx->cfg.node_to_block, bb_start); - assert(expected == bb); - #endif - - dyn_array_clear(ctx->phi_vals); - greedy_scheduler(ctx->p, &ctx->cfg, ws, &ctx->phi_vals, bb, bb->end); - - FOREACH_N(i, ctx->cfg.block_count, dyn_array_length(ws->items)) { - TB_Node* bot = ws->items[i]; - if (bot->type == TB_SYMBOL || bot->type == TB_PROJ || cfg_is_region(bot) || bot->type == TB_PHI || bot->type == TB_BRANCH) { - continue; - } - - // begin building up a tree (right now they'll all be tiny) - ctx->stack_n = 1; - ctx->stack[0] = bot; - - switch (bot->type) { - case TB_INTEGER_CONST: - case TB_FLOAT32_CONST: - case TB_FLOAT64_CONST: - break; - - // integer ops - case TB_AND: - case TB_OR: - case TB_XOR: - case TB_ADD: - case TB_SUB: - case TB_MUL: - case TB_SHL: - case TB_SHR: - case TB_SAR: - case TB_CMP_EQ: - case TB_CMP_NE: - case TB_CMP_SLT: - case TB_CMP_SLE: - case TB_CMP_ULT: - case TB_CMP_ULE: - case TB_CMP_FLT: - case TB_CMP_FLE: - case TB_FADD: - case TB_FSUB: - case TB_FMUL: - case TB_FDIV: - push_val(ctx, bot->inputs[2]); - push_val(ctx, bot->inputs[1]); - break; - - case TB_ARRAY_ACCESS: - push_val(ctx, bot->inputs[2]); - push_val(ctx, bot->inputs[1]); - break; - - case TB_MEMBER_ACCESS: - push_val(ctx, bot->inputs[1]); - break; - - case TB_CALL: - FOREACH_REVERSE_N(i, 3, bot->input_count) { - push_val(ctx, bot->inputs[i]); - } - break; - - case TB_LOAD: - push_val(ctx, bot->inputs[2]); - break; - - case TB_STORE: - push_val(ctx, bot->inputs[3]); - push_val(ctx, bot->inputs[2]); - break; - - case TB_RETURN: { - if (bot->input_count > 3) { - push_val(ctx, bot->inputs[3]); - } - break; - } - - default: tb_todo(); - } - - // ok emit ops now - FOREACH_REVERSE_N(i, 0, ctx->stack_n) { - TB_Node* n = ctx->stack[i]; - ValueDesc* val = &ctx->locals[n->gvn]; - - if (i != 0 && val->id >= 0) { - EMIT1(&ctx->emit, 0x20); - emit_uint(ctx, val->id); - continue; - } - - switch (n->type) { - case TB_INTEGER_CONST: { - TB_NodeInt* i = TB_NODE_GET_EXTRA(n); - EMIT1(&ctx->emit, 0x41); - emit_uint(ctx, i->value); - break; - } - - case TB_FLOAT32_CONST: { - TB_NodeFloat32* i = TB_NODE_GET_EXTRA(n); - - uint32_t x; - memcpy(&x, &i->value, sizeof(x)); - - EMIT1(&ctx->emit, 0x43); - EMIT4(&ctx->emit, x); - break; - } - - case TB_FLOAT64_CONST: { - TB_NodeFloat64* i = TB_NODE_GET_EXTRA(n); - - uint64_t x; - memcpy(&x, &i->value, sizeof(x)); - - EMIT1(&ctx->emit, 0x44); - EMIT8(&ctx->emit, x); - break; - } - - // integer ops - case TB_AND: - case TB_OR: - case TB_XOR: - case TB_ADD: - case TB_SUB: - case TB_MUL: { - static const uint8_t ops[] = { 0x71, 0x72, 0x73, 0x6A, 0x6B, 0x6C }; - - assert(n->dt.type == TB_INT); - int base = n->dt.data > 32 ? 0x7C - 0x6A : 0; - EMIT1(&ctx->emit, base + ops[n->type - TB_AND]); - break; - } - - // float ops - case TB_FADD: - case TB_FSUB: - case TB_FMUL: - case TB_FDIV: - case TB_FMIN: - case TB_FMAX: { - assert(n->dt.type == TB_FLOAT); - int base = n->dt.data == TB_FLT_64 ? 0xA0 : 0x92; - EMIT1(&ctx->emit, base + (n->type - TB_FADD)); - break; - } - - case TB_SHL: - case TB_SHR: - case TB_SAR: { - static const uint8_t ops[] = { 0 /* shl */, 2 /* shr */, 1 /* sar */ }; - - int base = n->dt.data > 32 ? 0x86 : 0x74; - EMIT1(&ctx->emit, base + ops[n->type - TB_SHL]); - break; - } - - case TB_LOAD: { - TB_CharUnits align = TB_NODE_GET_EXTRA_T(n, TB_NodeMemAccess)->align; - if (n->dt.type == TB_INT) { - if (0) {} - else if (n->dt.data <= 8) { EMIT1(&ctx->emit, 0x2D); } // i32.load8_u - else if (n->dt.data <= 16) { EMIT1(&ctx->emit, 0x2E); } // i32.load16_u - else if (n->dt.data <= 32) { EMIT1(&ctx->emit, 0x28); } // i32.load - else if (n->dt.data <= 64) { EMIT1(&ctx->emit, 0x29); } // i64.load - else tb_todo(); - } else { - if (0) {} - else if (n->dt.data == TB_FLT_32) { EMIT1(&ctx->emit, 0x2A); } // f32.load - else if (n->dt.data == TB_FLT_64) { EMIT1(&ctx->emit, 0x2B); } // f64.load - else tb_todo(); - } - - // memarg - EMIT1(&ctx->emit, align); // align - EMIT1(&ctx->emit, 0x00); // offset - break; - } - - case TB_STORE: { - TB_CharUnits align = TB_NODE_GET_EXTRA_T(n, TB_NodeMemAccess)->align; - TB_DataType dt = n->dt; - if (dt.type == TB_INT) { - if (0) {} - else if (dt.data <= 8) { EMIT1(&ctx->emit, 0x3A); } // i32.store8_u - else if (dt.data <= 16) { EMIT1(&ctx->emit, 0x3B); } // i32.store16_u - else if (dt.data <= 32) { EMIT1(&ctx->emit, 0x36); } // i32.store - else if (dt.data <= 64) { EMIT1(&ctx->emit, 0x37); } // i64.store - else tb_todo(); - } else { - if (0) {} - else if (dt.data == TB_FLT_32) { EMIT1(&ctx->emit, 0x38); } // f32.store - else if (dt.data == TB_FLT_64) { EMIT1(&ctx->emit, 0x39); } // f64.store - else tb_todo(); - } - - // memarg - EMIT1(&ctx->emit, align); // align - EMIT1(&ctx->emit, 0x00); - break; - } - - case TB_MEMBER_ACCESS: { - int32_t offset = TB_NODE_GET_EXTRA_T(n, TB_NodeMember)->offset; - - // i32.const stride - EMIT1(&ctx->emit, 0x41); - emit_uint(ctx, offset); - // i32.add - EMIT1(&ctx->emit, 0x6A); - break; - } - - case TB_ARRAY_ACCESS: { - int32_t stride = TB_NODE_GET_EXTRA_T(n, TB_NodeArray)->stride; - - // i32.const stride - EMIT1(&ctx->emit, 0x41); - emit_uint(ctx, stride); - // i32.mul - EMIT1(&ctx->emit, 0x6C); - // i32.add - EMIT1(&ctx->emit, 0x6A); - break; - } - - case TB_CALL: { - assert(n->inputs[2]->type == TB_SYMBOL); - TB_Symbol* sym = TB_NODE_GET_EXTRA_T(n->inputs[2], TB_NodeSymbol)->sym; - - // call - EMIT1(&ctx->emit, 0x10); - EMIT4(&ctx->emit, 0x00808080); - tb_emit_symbol_patch(ctx->emit.output, sym, ctx->emit.count - 4); - break; - } - - case TB_CMP_EQ: - case TB_CMP_NE: - case TB_CMP_SLT: - case TB_CMP_SLE: - case TB_CMP_ULT: - case TB_CMP_ULE: - { - // position of the 32bit ops, we'll relocate for 64bit ones (same offsets) - static const uint8_t ops[] = { 0x46, 0x47, 0x48, 0x4C, 0x49, 0x4D }; - - if (n->dt.type == TB_INT || n->dt.type == TB_PTR) { - int bits = n->dt.type == TB_INT ? n->dt.data : 32; - int base = bits > 32 ? 0x51 - 0x46 : 0; - - EMIT1(&ctx->emit, base + ops[n->type - TB_CMP_EQ]); - } else { - tb_todo(); - } - break; - } - - case TB_RETURN: { - EMIT1(&ctx->emit, 0x0F); - break; - } - - default: tb_todo(); - } - } - - if (bot->dt.type == TB_INT || bot->dt.type == TB_PTR || bot->dt.type == TB_FLOAT) { - int dst = spill_tos(ctx, bot); - EMIT1(&ctx->emit, 0x21); - emit_uint(ctx, dst); - } - } - - // if the endpoint is a not a terminator, we've hit some implicit GOTO edge - TB_Node* end = bb->end; - if (!cfg_is_terminator(end)) { - // writeback phis - FOREACH_N(i, 0, dyn_array_length(ctx->phi_vals)) { - PhiVal* v = &ctx->phi_vals[i]; - - // get_local src - int src = ctx->locals[v->n->gvn].id; - EMIT1(&ctx->emit, 0x20); - emit_uint(ctx, src); - // set_local dst - int dst = spill_tos(ctx, v->phi); - EMIT1(&ctx->emit, 0x21); - emit_uint(ctx, dst); - } - } -} - -static TB_Node** successors(Ctx* ctx, Worklist* ws, TB_Node* end, size_t* out_succ_count) { - size_t succ_count = !cfg_is_endpoint(end); - if (end->type == TB_BRANCH) { - succ_count = TB_NODE_GET_EXTRA_T(end, TB_NodeBranch)->succ_count; - } - *out_succ_count = succ_count; - - size_t base = dyn_array_length(ws->items); - dyn_array_put_uninit(ws->items, succ_count); - TB_Node** succ_blocks = &ws->items[base]; - - if (end->type == TB_BRANCH) { - FOR_USERS(u, end) { - if (u->n->type == TB_PROJ) { - TB_Node* succ = cfg_next_bb_after_cproj(u->n); - int index = TB_NODE_GET_EXTRA_T(u->n, TB_NodeProj)->index; - succ_blocks[index] = succ; - } - } - } else if (!cfg_is_endpoint(end)) { - TB_Node* succ = cfg_next_control(end); - if (succ) { succ_blocks[0] = succ; } - } - - return succ_blocks; -} - -static bool wasm_is_natural_loop(Ctx* ctx, TB_Node* header) { - if (cfg_is_region(header)) { - FOREACH_N(i, 0, header->input_count) { - TB_Node* pred = cfg_get_pred(&ctx->cfg, header, i); - if (slow_dommy(&ctx->cfg, header, pred)) { - return true; - } - } - } - - return false; -} - -static bool has_merge_root(Ctx* ctx, TB_Node* n, int id) { - if (!cfg_is_region(n)) { - return false; - } - - FOREACH_N(i, 0, n->input_count) { - TB_Node* pred = cfg_get_pred(&ctx->cfg, n, i); - TB_BasicBlock* pred_bb = ctx->p->scheduled[pred->gvn]; - - if (pred_bb->id >= id) { - return false; - } - } - - return true; -} - -static WasmElem* do_dom_tree(Ctx* ctx, DomTree* node); - -static WasmElem* do_branch(Ctx* ctx, DomTree* src, TB_Node* bb_start) { - TB_BasicBlock* bb = ctx->p->scheduled[bb_start->gvn]; - DomTree* dst = &ctx->doms[bb->id]; - - if (dst->id < src->id || has_merge_root(ctx, bb_start, dst->id)) { - // forward or backwards edge (continue/break) - WasmElem* e = new_elem(WASM_BR, NULL); - assert(dst->elem); - e->_then = dst->elem; - dst->elem->refs = true; - return e; - } else { - return do_dom_tree(ctx, dst); - } -} - -static WasmElem* do_dom_tree(Ctx* ctx, DomTree* node) { - bool loop = wasm_is_natural_loop(ctx, node->start); - WasmElem* e = node->elem = new_elem(loop ? WASM_LOOP : WASM_BLOCK, node->start); - - // find merges (embed within each other) - size_t base = dyn_array_length(ctx->filtered); - WasmElem* last = NULL; - FOREACH_N(i, 0, dyn_array_length(node->kids)) { - TB_Node* start = node->kids[i]->start; - TB_BasicBlock* start_bb = ctx->p->scheduled[start->gvn]; - - DomTree* kid = node->kids[i]; - if (has_merge_root(ctx, start, start_bb->id)) { - dyn_array_put(ctx->filtered, kid); - - if (last) { - WasmElem* new_e = new_elem(WASM_BLOCK2, start); - new_e->_then = last; - last = kid->elem = new_e; - } else { - last = kid->elem; - } - } - } - - // exit path of node - size_t succ_count; - TB_Node** succ_blocks = successors(ctx, &ctx->p->worklist, node->end, &succ_count); - if (succ_count == 1) { - e->_then = do_branch(ctx, node, succ_blocks[0]); - } else if (succ_count == 2) { - WasmElem* if_block = new_elem(WASM_IF, NULL); - e->_then = if_block; - if_block->_then = do_branch(ctx, node, succ_blocks[0]); - if_block->_else = do_branch(ctx, node, succ_blocks[1]); - } else if (succ_count != 0) { - tb_todo(); - } - - // compile merges - FOREACH_N(i, base, dyn_array_length(ctx->filtered)) { - do_dom_tree(ctx, ctx->filtered[i]); - } - if (ctx->filtered) { - dyn_array_set_length(ctx->filtered, base); - } - return e; -} - -static const char* elem_name(WasmElem* elem) { - switch (elem->tag) { - case WASM_BLOCK: return "block"; - case WASM_BLOCK2: return "block"; - case WASM_LOOP: return "loop"; - case WASM_BR: return "br"; - case WASM_IF: return "if"; - default: return NULL; - } -} - -static uint8_t elem_byte(WasmElem* elem) { - switch (elem->tag) { - case WASM_BLOCK: return 0x02; - case WASM_BLOCK2: return 0x02; - case WASM_LOOP: return 0x03; - case WASM_IF: return 0x04; - default: return 0; - } -} - -static void render_block(Ctx* ctx, WasmElem* elem, int depth) { - elem->depth = depth; - - // no block necessary if no one jumps to it - bool is_block = elem->tag == WASM_IF || elem->refs > 0; - if (is_block) { - indent(depth), printf("%s", elem_name(elem)); - if (elem->tag != WASM_BR) { - printf(" (depth=%d)", depth); - } - printf("\n"); - } - - int inner_depth = depth + is_block; - switch (elem->tag) { - case WASM_BLOCK: - case WASM_LOOP: - if (elem->body) { indent(inner_depth), printf("v%u\n", elem->body->gvn); } - if (elem->_then) { render_block(ctx, elem->_then, inner_depth); } - break; - - case WASM_BLOCK2: - if (elem->_then) { render_block(ctx, elem->_then, inner_depth); } - if (elem->body) { indent(inner_depth), printf("v%u\n", elem->body->gvn); } - break; - - case WASM_IF: - render_block(ctx, elem->_then, inner_depth); - indent(depth), printf("else\n"); - render_block(ctx, elem->_else, inner_depth); - break; - - case WASM_BR: - indent(depth), printf("br %d\n", elem->_then->depth); - break; - } - - if (is_block) { - indent(depth), printf("end\n"); - } -} - -static void compile_block(Ctx* ctx, WasmElem* elem, int depth) { - elem->depth = depth; - - // no block necessary if no one jumps to it - bool is_block = elem->tag == WASM_IF || elem->refs > 0; - if (is_block) { - EMIT1(&ctx->emit, elem_byte(elem)); - EMIT1(&ctx->emit, 0x40); - } - - int inner_depth = depth + is_block; - switch (elem->tag) { - case WASM_BLOCK: - case WASM_LOOP: - if (elem->body) { compile_bb(ctx, elem->body, inner_depth); } - if (elem->_then) { compile_block(ctx, elem->_then, inner_depth); } - break; - - case WASM_BLOCK2: - if (elem->_then) { compile_block(ctx, elem->_then, inner_depth); } - if (elem->body) { compile_bb(ctx, elem->body, inner_depth); } - break; - - case WASM_IF: - compile_block(ctx, elem->_then, inner_depth); - EMIT1(&ctx->emit, 0x05); - compile_block(ctx, elem->_else, inner_depth); - break; - - case WASM_BR: - EMIT1(&ctx->emit, 0x0C); - emit_uint(ctx, depth - elem->_then->depth); - break; - } - - if (is_block) { - EMIT1(&ctx->emit, 0x0B); - } -} - -static int dom_sort_cmp(const void* a, const void* b) { - DomTree* const* aa = a; - DomTree* const* bb = b; - return aa[0]->id - bb[0]->id; -} - -static void compile_function(TB_Passes* restrict p, TB_FunctionOutput* restrict func_out, const TB_FeatureSet* features, TB_Arena* code_arena, bool emit_asm) { - verify_tmp_arena(p); - - TB_Arena* arena = tmp_arena; - TB_ArenaSavepoint sp = tb_arena_save(arena); - - TB_Function* restrict f = p->f; - TB_OPTDEBUG(CODEGEN)(tb_pass_print(p)); - - Ctx ctx = { - .module = f->super.module, - .f = f, - .p = p, - .emit = { - .output = func_out, - .arena = arena, - .has_comments = false, - } - }; - - // allocate entire top of the code arena (we'll trim it later if possible) - ctx.emit.capacity = code_arena->high_point - code_arena->watermark; - ctx.emit.data = tb_arena_alloc(code_arena, ctx.emit.capacity); - ctx.local_desc = dyn_array_create(uint8_t, 32); - - // patch place for the code size of the function - EMIT4(&ctx.emit, 0x00808080); - EMIT4(&ctx.emit, 0x00808080); - - ctx.locals = tb_arena_alloc(arena, f->node_count * sizeof(ValueDesc)); - FOREACH_N(i, 0, f->node_count) { - ctx.locals[i].id = -1; - } - - FOR_USERS(u, f->root_node) if (u->n->type == TB_PROJ) { - int i = TB_NODE_GET_EXTRA_T(u->n, TB_NodeProj)->index; - if (i >= 3) { - // params fit into the first few locals - ctx.locals[u->n->gvn].id = i - 3; - ctx.locals[u->n->gvn].uses = use_count(u->n); - dyn_array_put(ctx.local_desc, get_wasm_type(u->n->dt)); - } - } - - if (features == NULL) { - ctx.features = (TB_FeatureSet){ 0 }; - } else { - ctx.features = *features; - } - - Worklist* restrict ws = &p->worklist; - - // legalize step takes out any of our 16bit and 8bit math ops - tb_pass_prep(p); - tb_pass_legalize(p, f->super.module->target_arch); - - worklist_clear(ws); - - CUIK_TIMED_BLOCK("global sched") { - // We need to generate a CFG - ctx.cfg = tb_compute_rpo(f, p); - ctx.block_count = dyn_array_length(ws->items); - // And perform global scheduling - tb_pass_schedule(p, ctx.cfg, false); - } - - DomTree* doms = ctx.doms = tb_arena_alloc(tmp_arena, ctx.cfg.block_count * sizeof(DomTree)); - FOREACH_N(i, 0, ctx.cfg.block_count) { - doms[i].id = i; - doms[i].elem = NULL; - doms[i].start = NULL; - doms[i].end = NULL; - doms[i].kids = NULL; - } - - FOREACH_N(i, 0, ctx.cfg.block_count) { - TB_BasicBlock* bb = ctx.p->scheduled[ws->items[i]->gvn]; - doms[i].start = ws->items[i]; - doms[i].end = bb->end; - - dyn_array_put(doms[bb->dom->id].kids, &doms[bb->id]); - } - - FOREACH_N(i, 0, ctx.cfg.block_count) { - DomTree* node = &ctx.doms[i]; - qsort(node->kids, dyn_array_length(node->kids), sizeof(DomTree*), dom_sort_cmp); - } - - CUIK_TIMED_BLOCK("emit") { - worklist_clear_visited(ws); - - WasmElem* root = do_dom_tree(&ctx, &doms[0]); - #if TB_OPTDEBUG_CODEGEN - render_block(&ctx, root, 0); - #endif - compile_block(&ctx, root, 0); - } - - tb_free_cfg(&ctx.cfg); - dyn_array_destroy(ctx.phi_vals); - - // uleb code size patch - patch_uint(ctx.emit.data, ctx.emit.count - 4); - patch_uint(&ctx.emit.data[4], 0); - - // trim code arena (it fits in a single chunk so just arena free the top) - code_arena->watermark = (char*) &ctx.emit.data[ctx.emit.count]; - tb_arena_realign(code_arena); - - // TODO(NeGate): move the assembly output to code arena - if (emit_asm) CUIK_TIMED_BLOCK("dissassembly") { - // __debugbreak(); - } - - log_debug("%s: code_arena=%.1f KiB", f->super.name, tb_arena_current_size(code_arena) / 1024.0f); - tb_arena_restore(arena, sp); - p->scheduled = NULL; - - // we're done, clean up - func_out->asm_out = ctx.emit.head_asm; - func_out->code = ctx.emit.data; - func_out->code_size = ctx.emit.count; - // func_out->locations = ctx.locations; - // func_out->stack_slots = ctx.debug_stack_slots; - // func_out->stack_usage = ctx.stack_usage; - // func_out->prologue_length = ctx.prologue_length; - // func_out->epilogue_length = ctx.epilogue_length; -} - -static size_t emit_call_patches(TB_Module* restrict m, TB_FunctionOutput* out_f) { - size_t r = 0; - uint32_t src_section = out_f->section; - - for (TB_SymbolPatch* patch = out_f->first_patch; patch; patch = patch->next) { - if (patch->target->tag == TB_SYMBOL_FUNCTION) { - uint32_t dst_section = ((TB_Function*) patch->target)->output->section; - - // we can relocate across section on WASM, the "object" file isn't really - // relocatable in the same sense as other objects. - assert(patch->pos < out_f->code_size); - size_t actual_pos = out_f->code_pos + patch->pos + 4; - - uint32_t p = ((TB_Function*) patch->target)->output->code_pos - actual_pos; - patch_uint(&out_f->code[patch->pos], p); - - r += 1; - patch->internal = true; - } - } - - return out_f->patch_count - r; -} - -static void get_data_type_size(TB_DataType dt, size_t* out_size, size_t* out_align) { - switch (dt.type) { - case TB_INT: { - // above 64bits we really dont care that much about natural alignment - bool is_big_int = dt.data > 64; - - // round up bits to a byte - int bits = is_big_int ? ((dt.data + 7) / 8) : tb_next_pow2(dt.data - 1); - - *out_size = ((bits+7) / 8); - *out_align = is_big_int ? 8 : ((dt.data + 7) / 8); - break; - } - case TB_FLOAT: { - int s = 0; - if (dt.data == TB_FLT_32) s = 4; - else if (dt.data == TB_FLT_64) s = 8; - else tb_unreachable(); - - *out_size = s; - *out_align = s; - break; - } - case TB_PTR: { - *out_size = 8; - *out_align = 8; - break; - } - default: tb_unreachable(); - } -} - -ICodeGen tb__wasm32_codegen = { - .minimum_addressable_size = 8, - .pointer_size = 32, - .emit_win64eh_unwind_info = NULL, - .emit_call_patches = emit_call_patches, - .get_data_type_size = get_data_type_size, - .compile_function = compile_function, -}; -#endif diff --git a/vendor/tb/src/x64/x64.c b/vendor/tb/src/x64/x64.c deleted file mode 100644 index 6604ca2b..00000000 --- a/vendor/tb/src/x64/x64.c +++ /dev/null @@ -1,2391 +0,0 @@ -#include "x64.h" -#include "x64_emitter.h" -#include "../objects/win64eh.h" - -#include -#include "x64_disasm.c" - -enum { - CG_REGISTER_CLASSES = 2 -}; - -enum { - REG_CLASS_GPR, - REG_CLASS_XMM, - - FIRST_GPR = 0, - FIRST_XMM = 16, // we're getting more GPRs in intel APX so this might change :) -}; - -static const struct ParamDescriptor { - int gpr_count; - int xmm_count; - uint16_t caller_saved_xmms; // XMM0 - XMMwhatever - uint16_t caller_saved_gprs; // bitfield - - GPR gprs[6]; -} param_descs[] = { - // win64 - { 4, 4, 6, WIN64_ABI_CALLER_SAVED, { RCX, RDX, R8, R9, 0, 0 } }, - // system v - { 6, 4, 5, SYSV_ABI_CALLER_SAVED, { RDI, RSI, RDX, RCX, R8, R9 } }, - // syscall - { 6, 4, 5, SYSCALL_ABI_CALLER_SAVED, { RDI, RSI, RDX, R10, R8, R9 } }, -}; - -#include "../generic_cg.h" - -static size_t emit_prologue(Ctx* restrict ctx); -static void emit_epilogue(Ctx* restrict ctx); - -// initialize register allocator state -static void init_regalloc(Ctx* restrict ctx) { - // Generate intervals for physical registers - FOREACH_N(i, 0, 32) { - bool is_gpr = i < 16; - int reg = i % 16; - - dyn_array_put(ctx->intervals, (LiveInterval){ - .reg_class = is_gpr ? REG_CLASS_GPR : REG_CLASS_XMM, - .dt = is_gpr ? TB_X86_TYPE_QWORD : TB_X86_TYPE_XMMWORD, - .reg = reg, .assigned = reg, .hint = -1, .split_kid = -1, - }); - - LiveInterval* it = &ctx->intervals[i]; - it->ranges = tb_platform_heap_alloc(4 * sizeof(LiveRange)); - it->range_count = 1; - it->range_cap = 4; - it->ranges[0] = (LiveRange){ INT_MAX, INT_MAX }; - } -} - -static void mark_callee_saved_constraints(Ctx* restrict ctx, uint64_t callee_saved[CG_REGISTER_CLASSES]) { - bool is_sysv = (ctx->target_abi == TB_ABI_SYSTEMV); - const struct ParamDescriptor* restrict desc = ¶m_descs[is_sysv ? 1 : 0]; - - // don't include RBP and RSP, those are special cases - uint32_t callee_saved_gprs = ~desc->caller_saved_gprs; - callee_saved_gprs &= ~(1u << RBP); - callee_saved_gprs &= ~(1u << RSP); - callee_saved[0] = callee_saved_gprs; - - // mark XMM callees - callee_saved[1] = 0; - FOREACH_N(i, desc->caller_saved_xmms, 16) { - callee_saved[1] |= (1ull << i); - } -} - -// *out_mask of 0 means no mask -static TB_X86_DataType legalize_int(TB_DataType dt, uint64_t* out_mask) { - assert(dt.type == TB_INT || dt.type == TB_PTR); - if (dt.type == TB_PTR) return *out_mask = 0, TB_X86_TYPE_QWORD; - - TB_X86_DataType t = TB_X86_TYPE_NONE; - int bits = 0; - - if (dt.data <= 8) bits = 8, t = TB_X86_TYPE_BYTE; - else if (dt.data <= 16) bits = 16, t = TB_X86_TYPE_WORD; - else if (dt.data <= 32) bits = 32, t = TB_X86_TYPE_DWORD; - else if (dt.data <= 64) bits = 64, t = TB_X86_TYPE_QWORD; - - assert(bits != 0 && "TODO: large int support"); - uint64_t mask = dt.data == 0 ? 0 : ~UINT64_C(0) >> (64 - dt.data); - - *out_mask = (dt.data == bits) ? 0 : mask; - return t; -} - -static TB_X86_DataType legalize_int2(TB_DataType dt) { - uint64_t m; - return legalize_int(dt, &m); -} - -static TB_X86_DataType legalize_float(TB_DataType dt) { - assert(dt.type == TB_FLOAT); - return (dt.data == TB_FLT_64 ? TB_X86_TYPE_SSE_SD : TB_X86_TYPE_SSE_SS); -} - -static TB_X86_DataType legalize(TB_DataType dt) { - if (dt.type == TB_FLOAT) { - return legalize_float(dt); - } else { - uint64_t m; - return legalize_int(dt, &m); - } -} - -static int classify_reg_class(TB_DataType dt) { - return dt.type == TB_FLOAT ? REG_CLASS_XMM : REG_CLASS_GPR; -} - -static bool wont_spill_around(int t) { - return t == INST_LABEL || t == TEST || t == CMP || t == JMP || (t >= JO && t <= JG); -} - -static bool is_terminator(int t) { - return t == INST_TERMINATOR || t == INT3 || t == UD2; -} - -static bool try_for_imm32(Ctx* restrict ctx, int bits, TB_Node* n, int32_t* out_x) { - if (n->type != TB_INTEGER_CONST) { - return false; - } - - TB_NodeInt* i = TB_NODE_GET_EXTRA(n); - if (bits > 32) { - bool sign = (i->value >> 31ull) & 1; - uint64_t top = i->value >> 32ull; - - // if the sign matches the rest of the top bits, we can sign extend just fine - if (top != (sign ? 0xFFFFFFFF : 0)) { - return false; - } - } - - *out_x = i->value; - return true; -} - -static int get_stack_slot(Ctx* restrict ctx, TB_Node* n) { - ptrdiff_t search = nl_map_get(ctx->stack_slots, n); - if (search >= 0) { - return ctx->stack_slots[search].v; - } else { - TB_NodeLocal* local = TB_NODE_GET_EXTRA(n); - - int pos = STACK_ALLOC(local->size, local->align); - nl_map_put(ctx->stack_slots, n, pos); - - add_debug_local(ctx, n, pos); - return pos; - } -} - -static Inst* inst_jmp(int target) { - Inst* i = alloc_inst(JMP, TB_TYPE_VOID, 0, 0, 0); - i->flags = INST_NODE; - i->l = target; - return i; -} - -static Inst* inst_jmp_reg(int target) { - Inst* i = alloc_inst(JMP, TB_TYPE_VOID, 0, 1, 0); - i->operands[0] = target; - return i; -} - -static Inst* inst_jcc(int target, Cond cc) { - Inst* i = alloc_inst(JO + cc, TB_TYPE_VOID, 0, 0, 0); - i->flags = INST_NODE; - i->l = target; - return i; -} - -// store(binop(load(a), b)) -static int can_folded_store(Ctx* restrict ctx, TB_Node* mem, TB_Node* addr, TB_Node* src) { - switch (src->type) { - default: return -1; - - case TB_AND: - case TB_OR: - case TB_XOR: - case TB_ADD: - case TB_SUB: { - if (src->inputs[1]->type == TB_LOAD && - src->inputs[1]->inputs[1] == mem && - src->inputs[1]->inputs[2] == addr && - on_last_use(ctx, src) && - on_last_use(ctx, src->inputs[1])) { - const static InstType ops[] = { AND, OR, XOR, ADD, SUB }; - return ops[src->type - TB_AND]; - } - - return -1; - } - } -} - -// generates an LEA for computing the address of n. -static Inst* isel_addr(Ctx* restrict ctx, TB_Node* n, int dst, int store_op, int src) { - bool has_second_in = store_op < 0 && src >= 0; - - int64_t offset = 0; - if (n->type == TB_SYMBOL) { - TB_Symbol* sym = TB_NODE_GET_EXTRA_T(n, TB_NodeSymbol)->sym; - assert(sym->tag != 0); - - Inst* i = alloc_inst(LEA, TB_TYPE_PTR, 1, 1+has_second_in, 0); - i->flags = INST_GLOBAL; - if (store_op < 0) { - i->operands[0] = dst; - if (has_second_in) { - i->mem_slot = 2; - i->operands[1] = src; - i->operands[2] = RSP; - } else { - i->mem_slot = 1; - i->operands[1] = RSP; - } - } else { - i->type = store_op; - i->mem_slot = 0; - i->operands[0] = RSP; - i->operands[1] = src; - } - i->s = sym; - return i; - } else if (n->type == TB_VA_START) { - assert(ctx->module->target_abi == TB_ABI_WIN64 && "How does va_start even work on SysV?"); - - // on Win64 va_start just means whatever is one parameter away from - // the parameter you give it (plus in Win64 the parameters in the stack - // are 8bytes, no fanciness like in SysV): - // void printf(const char* fmt, ...) { - // va_list args; - // va_start(args, fmt); // args = ((char*) &fmt) + 8; - // ... - // } - offset = 8; - - use(ctx, n); - n = n->inputs[1]; - } else if (n->type == TB_MEMBER_ACCESS) { - offset = TB_NODE_GET_EXTRA_T(n, TB_NodeMember)->offset; - - use(ctx, n); - n = n->inputs[1]; - } - - Scale scale = SCALE_X1; - int index = -1; - - if (n->type == TB_ARRAY_ACCESS) { - TB_Node* base = n->inputs[1]; - int64_t stride = TB_NODE_GET_EXTRA_T(n, TB_NodeArray)->stride; - - use(ctx, n); - n = n->inputs[2]; - - int32_t x; - if (n->type == TB_SHL && try_for_imm32(ctx, 64, n->inputs[2], &x)) { - use(ctx, n); - use(ctx, n->inputs[2]); - - n = n->inputs[1]; - stride *= (1ull << x); - } - - index = input_reg(ctx, n); - int addr = index; - - // compute index - if (stride == 1) { - // no scaling required - } else if (tb_is_power_of_two(stride)) { - scale = tb_ffs(stride) - 1; - - if (scale > 3) { - addr = DEF(NULL, TB_TYPE_I64); - - // we can't fit this into an LEA, might as well just do a shift - SUBMIT(inst_op_rri(SHL, TB_TYPE_I64, addr, index, scale)); - index = addr, scale = 0; - } - } else { - // needs a proper multiply (we may wanna invest in a few special patterns - // for reducing simple multiplies into shifts) - // - // a * 24 => (a * 8) * 3 - // b * 3 => b<<1 + b - // - // thus - // - // LEA b, [a * 8] - // LEA dst, [b * 2 + b] - addr = DEF(NULL, TB_TYPE_I64); - SUBMIT(inst_op_rri(IMUL, TB_TYPE_I64, addr, index, stride)); - index = addr; - } - - n = base; - } - - int base; - if (n->type == TB_LOCAL) { - use(ctx, n); - offset += get_stack_slot(ctx, n); - base = RBP; - } else { - base = input_reg(ctx, n); - } - - // compute base - if (store_op < 0) { - if (has_second_in) { - return inst_op_rrm(LEA, n->dt, dst, src, base, index, scale, offset); - } else { - return inst_op_rm(LEA, n->dt, dst, base, index, scale, offset); - } - } else { - return inst_op_mr(store_op, n->dt, base, index, scale, offset, src); - } -} - -static Inst* isel_addr2(Ctx* restrict ctx, TB_Node* n, int dst, int store_op, int src) { - // compute base - if (n->type == TB_ARRAY_ACCESS && (ctx->values[n->gvn].uses > 2 || ctx->values[n->gvn].vreg >= 0)) { - int base = input_reg(ctx, n); - if (store_op < 0) { - if (src >= 0) { - return inst_op_rrm(LEA, TB_TYPE_PTR, dst, src, base, -1, SCALE_X1, 0); - } else { - return inst_op_rm(LEA, TB_TYPE_PTR, dst, base, -1, SCALE_X1, 0); - } - } else { - return inst_op_mr(store_op, TB_TYPE_PTR, base, -1, SCALE_X1, 0, src); - } - } else { - return isel_addr(ctx, n, dst, store_op, src); - } -} - -static Cond isel_cmp(Ctx* restrict ctx, TB_Node* n) { - bool invert = false; - if (n->type == TB_CMP_EQ && n->dt.type == TB_INT && n->dt.data == 1 && n->inputs[2]->type == TB_INTEGER_CONST) { - TB_NodeInt* b = TB_NODE_GET_EXTRA_T(n->inputs[2], TB_NodeInt); - if (b->value == 0) { - invert = true; - n = n->inputs[1]; - } - } - - if (n->type >= TB_CMP_EQ && n->type <= TB_CMP_FLE) { - TB_DataType cmp_dt = TB_NODE_GET_EXTRA_T(n, TB_NodeCompare)->cmp_dt; - - Cond cc = -1; - use(ctx, n); - - if (TB_IS_FLOAT_TYPE(cmp_dt)) { - int lhs = input_reg(ctx, n->inputs[1]); - int rhs = input_reg(ctx, n->inputs[2]); - SUBMIT(inst_op_rr_no_dst(FP_UCOMI, cmp_dt, lhs, rhs)); - - switch (n->type) { - case TB_CMP_EQ: cc = E; break; - case TB_CMP_NE: cc = NE; break; - case TB_CMP_FLT: cc = B; break; - case TB_CMP_FLE: cc = BE; break; - default: tb_unreachable(); - } - } else { - bool invert = false; - int32_t x; - int lhs = input_reg(ctx, n->inputs[1]); - if (try_for_imm32(ctx, cmp_dt.type == TB_PTR ? 64 : cmp_dt.data, n->inputs[2], &x)) { - use(ctx, n->inputs[2]); - - if (x == 0 && (n->type == TB_CMP_EQ || n->type == TB_CMP_NE)) { - SUBMIT(inst_op_rr_no_dst(TEST, cmp_dt, lhs, lhs)); - } else { - SUBMIT(inst_op_ri(CMP, cmp_dt, lhs, x)); - } - } else if (n->inputs[2]->type == TB_LOAD && on_last_use(ctx, n->inputs[2])) { - use(ctx, n->inputs[2]); - - Inst* inst = isel_addr2(ctx, n->inputs[2]->inputs[2], lhs, -1, lhs); - inst->type = CMP; - inst->dt = legalize(cmp_dt); - SUBMIT(inst); - } else { - int rhs = input_reg(ctx, n->inputs[2]); - SUBMIT(inst_op_rr_no_dst(CMP, cmp_dt, lhs, rhs)); - } - - switch (n->type) { - case TB_CMP_EQ: cc = E; break; - case TB_CMP_NE: cc = NE; break; - case TB_CMP_SLT: cc = invert ? G : L; break; - case TB_CMP_SLE: cc = invert ? GE : LE; break; - case TB_CMP_ULT: cc = invert ? A : B; break; - case TB_CMP_ULE: cc = invert ? NB : BE; break; - default: tb_unreachable(); - } - } - - return cc ^ invert; - } else { - int src = input_reg(ctx, n); - - TB_DataType dt = n->dt; - if (TB_IS_FLOAT_TYPE(dt)) { - int tmp = DEF(n, n->dt); - SUBMIT(inst_op_zero(dt, tmp)); - SUBMIT(inst_op_rr_no_dst(FP_UCOMI, dt, src, tmp)); - } else { - SUBMIT(inst_op_rr_no_dst(TEST, dt, src, src)); - } - return NE ^ invert; - } -} - -static bool should_rematerialize(TB_Node* n) { - if ((n->type == TB_INT2FLOAT || n->type == TB_INT2PTR) && n->inputs[1]->type == TB_INTEGER_CONST) { - return true; - } - - return (n->type == TB_PROJ && (n->dt.type == TB_CONT || n->inputs[0]->type == TB_START)) || - n->type == TB_FLOAT32_CONST || n->type == TB_FLOAT64_CONST || - n->type == TB_INTEGER_CONST || n->type == TB_MEMBER_ACCESS || - n->type == TB_LOCAL || n->type == TB_SYMBOL; -} - -static void isel(Ctx* restrict ctx, TB_Node* n, const int dst) { - TB_NodeTypeEnum type = n->type; - switch (type) { - case TB_PHI: break; - case TB_REGION: break; - - case TB_POISON: { - Inst* inst = alloc_inst(INST_INLINE, TB_TYPE_VOID, 1, 0, 0); - inst->operands[0] = dst; - append_inst(ctx, inst); - break; - } - - case TB_START: { - TB_NodeRegion* start = TB_NODE_GET_EXTRA(n); - const TB_FunctionPrototype* restrict proto = ctx->f->prototype; - bool is_sysv = (ctx->target_abi == TB_ABI_SYSTEMV); - - const GPR* gpr_params = is_sysv ? SYSV_GPR_PARAMETERS : WIN64_GPR_PARAMETERS; - size_t gpr_param_count = is_sysv ? COUNTOF(SYSV_GPR_PARAMETERS) : COUNTOF(WIN64_GPR_PARAMETERS); - int xmm_param_count = is_sysv ? 8 : 4; - - Inst* prev = ctx->head; - - int out_count = 0; - RegIndex outs[16]; - - // handle known parameters - int used_gpr = 0, used_xmm = 0; - TB_Node** params = ctx->f->params; - FOREACH_N(i, 0, ctx->f->param_count) { - TB_Node* proj = params[3 + i]; - bool is_float = proj->dt.type == TB_FLOAT; - - // copy from parameter - int reg_class = (is_float ? REG_CLASS_XMM : REG_CLASS_GPR); - int id = is_float ? used_xmm : used_gpr; - if (is_sysv) { - if (is_float) used_xmm += 1; - else used_gpr += 1; - } else { - // win64 will expend the slot regardless of if it's used - used_gpr += 1; - used_xmm += 1; - } - - int reg_limit = is_float ? xmm_param_count : gpr_param_count; - if (id < reg_limit) { - ValueDesc* v = lookup_val(ctx, proj); - if (v != NULL) { - assert(v->vreg < 0 && "shouldn't have been initialized yet?"); - v->vreg = DEF(proj, proj->dt); - - int reg_num = is_float ? id : gpr_params[id]; - int vreg = (is_float ? FIRST_XMM : 0) + reg_num; - - hint_reg(ctx, v->vreg, vreg); - SUBMIT(inst_move(proj->dt, v->vreg, vreg)); - - outs[out_count++] = vreg; - } - } - } - - // insert INST_ENTRY (this is where parameter register come from) - Inst* entry_inst = alloc_inst(INST_ENTRY, TB_TYPE_I64, out_count, 0, 0); - memcpy(entry_inst->operands, outs, out_count * sizeof(RegIndex)); - - entry_inst->next = prev->next; - if (prev->next == NULL) { - ctx->head = entry_inst; - } - prev->next = entry_inst; - - // walk the entry to find any parameter stack slots - bool has_param_slots = false; - FOREACH_N(i, 0, ctx->f->param_count) { - TB_Node* proj = params[3 + i]; - User* use = find_users(ctx->p, proj); - if (use == NULL || use->next != NULL || use->slot == 0) { - continue; - } - - TB_Node* store_op = use->n; - if (store_op->type != TB_STORE || tb_get_parent_region(store_op->inputs[0]) != n) { - continue; - } - - TB_Node* addr = store_op->inputs[2]; - if (addr->type != TB_LOCAL) { - continue; - } - - int pos = 16 + (i * 8); - nl_map_put(ctx->stack_slots, addr, pos); - - if (i >= 4 && ctx->target_abi == TB_ABI_WIN64) { - // marks as visited (stores don't return so we can -1) - put_val(ctx, store_op, 0); - } - - // add parameter to debug info - add_debug_local(ctx, addr, pos); - has_param_slots = true; - } - - if (has_param_slots) { - ctx->stack_usage += 16 + (proto->param_count * 8); - } else { - ctx->stack_usage += 16; - } - - // Handle unknown parameters (if we have varargs) - if (proto->has_varargs) { - const GPR* parameter_gprs = is_sysv ? SYSV_GPR_PARAMETERS : WIN64_GPR_PARAMETERS; - - // spill the rest of the parameters (assumes they're all in the GPRs) - size_t gpr_count = is_sysv ? 6 : 4; - size_t extra_param_count = proto->param_count > gpr_count ? 0 : gpr_count - proto->param_count; - - FOREACH_N(i, 0, extra_param_count) { - size_t param_num = proto->param_count + i; - - int dst_pos = 16 + (param_num * 8); - GPR src = parameter_gprs[param_num]; - - SUBMIT(inst_op_mr(MOV, TB_TYPE_I64, RBP, GPR_NONE, SCALE_X1, dst_pos, src)); - } - - ctx->stack_usage += (extra_param_count * 8); - } - break; - } - - case TB_INTEGER_CONST: { - uint64_t x = TB_NODE_GET_EXTRA_T(n, TB_NodeInt)->value; - - // mask off bits - uint64_t bits_in_type = n->dt.type == TB_PTR ? 64 : n->dt.data; - if (bits_in_type < 64) { - x &= (1ull << bits_in_type) - 1; - } - - if (x == 0) { - SUBMIT(inst_op_zero(n->dt, dst)); - } else if ((x >> 32ull) == UINT32_MAX) { - // mov but zero ext - SUBMIT(inst_op_imm(MOV, TB_TYPE_I32, dst, x)); - } else if (bits_in_type <= 32 || (x >> 31ull) == 0) { - SUBMIT(inst_op_imm(MOV, n->dt, dst, x)); - } else { - // movabs reg, imm64 - SUBMIT(inst_op_abs(MOVABS, n->dt, dst, x)); - } - break; - } - - case TB_SELECT: { - assert(n->dt.type != TB_FLOAT); - int lhs = input_reg(ctx, n->inputs[2]); - int rhs = input_reg(ctx, n->inputs[3]); - - Cond cc = isel_cmp(ctx, n->inputs[1]); - SUBMIT(inst_move(n->dt, dst, rhs)); - SUBMIT(inst_op_rr(CMOVO + cc, n->dt, dst, lhs)); - break; - } - - case TB_AND: - case TB_OR: - case TB_XOR: - case TB_ADD: - case TB_SUB: { - const static InstType ops[] = { AND, OR, XOR, ADD, SUB }; - InstType op = ops[type - TB_AND]; - - int lhs = input_reg(ctx, n->inputs[1]); - hint_reg(ctx, dst, lhs); - - int32_t x; - if (n->inputs[2]->type == TB_LOAD && on_last_use(ctx, n->inputs[2])) { - use(ctx, n->inputs[2]); - - SUBMIT(inst_move(n->dt, dst, lhs)); - - Inst* inst = isel_addr2(ctx, n->inputs[2]->inputs[2], dst, -1, dst); - inst->type = op; - inst->dt = legalize(n->dt); - SUBMIT(inst); - } else if (try_for_imm32(ctx, n->dt.data, n->inputs[2], &x)) { - use(ctx, n->inputs[2]); - - if (0 && type == TB_ADD) { - SUBMIT(inst_op_rm(LEA, TB_TYPE_I64, dst, lhs, -1, SCALE_X1, x)); - } else { - SUBMIT(inst_move(n->dt, dst, lhs)); - SUBMIT(inst_op_rri(op, n->dt, dst, dst, x)); - } - } else { - int rhs = input_reg(ctx, n->inputs[2]); - - SUBMIT(inst_move(n->dt, dst, lhs)); - SUBMIT(inst_op_rrr(op, n->dt, dst, dst, rhs)); - } - break; - } - - case TB_MUL: { - int lhs = input_reg(ctx, n->inputs[1]); - hint_reg(ctx, dst, lhs); - - // promote any <16bit multiplies up a bit: - // should be fair game to compute the multiply - // with garbage bits at the top as long as we - // don't read them. - TB_DataType dt = n->dt; - assert(dt.type == TB_INT); - if (dt.data < 16) { - dt.data = 16; - } - - int32_t x; - if (try_for_imm32(ctx, dt.data, n->inputs[2], &x)) { - use(ctx, n->inputs[2]); - - SUBMIT(inst_move(dt, dst, lhs)); - SUBMIT(inst_op_rri(IMUL, dt, dst, dst, x)); - } else { - int rhs = input_reg(ctx, n->inputs[2]); - - SUBMIT(inst_move(dt, dst, lhs)); - SUBMIT(inst_op_rrr(IMUL, dt, dst, dst, rhs)); - } - break; - } - case TB_MULPAIR: { - // returns into both lo and hi - TB_Node* projs[2] = { 0 }; - for (User* u = n->users; u; u = u->next) { - if (u->n->type == TB_PROJ) { - int index = TB_NODE_GET_EXTRA_T(u->n, TB_NodeProj)->index; - projs[index] = u->n; - } - } - - // at least one needs to be alive - assert(projs[0] != NULL || projs[1] != NULL); - - TB_DataType dt = projs[0] ? projs[0]->dt : projs[1]->dt; - { - Inst* inst = alloc_inst(MUL, dt, 2, 2, 0); - inst->operands[0] = RAX; - inst->operands[1] = RDX; - inst->operands[2] = RAX; - inst->operands[3] = RDX; - SUBMIT(inst); - } - - if (projs[0]) SUBMIT(inst_move(dt, input_reg(ctx, projs[0]), RAX)); - if (projs[1]) SUBMIT(inst_move(dt, input_reg(ctx, projs[1]), RDX)); - break; - } - - // bit magic - case TB_CTZ: - case TB_CLZ: { - int op = type == TB_CLZ ? BSR : BSF; - int lhs = input_reg(ctx, n->inputs[1]); - hint_reg(ctx, dst, lhs); - - // we only wanna deal with 32 or 64 ops for - // this (16 is annoying and 8 is unavailable) - TB_DataType dt = n->dt; - if (dt.data < 64) { - // make sure the bits are zero'd above - if (dt.data < 32) { - assert(type == TB_CLZ && "clz is different, and im stupid"); - SUBMIT(inst_op_zero(TB_TYPE_I32, dst)); - } - - dt.data = 32; - } - - Inst* inst = inst_op_rr(op, dt, dst, lhs); - if (type == TB_CLZ) { - // the difference between bsf and tzcnt - inst->flags |= INST_REP; - } - SUBMIT(inst); - - // flip bits to make CLZ instead of bitscanreverse - SUBMIT(inst_op_rri(XOR, dt, dst, dst, 63)); - break; - } - - // bit shifts - case TB_SHL: - case TB_SHR: - case TB_SAR: - case TB_ROL: - case TB_ROR: { - const static InstType ops[] = { SHL, SHR, SAR, ROL, ROR }; - InstType op = ops[type - TB_SHL]; - - int lhs = input_reg(ctx, n->inputs[1]); - hint_reg(ctx, dst, lhs); - - int32_t x; - if (try_for_imm32(ctx, n->inputs[2]->dt.data, n->inputs[2], &x) && x >= 0 && x < 64) { - use(ctx, n->inputs[2]); - SUBMIT(inst_move(n->dt, dst, lhs)); - SUBMIT(inst_op_rri(op, n->dt, dst, dst, x)); - break; - } - - // the shift operations need their right hand side in CL (RCX's low 8bit) - int rhs = input_reg(ctx, n->inputs[2]); - - SUBMIT(inst_move(n->dt, dst, lhs)); - SUBMIT(inst_move(n->dt, RCX, rhs)); - SUBMIT(inst_op_rrr_tmp(op, n->dt, dst, dst, RCX, RCX)); - break; - } - - case TB_UDIV: - case TB_SDIV: - case TB_UMOD: - case TB_SMOD: { - bool is_signed = (type == TB_SDIV || type == TB_SMOD); - bool is_div = (type == TB_UDIV || type == TB_SDIV); - - TB_DataType dt = n->dt; - assert(dt.type == TB_INT); - - int op = MOV; - if (dt.data <= 8) op = is_signed ? MOVSXB : MOVZXB; - else if (dt.data <= 16) op = is_signed ? MOVSXW : MOVZXW; - - // division is scaled up to 32bit - if (dt.data < 32) dt.data = 32; - - // mov rax, lhs - int lhs = input_reg(ctx, n->inputs[1]); - SUBMIT(inst_op_rr(op, dt, RAX, lhs)); - - int rhs = input_reg(ctx, n->inputs[2]); - if (n->dt.data < 32) { - // add cast - int new_rhs = DEF(n->inputs[2], TB_TYPE_I32); - SUBMIT(inst_op_rr(op, TB_TYPE_I32, new_rhs, rhs)); - rhs = new_rhs; - } - - // if signed: - // cqo/cdq (sign extend RAX into RDX) - // else: - // xor rdx, rdx - if (is_signed) { - Inst* inst = alloc_inst(CAST, dt, 1, 1, 0); - inst->operands[0] = RDX; - inst->operands[1] = RAX; - SUBMIT(inst); - } else { - SUBMIT(inst_op_zero(dt, RDX)); - } - - { - Inst* inst = alloc_inst(is_signed ? IDIV : DIV, dt, 2, 3, 0); - inst->operands[0] = is_div ? RAX : RDX; - inst->operands[1] = is_div ? RDX : RAX; - inst->operands[2] = rhs; - inst->operands[3] = RDX; - inst->operands[4] = RAX; - SUBMIT(inst); - } - - hint_reg(ctx, dst, is_div ? RAX : RDX); - SUBMIT(inst_move(dt, dst, is_div ? RAX : RDX)); - break; - } - - case TB_FLOAT32_CONST: { - assert(n->dt.type == TB_FLOAT); - uint32_t imm = (Cvt_F32U32) { .f = TB_NODE_GET_EXTRA_T(n, TB_NodeFloat32)->value }.i; - - if (imm == 0) { - SUBMIT(inst_op_zero(n->dt, dst)); - } else { - TB_Global* g = tb__small_data_intern(ctx->module, sizeof(float), &imm); - SUBMIT(inst_op_global(FP_MOV, n->dt, dst, (TB_Symbol*) g)); - } - break; - } - case TB_FLOAT64_CONST: { - assert(n->dt.type == TB_FLOAT); - uint64_t imm = (Cvt_F64U64){ .f = TB_NODE_GET_EXTRA_T(n, TB_NodeFloat64)->value }.i; - - if (imm == 0) { - SUBMIT(inst_op_zero(n->dt, dst)); - } else { - TB_Global* g = tb__small_data_intern(ctx->module, sizeof(double), &imm); - SUBMIT(inst_op_global(FP_MOV, n->dt, dst, (TB_Symbol*) g)); - } - break; - } - case TB_FLOAT_EXT: { - int src = input_reg(ctx, n->inputs[1]); - SUBMIT(inst_op_rr(FP_CVT, n->inputs[1]->dt, dst, src)); - break; - } - case TB_NEG: - case TB_NOT: { - if (n->dt.type != TB_FLOAT) { - int src = input_reg(ctx, n->inputs[1]); - - SUBMIT(inst_move(n->dt, dst, src)); - SUBMIT(inst_op_rr(type == TB_NOT ? NOT : NEG, n->dt, dst, dst)); - } else { - if (type == TB_NEG) { - TB_Global* g = NULL; - if (n->dt.data == TB_FLT_32) { - uint32_t buffer[4] = { 1u << 31u, 1u << 31u, 1u << 31u, 1u << 31u }; - g = tb__small_data_intern(ctx->module, 16, buffer); - } else if (n->dt.data == TB_FLT_64) { - uint64_t buffer[4] = { 1ull << 63ull, 1ull << 63ull }; - g = tb__small_data_intern(ctx->module, 16, buffer); - } else { - tb_todo(); - } - - SUBMIT(inst_op_global(FP_XOR, n->dt, dst, (TB_Symbol*) g)); - } else { - tb_todo(); - } - } - break; - } - case TB_FADD: - case TB_FSUB: - case TB_FMUL: - case TB_FDIV: - case TB_FMAX: - case TB_FMIN: { - const static InstType ops[] = { FP_ADD, FP_SUB, FP_MUL, FP_DIV, FP_MAX, FP_MIN }; - - int lhs = input_reg(ctx, n->inputs[1]); - hint_reg(ctx, dst, lhs); - SUBMIT(inst_move(n->dt, dst, lhs)); - - if (n->inputs[2]->type == TB_LOAD && on_last_use(ctx, n->inputs[2])) { - use(ctx, n->inputs[2]); - - Inst* inst = isel_addr2(ctx, n->inputs[2]->inputs[2], dst, -1, dst); - inst->type = ops[type - TB_FADD]; - inst->dt = legalize(n->dt); - SUBMIT(inst); - } else { - int rhs = input_reg(ctx, n->inputs[2]); - SUBMIT(inst_op_rrr(ops[type - TB_FADD], n->dt, dst, dst, rhs)); - } - break; - } - case TB_UINT2FLOAT: - case TB_INT2FLOAT: { - TB_DataType src_dt = n->inputs[1]->dt; - assert(src_dt.type == TB_INT); - - // it's either 32bit or 64bit conversion - // CVTSI2SS r/m32, xmm1 - // CVTSI2SD r/m64, xmm1 - bool is_64bit = src_dt.data > 32; - - int src = input_reg(ctx, n->inputs[1]); - SUBMIT(inst_op_rr(is_64bit ? FP_CVT64 : FP_CVT32, n->dt, dst, src)); - break; - } - - case TB_FLOAT2INT: - case TB_FLOAT2UINT: { - TB_DataType src_dt = n->inputs[1]->dt; - assert(src_dt.type == TB_FLOAT); - - // it's either 32bit or 64bit conversion - // F3 0F 2C /r CVTTSS2SI xmm1, r/m32 - // F3 REX.W 0F 2C /r CVTTSS2SI xmm1, r/m64 - // F2 0F 2C /r CVTTSD2SI xmm1, r/m32 - // F2 REX.W 0F 2C /r CVTTSD2SI xmm1, r/m64 - int src = input_reg(ctx, n->inputs[1]); - SUBMIT(inst_op_rr(FP_CVTT, src_dt, dst, src)); - break; - } - - // pointer arithmatic - case TB_LOCAL: - case TB_VA_START: - case TB_MEMBER_ACCESS: - case TB_ARRAY_ACCESS: { - SUBMIT(isel_addr(ctx, n, dst, -1, -1)); - break; - } - - // bitcasting - case TB_BITCAST: { - TB_DataType src_dt = n->inputs[1]->dt; - int src = input_reg(ctx, n->inputs[1]); - - if (src_dt.type == TB_FLOAT && n->dt.type == TB_INT) { - // float -> int - SUBMIT(inst_op_rr(MOV_F2I, n->dt, dst, src)); - } else if (src_dt.type == TB_INT && n->dt.type == TB_FLOAT) { - // int -> float - SUBMIT(inst_op_rr(MOV_I2F, src_dt, dst, src)); - } else { - SUBMIT(inst_move(n->dt, dst, src)); - } - break; - } - - // downcasting - case TB_PTR2INT: - case TB_TRUNCATE: { - int src = input_reg(ctx, n->inputs[1]); - - if (n->dt.type == TB_FLOAT) { - SUBMIT(inst_op_rr(FP_CVT, n->inputs[1]->dt, dst, src)); - } else { - SUBMIT(inst_move(n->dt, dst, src)); - } - break; - } - - // upcasting - case TB_INT2PTR: - case TB_SIGN_EXT: - case TB_ZERO_EXT: { - TB_Node* src = n->inputs[1]; - - TB_DataType src_dt = src->dt; - bool sign_ext = (type == TB_SIGN_EXT); - int bits_in_type = src_dt.type == TB_PTR ? 64 : src_dt.data; - - int32_t imm; - if (try_for_imm32(ctx, bits_in_type, src, &imm)) { - #define MASK_UPTO(pos) (~UINT64_C(0) >> (64 - pos)) - use(ctx, src); - - uint64_t src = imm; - uint64_t sign_bit = (src >> (bits_in_type - 1)) & 1; - uint64_t mask = MASK_UPTO(64) & ~MASK_UPTO(bits_in_type); - - src = (src & ~mask) | (sign_bit ? mask : 0); - if (!fits_into_int32(src)) { - // movabs reg, imm64 - SUBMIT(inst_op_abs(MOVABS, n->dt, dst, src)); - } else { - SUBMIT(inst_op_imm(MOV, n->dt, dst, src)); - } - - #undef MASK_UPTO - } else { - TB_DataType dt = n->dt; - - int op = MOV; - if (bits_in_type <= 8) op = sign_ext ? MOVSXB : MOVZXB; - else if (bits_in_type <= 16) op = sign_ext ? MOVSXW : MOVZXW; - else if (bits_in_type <= 32) { - if (sign_ext) op = MOVSXD; - else dt = src_dt; - } else if (bits_in_type <= 64) op = MOV; - else tb_todo(); - - int val = input_reg(ctx, src); - SUBMIT(inst_op_rr(op, dt, dst, val)); - } - break; - } - - case TB_TAILCALL: - case TB_SYSCALL: - case TB_CALL: { - static int ret_gprs[2] = { RAX, RDX }; - - bool is_sysv = (ctx->target_abi == TB_ABI_SYSTEMV); - const struct ParamDescriptor* restrict desc = ¶m_descs[is_sysv ? 1 : 0]; - if (type == TB_SYSCALL) { - desc = ¶m_descs[2]; - } - - uint32_t caller_saved_gprs = desc->caller_saved_gprs; - uint32_t caller_saved_xmms = ~0ull >> (64 - desc->caller_saved_xmms); - - TB_FunctionPrototype* proto = TB_NODE_GET_EXTRA_T(n, TB_NodeCall)->proto; - - TB_Node* ret_nodes[2] = { 0 }; - int rets[2] = { -1, -1 }; - int ret_count = 0; - int proj_base = type == TB_TAILCALL ? 3 : 2; - - if (n->type != TB_TAILCALL) { - assert(proto->return_count <= 2); - FOREACH_N(i, 0, proto->return_count) { - TB_Node* ret_node = TB_NODE_GET_EXTRA_T(n, TB_NodeCall)->projs[proj_base + i]; - if (!has_users(ctx, ret_node)) { - ret_node = NULL; - } - - if (ret_node != NULL) { - ret_nodes[i] = ret_node; - rets[i] = input_reg(ctx, ret_node); - ret_count++; - - bool use_xmm_ret = TB_IS_FLOAT_TYPE(ret_node->dt); - if (use_xmm_ret) { - caller_saved_xmms &= ~(1ull << (XMM0 + i)); - } else { - caller_saved_gprs &= ~(1ull << ret_gprs[i]); - } - } - } - } - - // system calls don't count, we track this for ABI - // and stack allocation purposes. - if (ctx->caller_usage < n->input_count - 3) { - ctx->caller_usage = n->input_count - 3; - } - - // parameter passing is separate from eval from regalloc reasons - size_t in_count = 0; - RegIndex ins[64]; - RegIndex param_srcs[64]; - - int vararg_cutoff = proto && proto->has_varargs ? proto->param_count : n->input_count-2; - - size_t xmms_used = 0, gprs_used = 0; - FOREACH_N(i, 3, n->input_count) { - TB_Node* param = n->inputs[i]; - TB_DataType param_dt = param->dt; - - bool use_xmm = TB_IS_FLOAT_TYPE(param_dt); - int reg = use_xmm ? xmms_used : gprs_used; - if (is_sysv) { - if (use_xmm) { - xmms_used++; - } else { - gprs_used++; - caller_saved_gprs &= ~(1u << gprs_used); - } - } else { - // win64 will always expend a register - xmms_used++; - gprs_used++; - } - - // first few parameters are passed as inputs to the CALL instruction. - // the rest are written into the stack at specific places. - RegIndex src = input_reg(ctx, param); - if (reg >= desc->gpr_count) { - SUBMIT(inst_op_mr(use_xmm ? FP_MOV : MOV, param->dt, RSP, GPR_NONE, SCALE_X1, reg * 8, src)); - } else { - int phys_reg = use_xmm ? reg : desc->gprs[reg]; - int dst = (use_xmm ? FIRST_XMM : FIRST_GPR) + phys_reg; - - hint_reg(ctx, src, dst); - - param_srcs[in_count] = src; - ins[in_count] = dst; - in_count += 1; - - if (use_xmm) { - caller_saved_xmms &= ~(1ull << phys_reg); - } else { - caller_saved_gprs &= ~(1ull << phys_reg); - } - } - } - - // perform last minute copies (this avoids keeping parameter registers alive for too long) - FOREACH_N(i, 0, in_count) { - TB_DataType dt = n->inputs[3 + i]->dt; - - bool use_xmm = TB_IS_FLOAT_TYPE(dt); - SUBMIT(inst_move(dt, ins[i], param_srcs[i])); - - // in win64, float params past the vararg cutoff are - // duplicated in their respective GPR slot - if (use_xmm && i >= vararg_cutoff && i < desc->gpr_count) { - int phys_reg = desc->gprs[i]; - SUBMIT(inst_op_rr(MOV_F2I, TB_TYPE_I64, phys_reg, ins[i])); - ins[in_count++] = FIRST_GPR + phys_reg; - } - } - - // compute the target (unless it's a symbol) before the - // registers all need to be forcibly shuffled - TB_Node* target = n->inputs[2]; - bool static_call = n->type == TB_CALL && target->type == TB_SYMBOL; - - int target_val = RSP; // placeholder really - if (!static_call) { - target_val = input_reg(ctx, target); - } - - if (type == TB_TAILCALL) { - hint_reg(ctx, target_val, RAX); - SUBMIT(inst_move(TB_TYPE_I64, RAX, target_val)); - } else if (type == TB_SYSCALL) { - ins[in_count++] = FIRST_GPR + RAX; - hint_reg(ctx, target_val, RAX); - SUBMIT(inst_move(target->dt, RAX, target_val)); - } else { - // the number of float parameters is written into AL - if (proto->has_varargs && is_sysv) { - SUBMIT(inst_op_imm(MOV, TB_TYPE_I8, RAX, xmms_used)); - ins[in_count++] = FIRST_GPR + RAX; - caller_saved_gprs &= ~(1ull << RAX); - } - } - - // all these registers need to be spilled and reloaded if they're used across - // the function call boundary... you might see why inlining could be nice to implement - size_t clobber_count = tb_popcount(caller_saved_gprs) + tb_popcount(caller_saved_xmms); - - int op = SYSCALL; - if (type == TB_CALL) op = CALL; - if (type == TB_TAILCALL) op = NOP; - - Inst* call_inst = alloc_inst(op, TB_TYPE_PTR, ret_count, 1 + in_count, clobber_count); - - // mark clobber list - { - RegIndex* clobbers = &call_inst->operands[call_inst->out_count + call_inst->in_count]; - FOREACH_N(i, 0, 16) if (caller_saved_gprs & (1u << i)) { - *clobbers++ = FIRST_GPR + i; - } - - FOREACH_N(i, 0, 16) if (caller_saved_xmms & (1u << i)) { - *clobbers++ = FIRST_XMM + i; - } - } - - // return value (either XMM0 or RAX) - RegIndex* dst_ins = call_inst->operands; - FOREACH_N(i, 0, 2) if (ret_nodes[i] != NULL) { - bool use_xmm_ret = TB_IS_FLOAT_TYPE(ret_nodes[i]->dt); - if (use_xmm_ret) { - *dst_ins++ = FIRST_XMM + i; - } else { - *dst_ins++ = ret_gprs[i]; - } - } - - // write inputs - if (static_call) { - call_inst->flags |= INST_GLOBAL; - call_inst->mem_slot = call_inst->out_count; - call_inst->s = TB_NODE_GET_EXTRA_T(target, TB_NodeSymbol)->sym; - } - - *dst_ins++ = target_val; - memcpy(dst_ins, ins, in_count * sizeof(RegIndex)); - - SUBMIT(call_inst); - - if (type == TB_TAILCALL) { - Inst* i = alloc_inst(INST_EPILOGUE, TB_TYPE_VOID, 0, 0, 0); - append_inst(ctx, i); - - SUBMIT(inst_jmp_reg(RAX)); - } else { - // copy out return - FOREACH_N(i, 0, 2) if (ret_nodes[i] != NULL) { - assert(rets[i] >= 0); - TB_DataType dt = ret_nodes[i]->dt; - bool use_xmm_ret = TB_IS_FLOAT_TYPE(dt); - if (use_xmm_ret) { - hint_reg(ctx, rets[i], FIRST_XMM + i); - SUBMIT(inst_move(dt, rets[i], FIRST_XMM + i)); - } else { - hint_reg(ctx, rets[i], ret_gprs[i]); - SUBMIT(inst_move(dt, rets[i], ret_gprs[i])); - } - } - } - break; - } - - case TB_CMP_EQ: - case TB_CMP_NE: - case TB_CMP_SLT: - case TB_CMP_SLE: - case TB_CMP_ULT: - case TB_CMP_ULE: - case TB_CMP_FLT: - case TB_CMP_FLE: { - // SUBMIT(inst_op_zero(n->dt, dst)); - - // use SETcc to convert into integer - Cond cc = isel_cmp(ctx, n); - SUBMIT(inst_op_r(SETO + cc, TB_TYPE_I8, dst)); - break; - } - - // table lookup constant -> constant - case TB_LOOKUP: { - TB_NodeLookup* l = TB_NODE_GET_EXTRA(n); - - TB_LookupEntry* min = &l->entries[1]; - TB_LookupEntry* max = &l->entries[l->entry_count - 1]; - - // "+ 2" because of the inclusive range + the default case - int64_t range = (max->key - min->key) + 2; - - double dist_avg = 0; - double inv_key_count = 1.0 / (l->entry_count - 1); - - int64_t last = l->entries[1].key; - FOREACH_N(i, 2, l->entry_count) { - int64_t key = l->entries[i].key; - - dist_avg += (key - last) * inv_key_count; - last = key; - } - - // we wanna figure out how many bits per table entry - assert(n->dt.type == TB_INT); - size_t bits = tb_next_pow2((n->dt.data + 7) / 8) * 8; - if (ctx->p->universe.arena != NULL) { - // lattice is the superior type, trust it over the - // TB_DataType, in this case if we see a 1bit value - // let's just do bitsets. - Lattice* l = lattice_universe_get(&ctx->p->universe, n); - if (l->tag == LATTICE_INT && l->_int.min == 0 && l->_int.max == 1) { - bits = 1; - } - } - - // in QWORDs - size_t table_size = ((range * bits) + 63) / 64; - - // flat table from start to finish (first element is the default) - TB_Function* f = ctx->f; - TB_Global* table_sym = tb_global_create(f->super.module, 0, NULL, NULL, TB_LINKAGE_PRIVATE); - tb_global_set_storage(f->super.module, tb_module_get_rdata(f->super.module), table_sym, table_size * sizeof(uint64_t), 8, 1); - uint64_t* table_data = tb_global_add_region(f->super.module, table_sym, 0, table_size * sizeof(uint64_t)); - - memset(table_data, 0, table_size * sizeof(uint64_t)); - - // encode every entry - int amt = tb_ffs(64 / bits) - 1; - FOREACH_N(i, 0, l->entry_count) { - uint64_t index = 0; - if (i > 0) { - index = (l->entries[i].key - min->key) + 1; - } - - uint64_t mask = (1ull << bits) - 1; - uint64_t shift = index & ((1ull << amt) - 1); - table_data[index >> amt] |= (l->entries[i].val & mask) << shift; - } - - TB_DataType dt = n->dt; - int key = input_reg(ctx, n->inputs[1]); - - int index = DEF(NULL, dt); - hint_reg(ctx, index, key); - SUBMIT(inst_move(dt, index, key)); - // Simple range check: - // if ((key - min) >= (max - min)) goto default - if (1) { - int zero = DEF(NULL, dt); - SUBMIT(inst_op_zero(dt, zero)); - - if (min != 0) { - SUBMIT(inst_op_rri(SUB, dt, index, index, min->key - 1)); - } - SUBMIT(inst_op_ri(CMP, dt, index, range)); - SUBMIT(inst_op_rr(CMOVA, n->dt, index, zero)); - } - // lea table, [rip + TABLE] - int table = DEF(NULL, TB_TYPE_I64); - SUBMIT(inst_op_global(LEA, TB_TYPE_I64, table, (TB_Symbol*) table_sym)); - - // word_index = key - int word_index = DEF(NULL, dt); - hint_reg(ctx, word_index, index); - SUBMIT(inst_move(dt, word_index, index)); - if (bits != 64) { - // word_index /= (64 / bits) - SUBMIT(inst_op_rri(SHR, TB_TYPE_I64, word_index, word_index, amt)); - } - // mov table, [table + index*8] - SUBMIT(inst_op_rm(MOV, TB_TYPE_I64, dst, table, word_index, SCALE_X8, 0)); - // we need to extract bits - if (bits != 64) { - // mov RCX, index - hint_reg(ctx, index, RCX); - SUBMIT(inst_move(dt, RCX, index)); - // and dst, amt_mask - uint64_t amt_mask = (1ull << amt) - 1; - if (amt_mask != 0x3F) { - SUBMIT(inst_op_rri(AND, TB_TYPE_I64, RCX, RCX, amt_mask)); - } - // shr dst, RCX - SUBMIT(inst_op_rrr(SHR, TB_TYPE_I64, dst, dst, RCX)); - // and dst, mask - uint64_t mask = (1ull << bits) - 1; - SUBMIT(inst_op_rri(AND, TB_TYPE_I64, dst, dst, mask)); - } - break; - } - - case TB_BRANCH: { - TB_Node* bb = tb_get_parent_region(n); - TB_NodeBranch* br = TB_NODE_GET_EXTRA(n); - - // the arena on the function should also be available at this time, we're - // in the TB_Passes - TB_Arena* arena = ctx->f->arena; - TB_ArenaSavepoint sp = tb_arena_save(arena); - int* succ = tb_arena_alloc(arena, br->succ_count * sizeof(int)); - - // fill successors - bool has_default = false; - for (User* u = n->users; u; u = u->next) { - if (u->n->type == TB_PROJ) { - int index = TB_NODE_GET_EXTRA_T(u->n, TB_NodeProj)->index; - TB_Node* succ_n = cfg_next_bb_after_cproj(u->n); - - if (index == 0) { - has_default = !cfg_is_unreachable(succ_n); - } - - succ[index] = nl_map_get_checked(ctx->cfg.node_to_block, succ_n).id; - } - } - - TB_DataType dt = n->inputs[1]->dt; - - SUBMIT(alloc_inst(INST_TERMINATOR, TB_TYPE_VOID, 0, 0, 0)); - - if (br->succ_count == 1) { - assert(0 && "degenerate branch? that's odd"); - } else if (br->succ_count == 2) { - int f = succ[1], t = succ[0]; - - Cond cc; - if (br->keys[0] == 0) { - cc = isel_cmp(ctx, n->inputs[1]); - } else { - int key = input_reg(ctx, n->inputs[1]); - SUBMIT(inst_op_ri(CMP, dt, key, br->keys[0])); - cc = NE; - } - - // if flipping avoids a jmp, do that - if (ctx->fallthrough == t) { - SUBMIT(inst_jcc(f, cc ^ 1)); - } else { - SUBMIT(inst_jcc(t, cc)); - if (ctx->fallthrough != f) { - SUBMIT(inst_jmp(f)); - } - } - } else { - int key = input_reg(ctx, n->inputs[1]); - - // check if there's at most only one space between entries - int64_t last = br->keys[0]; - int64_t min = last, max = last; - - double dist_avg = 0; - double inv_succ_count = 1.0 / (br->succ_count - 2); - - FOREACH_N(i, 2, br->succ_count) { - int64_t key = br->keys[i - 1]; - min = (min > key) ? key : min; - max = (max > key) ? max : key; - - dist_avg += (key - last) * inv_succ_count; - last = key; - } - - enum { - IF_ELSE_CHAIN, - JUMP_TABLE, - } r = IF_ELSE_CHAIN; - - int64_t range = (max - min) + 1; - if (range >= 4 && dist_avg < 2.0) { - r = JUMP_TABLE; - } - - switch (r) { - case IF_ELSE_CHAIN: { - // Basic if-else chain - FOREACH_N(i, 1, br->succ_count) { - uint64_t curr_key = br->keys[i-1]; - - if (fits_into_int32(curr_key)) { - SUBMIT(inst_op_ri(CMP, dt, key, curr_key)); - } else { - int tmp = DEF(n, dt); - SUBMIT(inst_op_abs(MOVABS, dt, tmp, curr_key)); - SUBMIT(inst_op_rr(CMP, dt, key, tmp)); - } - SUBMIT(inst_jcc(succ[i], E)); - } - SUBMIT(inst_jmp(succ[0])); - break; - } - - case JUMP_TABLE: { - // make a jump table with 4 byte relative pointers for each target - TB_Function* f = ctx->f; - TB_Global* jump_table = tb_global_create(f->super.module, -1, "jumptbl", NULL, TB_LINKAGE_PRIVATE); - tb_global_set_storage(f->super.module, tb_module_get_rdata(f->super.module), jump_table, range*4, 4, 1); - - // generate patches for later - uint32_t* jump_entries = tb_global_add_region(f->super.module, jump_table, 0, range*4); - - Set entries_set = set_create_in_arena(arena, range); - FOREACH_N(i, 1, br->succ_count) { - uint64_t key_idx = br->keys[i - 1] - min; - assert(key_idx < range); - - JumpTablePatch p; - p.pos = &jump_entries[key_idx]; - p.target = succ[i]; - dyn_array_put(ctx->jump_table_patches, p); - set_put(&entries_set, key_idx); - } - - // handle default cases - FOREACH_N(i, 0, range) { - if (!set_get(&entries_set, i)) { - JumpTablePatch p; - p.pos = &jump_entries[i]; - p.target = succ[0]; - dyn_array_put(ctx->jump_table_patches, p); - } - } - - int tmp = DEF(NULL, dt); - hint_reg(ctx, tmp, key); - if (dt.data >= 32) { - SUBMIT(inst_move(dt, tmp, key)); - } else if (dt.data == 16) { - dt = TB_TYPE_I32; - SUBMIT(inst_op_rr(MOVZXW, dt, tmp, key)); - } else if (dt.data == 8) { - dt = TB_TYPE_I32; - SUBMIT(inst_op_rr(MOVZXB, dt, tmp, key)); - } else { - dt = TB_TYPE_I32; - uint64_t mask = tb__mask(dt.data); - - SUBMIT(inst_move(dt, tmp, key)); - SUBMIT(inst_op_rri(AND, dt, tmp, tmp, mask)); - } - - // Simple range check: - // if ((key - min) >= (max - min)) goto default - if (has_default) { - if (min != 0) { - SUBMIT(inst_op_rri(SUB, dt, tmp, tmp, min)); - } - SUBMIT(inst_op_ri(CMP, dt, tmp, range)); - SUBMIT(inst_jcc(succ[0], NB)); - } - // lea target, [rip + f] - int target = DEF(NULL, TB_TYPE_I64); - SUBMIT(inst_op_global(LEA, TB_TYPE_I64, target, (TB_Symbol*) f)); - // lea table, [rip + JUMP_TABLE] - int table = DEF(NULL, TB_TYPE_I64); - SUBMIT(inst_op_global(LEA, TB_TYPE_I64, table, (TB_Symbol*) jump_table)); - // movsxd table, [table + key*4] - SUBMIT(inst_op_rm(MOVSXD, TB_TYPE_I64, table, table, tmp, SCALE_X4, 0)); - // add target, table - SUBMIT(inst_op_rrr(ADD, TB_TYPE_I64, target, target, table)); - // jmp target - SUBMIT(inst_jmp_reg(target)); - break; - } - } - } - - tb_arena_restore(arena, sp); - break; - } - - case TB_DEBUGBREAK: { - SUBMIT(alloc_inst(INT3, TB_TYPE_VOID, 0, 0, 0)); - break; - } - - case TB_UNREACHABLE: - case TB_TRAP: { - SUBMIT(alloc_inst(UD2, TB_TYPE_VOID, 0, 0, 0)); - break; - } - - case TB_SYMBOL: { - TB_NodeSymbol* s = TB_NODE_GET_EXTRA(n); - SUBMIT(inst_op_global(LEA, n->dt, dst, s->sym)); - break; - } - case TB_LOAD: - case TB_ATOMIC_LOAD: { - int mov_op = TB_IS_FLOAT_TYPE(n->dt) ? FP_MOV : MOV; - TB_Node* addr = n->inputs[2]; - - Inst* ld_inst = isel_addr2(ctx, addr, dst, -1, -1); - ld_inst->type = mov_op; - ld_inst->dt = legalize(n->dt); - if (n->type == TB_ATOMIC_LOAD) { - ld_inst->flags |= INST_LOCK; - } - - SUBMIT(ld_inst); - break; - } - case TB_SAFEPOINT_POLL: { - TB_Node* addr = n->inputs[2]; - - tb_todo(); - - // force uses of the inputs - /*FOREACH_N(i, 3, n->input_count) { - input_reg(ctx, n->inputs[i]); - }*/ - - // test tmp, dword [poll_site] - int tmp = DEF(n, TB_TYPE_I32); - Inst* ld_inst = isel_addr2(ctx, addr, tmp, -1, -1); - ld_inst->type = TEST; - ld_inst->dt = TB_X86_TYPE_DWORD; - SUBMIT(ld_inst); - break; - } - - case TB_STORE: { - if (dst >= 0) { - use(ctx, n->inputs[2]); - use(ctx, n->inputs[3]); - break; - } - - TB_DataType store_dt = n->inputs[3]->dt; - - // if we can couple the LOAD & STORE - TB_Node* addr = n->inputs[2]; - TB_Node* src = n->inputs[3]; - int store_op = can_folded_store(ctx, n->inputs[1], addr, n->inputs[3]); - if (store_op >= 0) { - use(ctx, src); - use(ctx, addr); - use(ctx, src->inputs[1]); - use(ctx, src->inputs[1]->inputs[1]); - - src = src->inputs[2]; - } else { - store_op = TB_IS_FLOAT_TYPE(store_dt) ? FP_MOV : MOV; - } - - int32_t imm; - if (try_for_imm32(ctx, src->dt.type == TB_PTR ? 64 : src->dt.data, src, &imm)) { - use(ctx, src); - - Inst* st_inst = isel_addr2(ctx, addr, dst, store_op, -1); - st_inst->in_count -= 1; - st_inst->dt = legalize(store_dt); - st_inst->flags |= INST_IMM; - st_inst->imm = imm; - assert(st_inst->flags & (INST_MEM | INST_GLOBAL)); - - SUBMIT(st_inst); - } else { - int src_reg = input_reg(ctx, src); - - Inst* st_inst = isel_addr2(ctx, addr, dst, store_op, src_reg); - st_inst->dt = legalize(store_dt); - assert(st_inst->flags & (INST_MEM | INST_GLOBAL)); - - SUBMIT(st_inst); - } - break; - } - case TB_MEMSET: { - TB_DataType ptr_dt = TB_TYPE_I64; - int rdi = input_reg(ctx, n->inputs[2]); - int rax = input_reg(ctx, n->inputs[3]); - int rcx = input_reg(ctx, n->inputs[4]); - - hint_reg(ctx, rdi, RDI); - hint_reg(ctx, rax, RAX); - hint_reg(ctx, rcx, RCX); - SUBMIT(inst_move(ptr_dt, RDI, rdi)); - SUBMIT(inst_move(TB_TYPE_I8, RAX, rax)); - SUBMIT(inst_move(ptr_dt, RCX, rcx)); - - Inst* i = alloc_inst(STOSB, TB_TYPE_VOID, 0, 3, 0); - i->flags |= INST_REP; - i->operands[0] = RDI; - i->operands[1] = RAX; - i->operands[2] = RCX; - SUBMIT(i); - break; - } - case TB_MEMCPY: { - TB_DataType ptr_dt = TB_TYPE_I64; - int rdi = input_reg(ctx, n->inputs[2]); - int rsi = input_reg(ctx, n->inputs[3]); - int rcx = input_reg(ctx, n->inputs[4]); - - hint_reg(ctx, rdi, RDI); - hint_reg(ctx, rsi, RSI); - hint_reg(ctx, rcx, RCX); - SUBMIT(inst_move(ptr_dt, RDI, rdi)); - SUBMIT(inst_move(ptr_dt, RSI, rsi)); - SUBMIT(inst_move(ptr_dt, RCX, rcx)); - - Inst* i = alloc_inst(MOVSB, TB_TYPE_VOID, 0, 3, 0); - i->flags |= INST_REP; - i->operands[0] = RDI; - i->operands[1] = RSI; - i->operands[2] = RCX; - SUBMIT(i); - break; - } - - case TB_END: { - assert(n->input_count <= 5 && "At most 2 return values :("); - static int ret_gprs[2] = { RAX, RDX }; - - int rets = n->input_count - 3; - FOREACH_N(i, 0, rets) { - int src = input_reg(ctx, n->inputs[3+i]); - - // copy to return register - TB_DataType dt = n->inputs[3+i]->dt; - if (dt.type == TB_FLOAT) { - hint_reg(ctx, src, FIRST_XMM + i); - SUBMIT(inst_move(dt, FIRST_XMM + i, src)); - } else { - hint_reg(ctx, src, ret_gprs[i]); - SUBMIT(inst_move(dt, ret_gprs[i], src)); - } - } - - // we don't really need a fence if we're about to exit but we do - // need to mark that it's the epilogue to tell regalloc where callee - // regs need to get restored. - Inst* i = alloc_inst(INST_EPILOGUE, TB_TYPE_VOID, 0, 0, 0); - i->flags |= INST_RET; - append_inst(ctx, i); - break; - } - - case TB_MACHINE_OP: { - TB_NodeMachineOp* mach = TB_NODE_GET_EXTRA(n); - size_t total = mach->outs + mach->ins + mach->tmps; - - if (total == 0) tb_todo(); - - Inst* inst = alloc_inst(INST_INLINE, TB_TYPE_VOID, mach->outs, mach->ins, mach->tmps); - memcpy(inst->operands, mach->regs, total * sizeof(TB_PhysicalReg)); - append_inst(ctx, inst); - break; - } - - // atomic RMW - case TB_ATOMIC_XCHG: - case TB_ATOMIC_ADD: - case TB_ATOMIC_SUB: - case TB_ATOMIC_AND: - case TB_ATOMIC_XOR: - case TB_ATOMIC_OR: { - // tbl is for normal locking operations which don't care about the result, - // fetch will need to worry about it which means slightly different codegen. - const static int tbl[] = { MOV, ADD, SUB, AND, XOR, OR }; - const static int fetch_tbl[] = { XCHG, XADD, XADD, 0, 0, 0 }; - - TB_NodeAtomic* a = TB_NODE_GET_EXTRA(n); - TB_DataType dt = a->proj1->dt; - bool dst_users = !has_users(ctx, a->proj1); - - int op = (dst_users ? fetch_tbl : tbl)[type - TB_ATOMIC_XCHG]; - if (op == 0) { - tb_todo(); // unsupported atomic op, we need to emulate it :( - } - - int src = input_reg(ctx, n->inputs[3]); - - // copy src into proj1 because the operation will be swapping SRC with - // the old value of the address. - int proj1 = input_reg(ctx, a->proj1); - hint_reg(ctx, proj1, src); - SUBMIT(inst_move(dt, proj1, src)); - - // ATOMIC-OP [addr], proj1 - Inst* st_inst = isel_addr2(ctx, n->inputs[2], -1, op, proj1); - st_inst->flags |= INST_LOCK; - st_inst->dt = legalize(dt); - assert(st_inst->flags & INST_MEM); - SUBMIT(st_inst); - break; - } - - case TB_PREFETCH: { - TB_NodePrefetch* p = TB_NODE_GET_EXTRA(n); - - // we might wanna fold these into each other but yea - int addr = DEF(n, TB_TYPE_I64); - SUBMIT(isel_addr(ctx, n->inputs[2], addr, -1, -1)); - - // prefetch op - Inst* i = alloc_inst(PREFETCHNTA + p->level, TB_TYPE_I32, 0, 1, 0); - i->flags = INST_MEM; - i->mem_slot = 0; - i->operands[1] = addr; - i->disp = 0; - i->scale = SCALE_X1; - SUBMIT(i); - break; - } - - case TB_CYCLE_COUNTER: { - // rdtsc - Inst* inst = alloc_inst(RDTSC, TB_TYPE_I64, 2, 0, 0); - inst->operands[0] = RDX; - inst->operands[1] = RAX; - SUBMIT(inst); - - // shl rdx, 32 - SUBMIT(inst_op_rri(SHL, TB_TYPE_I64, RDX, RDX, 32)); - // or rax, rdx - SUBMIT(inst_op_rrr(OR, TB_TYPE_I64, RAX, RAX, RDX)); - // mov dst, rax - hint_reg(ctx, dst, RAX); - SUBMIT(inst_move(TB_TYPE_I64, dst, RAX)); - break; - } - - case TB_PROJ: { - if (n->inputs[0]->type == TB_START) { - int index = TB_NODE_GET_EXTRA_T(n, TB_NodeProj)->index - 3; - int param_gpr_count = ctx->target_abi == TB_ABI_WIN64 ? 4 : 6; - - // past the first register parameters, it's all stack - if (index >= param_gpr_count) { - InstType i = n->dt.type == TB_FLOAT ? FP_MOV : MOV; - SUBMIT(inst_op_rm(i, n->dt, dst, RBP, GPR_NONE, SCALE_X1, 16 + index*8)); - } - } - break; - } - - default: tb_todo(); - } -} - -static void inst2_print(TB_CGEmitter* restrict e, InstType type, Val* dst, Val* src, TB_X86_DataType dt) { - if (dt >= TB_X86_TYPE_SSE_SS && dt <= TB_X86_TYPE_SSE_PD) { - inst2sse(e, type, dst, src, dt); - } else { - inst2(e, type, dst, src, dt); - } -} - -static int resolve_interval(Ctx* restrict ctx, Inst* inst, int i, Val* val) { - LiveInterval* interval = &ctx->intervals[inst->operands[i]]; - - if ((inst->flags & (INST_MEM | INST_GLOBAL)) && i == inst->mem_slot) { - tb_assert(!interval->is_spill, "cannot use spilled value for memory operand"); - if (inst->flags & INST_MEM) { - *val = (Val){ - .type = VAL_MEM, - .reg = interval->assigned, - .index = GPR_NONE, - .scale = inst->scale, - .imm = inst->disp, - }; - - if (inst->flags & INST_INDEXED) { - interval = &ctx->intervals[inst->operands[i + 1]]; - tb_assert(!interval->is_spill, "cannot use spilled value for memory operand"); - - val->index = interval->assigned; - return 2; - } else { - return 1; - } - } else { - *val = val_global(inst->s, inst->disp); - return 1; - } - } - - if (interval->is_spill) { - *val = (Val){ - .type = VAL_MEM, - .reg = RBP, - .index = GPR_NONE, - .imm = -interval->spill->pos, - }; - } else { - *val = (Val){ - .type = interval->reg_class == REG_CLASS_XMM ? VAL_XMM : VAL_GPR, - .reg = interval->assigned - }; - } - - return 1; -} - -static void emit_code(Ctx* restrict ctx, TB_FunctionOutput* restrict func_out) { - TB_CGEmitter* e = &ctx->emit; - - // resolve stack usage - { - size_t caller_usage = ctx->caller_usage; - if (ctx->target_abi == TB_ABI_WIN64 && caller_usage > 0 && caller_usage < 4) { - caller_usage = 4; - } - - size_t usage = ctx->stack_usage + (caller_usage * 8); - - // Align stack usage to 16bytes + 8 to accommodate for the RIP being pushed by CALL - ctx->stack_usage = align_up(usage, 16); - } - - // emit prologue - func_out->prologue_length = emit_prologue(ctx); - - Inst* prev_line = NULL; - for (Inst* restrict inst = ctx->first; inst; inst = inst->next) { - size_t in_base = inst->out_count; - size_t inst_table_size = sizeof(inst_table) / sizeof(*inst_table); - InstCategory cat = inst->type >= inst_table_size ? INST_BINOP : inst_table[inst->type].cat; - - if (0) { - printf(" \x1b[32m# %s t=%d { outs:", inst->type < inst_table_size ? inst_table[inst->type].mnemonic : "???", inst->time); - FOREACH_N(i, 0, inst->out_count) { - printf(" v%d", inst->operands[i]); - } - printf(", ins: "); - FOREACH_N(i, inst->out_count, inst->out_count + inst->in_count) { - printf(" v%d", inst->operands[i]); - } - printf("}\x1b[0m\n"); - } - - if (inst->type == INST_ENTRY || inst->type == INST_TERMINATOR) { - // does nothing - } else if (inst->type == INST_LABEL) { - TB_Node* bb = inst->n; - uint32_t pos = GET_CODE_POS(&ctx->emit); - - int id = nl_map_get_checked(ctx->cfg.node_to_block, bb).id; - tb_resolve_rel32(&ctx->emit, &ctx->emit.labels[id], pos); - } else if (inst->type == INST_INLINE) { - if (inst->n) { - TB_NodeMachineOp* mach = TB_NODE_GET_EXTRA(inst->n); - - void* dst = tb_cgemit_reserve(&ctx->emit, mach->length); - memcpy(dst, mach->data, mach->length); - tb_cgemit_commit(&ctx->emit, mach->length); - } - } else if (inst->type == INST_EPILOGUE) { - // just a marker for regalloc - emit_epilogue(ctx); - - if (inst->flags & INST_RET) { - // ret - EMIT1(&ctx->emit, 0xC3); - } - } else if (inst->type == INST_LINE) { - TB_Function* f = ctx->f; - TB_NodeSafepoint* loc = TB_NODE_GET_EXTRA(inst->n); - uint32_t pos = GET_CODE_POS(&ctx->emit); - - size_t top = dyn_array_length(ctx->locations); - if (prev_line == NULL || (prev_line->next != inst && !is_same_location(TB_NODE_GET_EXTRA(prev_line->n), loc))) { - TB_Location l = { - .file = loc->file, - .line = loc->line, - .column = loc->column, - .pos = pos - }; - dyn_array_put(ctx->locations, l); - prev_line = inst; - } - continue; - } else if (cat == INST_BYTE || cat == INST_BYTE_EXT) { - // we don't emit NOPs, just annotate for the regalloc - if (inst->type != NOP) { - if (inst->flags & INST_REP) EMIT1(e, 0xF3); - inst0(e, inst->type, inst->dt); - } - } else if (inst->type == INST_ZERO) { - Val dst; - resolve_interval(ctx, inst, 0, &dst); - - bool is_xmm = inst->dt >= TB_X86_TYPE_PBYTE && inst->dt <= TB_X86_TYPE_XMMWORD; - inst2_print(e, is_xmm ? FP_XOR : XOR, &dst, &dst, inst->dt); - } else if (inst->type >= JMP && inst->type <= JG) { - Val target; - if (inst->flags & INST_NODE) { - target = val_label(inst->l); - } else if (inst->flags & INST_GLOBAL) { - target = val_global(inst->s, inst->disp); - } else { - assert(inst->in_count == 1); - resolve_interval(ctx, inst, in_base, &target); - } - - inst1(e, inst->type, &target, inst->dt); - } else if (inst->type == CALL) { - Val target; - size_t i = resolve_interval(ctx, inst, in_base, &target); - inst1(e, CALL, &target, TB_X86_TYPE_QWORD); - } else { - int mov_op = inst->dt >= TB_X86_TYPE_PBYTE && inst->dt <= TB_X86_TYPE_XMMWORD ? FP_MOV : MOV; - - // prefix - if (inst->flags & INST_LOCK) { - EMIT1(e, 0xF0); - } - - if (inst->flags & INST_REP) { - EMIT1(e, 0xF3); - } - - TB_X86_DataType dt = inst->dt; - if (dt == TB_X86_TYPE_XMMWORD) { - dt = TB_X86_TYPE_SSE_PD; - } - - // resolve output - Val out; - int i = 0; - if (inst->out_count == 1) { - i += resolve_interval(ctx, inst, i, &out); - assert(i == in_base); - } else { - i = in_base; - } - - // first parameter - bool ternary = false; - if (inst->in_count > 0) { - Val lhs; - i += resolve_interval(ctx, inst, i, &lhs); - - ternary = (i < in_base + inst->in_count) || (inst->flags & (INST_IMM | INST_ABS)); - if (ternary && inst->type == IMUL && (inst->flags & INST_IMM)) { - // there's a special case for ternary IMUL r64, r/m64, imm32 - inst2(e, IMUL3, &out, &lhs, dt); - if (inst->dt == TB_X86_TYPE_WORD) { - EMIT2(e, inst->imm); - } else { - EMIT4(e, inst->imm); - } - continue; - } - - if (inst->out_count == 0) { - out = lhs; - } else if (inst->type == IDIV || inst->type == DIV) { - inst1(e, inst->type, &lhs, dt); - continue; - } else if (cat == INST_UNARY || cat == INST_UNARY_EXT) { - if (!is_value_match(&out, &lhs)) { - inst2_print(e, MOV, &out, &lhs, dt); - } - } else { - if (ternary || inst->type == MOV || inst->type == FP_MOV) { - if (!is_value_match(&out, &lhs)) { - inst2_print(e, mov_op, &out, &lhs, dt); - } - } else { - inst2_print(e, inst->type, &out, &lhs, dt); - } - } - } - - // unary ops - if (cat == INST_UNARY || cat == INST_UNARY_EXT) { - inst1(e, inst->type, &out, dt); - continue; - } - - if (inst->flags & INST_IMM) { - Val rhs = val_imm(inst->imm); - inst2_print(e, inst->type, &out, &rhs, dt); - } else if (inst->flags & INST_ABS) { - Val rhs = val_abs(inst->abs); - inst2_print(e, inst->type, &out, &rhs, dt); - } else if (ternary) { - Val rhs; - i += resolve_interval(ctx, inst, i, &rhs); - - if (inst->type != MOV || (inst->type == MOV && !is_value_match(&out, &rhs))) { - inst2_print(e, inst->type, &out, &rhs, dt); - } - } - } - } - - // pad to 16bytes - static const uint8_t nops[8][8] = { - { 0x90 }, - { 0x66, 0x90 }, - { 0x0F, 0x1F, 0x00 }, - { 0x0F, 0x1F, 0x40, 0x00 }, - { 0x0F, 0x1F, 0x44, 0x00, 0x00 }, - { 0x66, 0x0F, 0x1F, 0x44, 0x00, 0x00 }, - { 0x0F, 0x1F, 0x80, 0x00, 0x00, 0x00, 0x00 }, - { 0x0F, 0x1F, 0x84, 0x00, 0x00, 0x00, 0x00, 0x00 }, - }; - - size_t pad = 16 - (ctx->emit.count & 15); - if (pad < 16) { - uint8_t* dst = tb_cgemit_reserve(&ctx->emit, pad); - tb_cgemit_commit(&ctx->emit, pad); - - if (pad > 8) { - size_t rem = pad - 8; - memset(dst, 0x66, rem); - pad -= rem, dst += rem; - } - memcpy(dst, nops[pad - 1], pad); - } -} - -static void emit_win64eh_unwind_info(TB_Emitter* e, TB_FunctionOutput* out_f, uint64_t stack_usage) { - size_t patch_pos = e->count; - UnwindInfo unwind = { - .version = 1, - .flags = 0, // UNWIND_FLAG_EHANDLER, - .prolog_length = out_f->prologue_length, - .code_count = 0, - .frame_register = RBP, - .frame_offset = 0, - }; - tb_outs(e, sizeof(UnwindInfo), &unwind); - - size_t code_count = 0; - if (stack_usage > 0) { - UnwindCode codes[] = { - // sub rsp, stack_usage - { .code_offset = 8, .unwind_op = UNWIND_OP_ALLOC_SMALL, .op_info = (stack_usage / 8) - 1 }, - // mov rbp, rsp - { .code_offset = 4, .unwind_op = UNWIND_OP_SET_FPREG, .op_info = 0 }, - // push rbp - { .code_offset = 1, .unwind_op = UNWIND_OP_PUSH_NONVOL, .op_info = RBP }, - }; - tb_outs(e, sizeof(codes), codes); - code_count += 3; - } - - tb_patch1b(e, patch_pos + offsetof(UnwindInfo, code_count), code_count); -} - -static size_t emit_prologue(Ctx* restrict ctx) { - uint64_t stack_usage = ctx->stack_usage; - if (stack_usage <= 16) { - return 0; - } - - TB_CGEmitter* e = &ctx->emit; - - // push rbp - if (stack_usage > 0) { - EMIT1(e, 0x50 + RBP); - - // mov rbp, rsp - EMIT1(e, rex(true, RSP, RBP, 0)); - EMIT1(e, 0x89); - EMIT1(e, mod_rx_rm(MOD_DIRECT, RSP, RBP)); - } - - // if there's more than 4096 bytes of stack, we need to insert a chkstk - if (ctx->target_abi == TB_ABI_WIN64 && stack_usage >= 4096) { - assert(ctx->f->super.module->chkstk_extern); - Val sym = val_global(ctx->f->super.module->chkstk_extern, 0); - Val imm = val_imm(stack_usage); - Val rax = val_gpr(RAX); - Val rsp = val_gpr(RSP); - - inst2_print(e, MOV, &rax, &imm, TB_X86_TYPE_DWORD); - inst1(e, CALL, &sym, TB_X86_TYPE_QWORD); - inst2_print(e, SUB, &rsp, &rax, TB_X86_TYPE_QWORD); - } else if (stack_usage > 0) { - if (stack_usage == (int8_t)stack_usage) { - // sub rsp, stack_usage - EMIT1(e, rex(true, 0x00, RSP, 0)); - EMIT1(e, 0x83); - EMIT1(e, mod_rx_rm(MOD_DIRECT, 0x05, RSP)); - EMIT1(e, stack_usage); - } else { - // sub rsp, stack_usage - EMIT1(e, rex(true, 0x00, RSP, 0)); - EMIT1(e, 0x81); - EMIT1(e, mod_rx_rm(MOD_DIRECT, 0x05, RSP)); - EMIT4(e, stack_usage); - } - } - - return e->count; -} - -static void emit_epilogue(Ctx* restrict ctx) { - uint64_t saved = ctx->regs_to_save, stack_usage = ctx->stack_usage; - TB_CGEmitter* e = &ctx->emit; - - if (stack_usage <= 16) { - return; - } - - size_t start = e->count; - - // add rsp, N - if (stack_usage > 0) { - if (stack_usage == (int8_t)stack_usage) { - EMIT1(&ctx->emit, rex(true, 0x00, RSP, 0)); - EMIT1(&ctx->emit, 0x83); - EMIT1(&ctx->emit, mod_rx_rm(MOD_DIRECT, 0x00, RSP)); - EMIT1(&ctx->emit, (int8_t) stack_usage); - } else { - EMIT1(&ctx->emit, rex(true, 0x00, RSP, 0)); - EMIT1(&ctx->emit, 0x81); - EMIT1(&ctx->emit, mod_rx_rm(MOD_DIRECT, 0x00, RSP)); - EMIT4(&ctx->emit, stack_usage); - } - } - - // pop rbp - if (stack_usage > 0) { - EMIT1(&ctx->emit, 0x58 + RBP); - } -} - -static size_t emit_call_patches(TB_Module* restrict m, TB_FunctionOutput* out_f) { - size_t r = 0; - uint32_t src_section = out_f->section; - - for (TB_SymbolPatch* patch = out_f->first_patch; patch; patch = patch->next) { - if (patch->target->tag == TB_SYMBOL_FUNCTION) { - uint32_t dst_section = ((TB_Function*) patch->target)->output->section; - - // you can't do relocations across sections - if (src_section == dst_section) { - assert(patch->pos < out_f->code_size); - - // x64 thinks of relative addresses as being relative - // to the end of the instruction or in this case just - // 4 bytes ahead hence the +4. - size_t actual_pos = out_f->code_pos + patch->pos + 4; - - uint32_t p = ((TB_Function*) patch->target)->output->code_pos - actual_pos; - memcpy(&out_f->code[patch->pos], &p, sizeof(uint32_t)); - - r += 1; - patch->internal = true; - } - } - } - - return out_f->patch_count - r; -} -#undef E - -#define E(fmt, ...) tb_asm_print(e, fmt, ## __VA_ARGS__) -static void disassemble_operands(TB_CGEmitter* e, Disasm* restrict d, TB_X86_Inst inst, size_t pos, int start) { - bool mem = true, imm = true; - for (int i = 0; i < 4; i++) { - if (inst->regs[i] == -1) { - if (mem && (inst->flags & TB_X86_INSTR_USE_MEMOP)) { - if (i > 0) E(", "); - - mem = false; - - if (inst->flags & TB_X86_INSTR_USE_RIPMEM) { - bool is_label = inst->opcode == 0xE8 || inst->opcode == 0xE9 - || (inst->opcode >= 0x70 && inst->opcode <= 0x7F) - || (inst->opcode >= 0x0F80 && inst->opcode <= 0x0F8F); - - if (!is_label) E("["); - - if (d->patch && d->patch->pos == pos + inst->length - 4) { - const TB_Symbol* target = d->patch->target; - - if (target->name[0] == 0) { - E("sym%p", target); - } else { - E("%s", target->name); - } - d->patch = d->patch->next; - } else { - uint32_t target = pos + inst->length + inst->disp; - int bb = tb_emit_get_label(e, target); - uint32_t landed = e->labels[bb] & 0x7FFFFFFF; - - if (landed != target) { - E(".bb%d + %d", bb, (int)target - (int)landed); - } else { - E(".bb%d", bb); - } - } - - if (!is_label) E("]"); - } else { - E("%s [", tb_x86_type_name(inst->data_type)); - if (inst->base != 255) { - E("%s", tb_x86_reg_name(inst->base, TB_X86_TYPE_QWORD)); - } - - if (inst->index != 255) { - E(" + %s*%d", tb_x86_reg_name(inst->index, TB_X86_TYPE_QWORD), 1 << inst->scale); - } - - if (inst->disp > 0) { - E(" + %d", inst->disp); - } else if (inst->disp < 0) { - E(" - %d", -inst->disp); - } - - E("]"); - } - } else if (imm && (inst->flags & (TB_X86_INSTR_IMMEDIATE | TB_X86_INSTR_ABSOLUTE))) { - if (i > 0) E(", "); - - imm = false; - if (inst->flags & TB_X86_INSTR_ABSOLUTE) { - E("%#llx", inst->abs); - } else { - E("%d", inst->imm); - } - } else { - break; - } - } else { - if (i > 0) { - E(", "); - - // special case for certain ops with two data types - if (inst->flags & TB_X86_INSTR_TWO_DATA_TYPES) { - E("%s", tb_x86_reg_name(inst->regs[i], inst->data_type2)); - continue; - } - } - - E("%s", tb_x86_reg_name(inst->regs[i], inst->data_type)); - } - } -} - -static void disassemble(TB_CGEmitter* e, Disasm* restrict d, int bb, size_t pos, size_t end) { - if (bb >= 0) { - E(".bb%d:\n", bb); - } - - while (pos < end) { - while (d->loc != d->end && d->loc->pos == pos) { - E(" // %s : line %d\n", d->loc->file->path, d->loc->line); - d->loc++; - } - - TB_X86_Inst inst; - if (!tb_x86_disasm(&inst, end - pos, &e->data[pos])) { - E(" ERROR\n"); - pos += 1; // skip ahead once... cry - continue; - } - - // special syntax: - // mov rax, rcx - // add rax, rdx INTO add rax, rcx, rdx - if (inst.opcode >= 0x88 && inst.opcode <= 0x8B && inst.regs[0] >= 0) { - size_t saved = pos; - size_t next = pos + inst.length; - pos = next; - - // skip REX - if ((e->data[pos] & 0xF0) == 0x40) { - pos += 1; - } - - // are we an x86 integer op - int op = (e->data[pos] >> 3) & 7; - if (op != 0) { - pos = saved; - goto normie; - } - - TB_X86_Inst inst2; - if (!tb_x86_disasm(&inst2, end - next, &e->data[next]) || inst2.regs[0] != inst.regs[0]) { - pos = saved; - goto normie; - } - - const char* mnemonic = tb_x86_mnemonic(&inst2); - E("%s ", mnemonic); - - // print operands for mov instruction - disassemble_operands(e, d, &inst, pos, 0); - E(", "); - // print operands for data op without the destination - disassemble_operands(e, d, &inst2, next, 1); - E("\n"); - - pos = next + inst2.length; - continue; - } - - normie: - const char* mnemonic = tb_x86_mnemonic(&inst); - E(" "); - if (inst.flags & TB_X86_INSTR_REP) { - E("rep "); - } - if (inst.flags & TB_X86_INSTR_LOCK) { - E("lock "); - } - E("%s", mnemonic); - if (inst.data_type >= TB_X86_TYPE_SSE_SS && inst.data_type <= TB_X86_TYPE_SSE_PD) { - static const char* strs[] = { "ss", "sd", "ps", "pd" }; - E(strs[inst.data_type - TB_X86_TYPE_SSE_SS]); - } - E(" "); - - disassemble_operands(e, d, &inst, pos, 0); - E("\n"); - - pos += inst.length; - } -} -#undef E - -ICodeGen tb__x64_codegen = { - .minimum_addressable_size = 8, - .pointer_size = 64, - .emit_win64eh_unwind_info = emit_win64eh_unwind_info, - .emit_call_patches = emit_call_patches, - .get_data_type_size = get_data_type_size, - .compile_function = compile_function, -}; diff --git a/vendor/tb/src/x64/x64.h b/vendor/tb/src/x64/x64.h deleted file mode 100644 index 1cf78ab7..00000000 --- a/vendor/tb/src/x64/x64.h +++ /dev/null @@ -1,264 +0,0 @@ -#pragma once -#include "../emitter.h" -#include "../tb_internal.h" -#include - -static_assert(sizeof(float) == sizeof(uint32_t), "Float needs to be a 32-bit float!"); -static_assert(sizeof(double) == sizeof(uint64_t), "Double needs to be a 64-bit float!"); - -typedef union { - float f; - uint32_t i; -} Cvt_F32U32; - -typedef union { - double f; - uint64_t i; -} Cvt_F64U64; - -typedef enum { - O, NO, B, NB, E, NE, BE, A, - S, NS, P, NP, L, GE, LE, G -} Cond; - -typedef enum { - RAX, RCX, RDX, RBX, RSP, RBP, RSI, RDI, - R8, R9, R10, R11, R12, R13, R14, R15, - - GPR_NONE = -1 -} GPR; - -typedef enum { - XMM0, XMM1, XMM2, XMM3, XMM4, XMM5, XMM6, XMM7, - XMM8, XMM9, XMM10, XMM11, XMM12, XMM13, XMM14, XMM15, - - XMM_NONE = -1 -} XMM; - -typedef enum { - VAL_NONE, VAL_FLAGS, VAL_GPR, VAL_XMM, VAL_IMM, VAL_MEM, VAL_GLOBAL, VAL_ABS, VAL_LABEL -} ValType; - -typedef enum { - SCALE_X1, SCALE_X2, SCALE_X4, SCALE_X8 -} Scale; - -enum { - MOD_INDIRECT = 0, // [rax] - MOD_INDIRECT_DISP8 = 1, // [rax + disp8] - MOD_INDIRECT_DISP32 = 2, // [rax + disp32] - MOD_DIRECT = 3, // rax -}; - -typedef enum Inst2FPFlags { - INST2FP_DOUBLE = (1u << 0), - INST2FP_PACKED = (1u << 1) -} Inst2FPFlags; - -typedef struct Val { - int8_t type; - - // if VAL_MEM then this is the base - int8_t reg; - - // used by VAL_MEM and VAL_GLOBAL - int8_t index, scale; - - // for VAL_IMM, memory displacement or signed immediate - int32_t imm; - - union { - // for VAL_ABS this is used - uint64_t abs; - // for VAL_GLOBAL this is used as the base - TB_Symbol* symbol; - // for VAL_LABEL - int label; - }; -} Val; - -typedef enum InstType { - #define X(a, ...) a, - #include "x64_insts.inc" -} InstType; - -// EXT variations use a 0F before the opcode -typedef enum { - // Nullary - INST_BYTE, - INST_BYTE_EXT, - // Unary - INST_UNARY, - INST_UNARY_EXT, // 0F - // Binop - INST_BINOP, - INST_BINOP_PLUS, // +r - INST_BINOP_EXT, // 0F - INST_BINOP_EXT2, // 0F (movzx, movsx) - INST_BINOP_EXT3, // 66 (movd, movq) - INST_BINOP_CL, // implicit CL, used by the shift ops - - // SSE - INST_BINOP_SSE, -} InstCategory; - -typedef struct InstDesc { - const char* mnemonic; - uint8_t cat; - - uint8_t op; - - // IMMEDIATES (or unary instructions) - uint8_t op_i; - uint8_t rx_i; -} InstDesc; - -static const GPR WIN64_GPR_PARAMETERS[4] = { RCX, RDX, R8, R9 }; -static const GPR SYSV_GPR_PARAMETERS[6] = { RDI, RSI, RDX, RCX, R8, R9 }; - -static const InstDesc inst_table[] = { - #define X(a, b, c, ...) [a] = { .mnemonic = b, .cat = INST_ ## c, __VA_ARGS__ }, - #include "x64_insts.inc" -}; - -// NOTE(NeGate): This is for Win64, we can handle SysV later -#define WIN64_ABI_CALLER_SAVED ((1u << RAX) | (1u << RCX) | (1u << RDX) | (1u << R8) | (1u << R9) | (1u << R10) | (1u << R11)) -#define WIN64_ABI_CALLEE_SAVED ~WIN64_ABI_CALLER_SAVED - -#define SYSV_ABI_CALLER_SAVED ((1u << RAX) | (1u << RCX) | (1u << RDX) | (1u << RSI) | (1u << RDI) | (1u << R8) | (1u << R9) | (1u << R10) | (1u << R11)) -#define SYSV_ABI_CALLEE_SAVED ((1u << RBX) | (1u << RSP) | (1u << RBP) | (1u << R12) | (1u << R13) | (1u << R14) | (1u << R15)) - -#define SYSCALL_ABI_CALLER_SAVED ((1u << RDI) | (1u << RSI) | (1u << RDX) | (1u << R10) | (1u << R8) | (1u << R9) | (1u << RAX) | (1u << R11)) -#define SYSCALL_ABI_CALLEE_SAVED ~SYSCALL_ABI_CALLER_SAVED - -inline static Val val_gpr(GPR g) { - return (Val) { .type = VAL_GPR, .reg = g }; -} - -inline static Val val_xmm(XMM x) { - return (Val) { .type = VAL_XMM, .reg = x }; -} - -inline static Val val_flags(Cond c) { - return (Val) { .type = VAL_FLAGS, .reg = c }; -} - -inline static Val val_global(TB_Symbol* s, int disp) { - return (Val) { .type = VAL_GLOBAL, .symbol = s, .imm = disp }; -} - -inline static Val val_imm(int32_t imm) { - return (Val) { .type = VAL_IMM, .imm = imm }; -} - -inline static Val val_abs(uint64_t abs) { - return (Val) { .type = VAL_ABS, .abs = abs }; -} - -inline static Val val_label(int target) { - return (Val) { .type = VAL_LABEL, .label = target }; -} - -inline static Val val_stack(int s) { - return (Val) { .type = VAL_MEM, .reg = RSP, .index = GPR_NONE, .scale = SCALE_X1, .imm = s }; -} - -inline static Val val_base_disp(GPR b, int d) { - return (Val) { - .type = VAL_MEM, .reg = b, .index = GPR_NONE, .scale = SCALE_X1, .imm = d - }; -} - -inline static Val val_base_index_disp(GPR b, GPR i, Scale s, int d) { - return (Val) { - .type = VAL_MEM, .reg = b, .index = i, .scale = s, .imm = d - }; -} - -inline static bool is_value_mem(const Val* v) { - return v->type == VAL_MEM || v->type == VAL_GLOBAL; -} - -inline static bool is_value_gpr(const Val* v, GPR g) { - if (v->type != VAL_GPR) return false; - - return (v->reg == g); -} - -inline static bool is_value_xmm(const Val* v, XMM x) { - if (v->type != VAL_XMM) return false; - - return (v->reg == x); -} - -inline static bool is_value_match(const Val* a, const Val* b) { - if (a->type != b->type) return false; - - if (a->type == VAL_MEM) { - return a->reg == b->reg && a->index == b->index && a->scale == b->scale && a->index == b->index; - } - - return (a->type == VAL_GPR || a->type == VAL_XMM) ? a->reg == b->reg : false; -} - -#ifdef __GNUC__ -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wunused-variable" -#endif -static const char* GPR_NAMES[] = { "RAX", "RCX", "RDX", "RBX", "RSP", "RBP", "RSI", "RDI", "R8", "R9", "R10", "R11", "R12", "R13", "R14", "R15" }; -static const char* XMM_NAMES[] = { "XMM0", "XMM1", "XMM2", "XMM3", "XMM4", "XMM5", "XMM6", "XMM7", "XMM8", "XMM9", "XMM10", "XMM11", "XMM12", "XMM13", "XMM14", "XMM15" }; -static const char* COND_NAMES[] = { - "O", "NO", "B", "NB", "E", "NE", "BE", "A", - "S", "NS", "P", "NP", "L", "GE", "LE", "G" -}; -#ifdef __GNUC__ -#pragma GCC diagnostic pop -#endif - -// shorthand macros -#define STACK_ALLOC(size, align) (ctx->stack_usage = align_up(ctx->stack_usage + (size), align), - ctx->stack_usage) -#define INST0(op, dt) inst0(&ctx->emit, op, dt) -#define INST1(op, a, dt) inst1(&ctx->emit, op, a, dt) -#define INST2(op, a, b, dt) inst2(&ctx->emit, op, a, b, dt) -#define INST2SSE(op, a, b, dt) inst2sse(&ctx->emit, op, a, b, dt) - -#define __(mnemonic, ...) x86_ ## mnemonic(e, __VA_ARGS__) -#define COMMENT(...) (e->has_comments ? tb_emit_comment(e, tmp_arena, __VA_ARGS__) : (void)0) - -typedef enum { - UNWIND_OP_PUSH_NONVOL = 0, /* info == register number */ - UNWIND_OP_ALLOC_LARGE, /* no info, alloc size in next 2 slots */ - UNWIND_OP_ALLOC_SMALL, /* info == size of allocation / 8 - 1 */ - UNWIND_OP_SET_FPREG, /* no info, FP = RSP + UNWIND_INFO.FPRegOffset*16 */ - UNWIND_OP_SAVE_NONVOL, /* info == register number, offset in next slot */ - UNWIND_OP_SAVE_NONVOL_FAR, /* info == register number, offset in next 2 slots */ - UNWIND_OP_SAVE_XMM128 = 8, /* info == XMM reg number, offset in next slot */ - UNWIND_OP_SAVE_XMM128_FAR, /* info == XMM reg number, offset in next 2 slots */ - UNWIND_OP_PUSH_MACHFRAME /* info == 0: no error-code, 1: error-code */ -} UnwindCodeOps; - -typedef union { - struct { - uint8_t code_offset; - uint8_t unwind_op : 4; - uint8_t op_info : 4; - }; - uint16_t frame_offset; -} UnwindCode; - -enum { - UNWIND_FLAG_EHANDLER = 0x01, - UNWIND_FLAG_UHANDLER = 0x02, - UNWIND_FLAG_CHAININFO = 0x04, -}; - -typedef struct { - uint8_t version : 3; - uint8_t flags : 5; - uint8_t prolog_length; - uint8_t code_count; - uint8_t frame_register : 4; - uint8_t frame_offset : 4; - UnwindCode code[]; // ((code_count + 1) & ~1) - 1 -} UnwindInfo; - diff --git a/vendor/tb/src/x64/x64_disasm.c b/vendor/tb/src/x64/x64_disasm.c deleted file mode 100644 index 68f14e6f..00000000 --- a/vendor/tb/src/x64/x64_disasm.c +++ /dev/null @@ -1,987 +0,0 @@ -#include -#include // __debugbreak - -// this is used to parse ModRM and SIB -#define UNPACK_233(a, b, c, src) \ -(a = (src >> 6), b = (src >> 3) & 7, c = (src & 7)) - -#define READ16(x) (memcpy(&(x), &data[current], 2), current += 2, x) -#define READ32(x) (memcpy(&(x), &data[current], 4), current += 4, x) -#define READ64(x) (memcpy(&(x), &data[current], 8), current += 8, x) - -#define READ16LE(x) READ16(x) -#define READ32LE(x) READ32(x) -#define READ64LE(x) READ64(x) - -bool tb_x86_disasm(TB_X86_Inst* restrict inst, size_t length, const uint8_t* data) { - size_t current = 0; - uint8_t rex = 0; - - inst->dt = TB_X86_TYPE_DWORD; - - // legacy prefixes - uint8_t b; - uint32_t flags = 0; - while (true) { - b = data[current++]; - switch (b) { - case 0xF0: flags |= TB_X86_INSTR_LOCK; break; - case 0xF3: flags |= TB_X86_INSTR_REP; break; - case 0xF2: flags |= TB_X86_INSTR_REPNE; break; - case 0x66: inst->dt = TB_X86_TYPE_WORD; break; - case 0x67: inst->dt = TB_X86_TYPE_DWORD; break; - case 0x2E: inst->segment = TB_X86_SEGMENT_CS; break; - case 0x36: inst->segment = TB_X86_SEGMENT_SS; break; - case 0x3E: inst->segment = TB_X86_SEGMENT_DS; break; - case 0x26: inst->segment = TB_X86_SEGMENT_ES; break; - case 0x64: inst->segment = TB_X86_SEGMENT_FS; break; - case 0x65: inst->segment = TB_X86_SEGMENT_GS; break; - default: goto done_prefixing; - } - } - - // rex/vex - done_prefixing:; - if ((b & 0xF0) == 0x40) { - rex = b; - b = data[current++]; - } - - // opcode translation (from 1-3 bytes to 10bits): - // op => 00________ - // 0F op => 01________ - // 0F 38 op => 10________ - // 0F 3A op => 11________ - uint32_t op = 0; - if (b == 0x0F) { - b = data[current++]; - switch (b) { - case 0x38: b = data[current++], op = 0x200 | b; break; - case 0x3A: b = data[current++], op = 0x300 | b; break; - default: op = 0x100 | b; break; - } - } else { - op |= b; - } - - enum { - // fancy x86 operand - OP_MODRM = 1, - // r/m operand will appear on the left side by default, this will flip that - OP_DIR = 2, - // immediate with a size is based on the operand size - OP_IMM = 4, - // signed 8bit immediate - OP_IMM8 = 8, - // operand size is forcibly 8bit - OP_8BIT = 16, - // operand size is forcibly 64bit - OP_64BIT = 32, - // the bottom 3bits of the opcode are the base reg - OP_PLUSR = 64, - // rx represents an extended piece of the opcode - OP_FAKERX = 128, - // no operands - OP_0ARY = 256, - // 2 datatypes on the instruction - OP_2DT = 512, - // rcx is the rx field - OP_RCX = 1024, - // rax is the rm field - OP_RAX = 2048, - // vector op - OP_SSE = 4096, - }; - - #define NORMIE_BINOP(op) [op+0] = OP_MODRM | OP_8BIT, [op+1] = OP_MODRM, [op+2] = OP_MODRM | OP_DIR | OP_8BIT, [op+3] = OP_MODRM | OP_DIR, [op+4] = OP_RAX | OP_IMM8, [op+5] = OP_RAX | OP_IMM - #define _0F(op) [0x100+op] - #define _0F2(a,b) [0x100+a ... 0x100+b] - static const uint32_t op_map[1024] = { - NORMIE_BINOP(0x00), // add - NORMIE_BINOP(0x08), // or - NORMIE_BINOP(0x20), // and - NORMIE_BINOP(0x28), // sub - NORMIE_BINOP(0x30), // xor - NORMIE_BINOP(0x38), // cmp - - // mov - [0x88] = OP_MODRM | OP_8BIT, - [0x89] = OP_MODRM, - [0x8A] = OP_MODRM | OP_DIR | OP_8BIT, - [0x8B] = OP_MODRM | OP_DIR, - - // push r+ - // pop r+ - [0x50 ... 0x5F] = OP_PLUSR | OP_64BIT, - // movsxd reg, r/m - [0x63] = OP_MODRM | OP_DIR | OP_2DT, - // imul reg, r/m, imm - [0x69] = OP_MODRM | OP_DIR | OP_IMM, - // jcc rel8 - [0x70 ... 0x7F] = OP_IMM8, - // OP r/m8, imm8 - [0x80] = OP_MODRM | OP_IMM | OP_8BIT | OP_FAKERX, - // OP r/m, imm32 - [0x81] = OP_MODRM | OP_IMM | OP_FAKERX, - // OP r/m, imm8 - [0x83] = OP_MODRM | OP_IMM8 | OP_FAKERX, - // test r/m, reg - [0x84] = OP_MODRM | OP_8BIT, - [0x85] = OP_MODRM, - // lea reg, r/m - [0x8D] = OP_MODRM | OP_DIR, - // nop - [0x90] = OP_0ARY, - // cbw/cwde/cdqe - [0x98] = OP_0ARY, - // cwd/cdq/cqo - [0x99] = OP_0ARY, - // mov r+ imm - [0xB0 ... 0xB7] = OP_PLUSR | OP_IMM | OP_8BIT, - [0xB8 ... 0xBF] = OP_PLUSR | OP_IMM, - // sar r/m, imm8 - // shl r/m, imm8 - // shr r/m, imm8 - [0xC0] = OP_MODRM | OP_IMM8 | OP_FAKERX | OP_8BIT, - [0xC1] = OP_MODRM | OP_IMM8 | OP_FAKERX, - // ret - [0xC3] = OP_0ARY, - // mov r/m imm - [0xC6] = OP_MODRM | OP_IMM | OP_8BIT | OP_FAKERX, - [0xC7] = OP_MODRM | OP_IMM | OP_FAKERX, - // int3 - [0xCC] = OP_0ARY, - // sar r/m, CL - // shl r/m, CL - // shr r/m, CL - // rol r/m, CL - // ror r/m, CL - [0xD2] = OP_MODRM | OP_RCX | OP_FAKERX | OP_8BIT, - [0xD3] = OP_MODRM | OP_RCX | OP_FAKERX, - // call rel32 - [0xE8] = OP_IMM, - // jmp rel32 - [0xE9] = OP_IMM, - // jmp rel8 - [0xEB] = OP_IMM8, - // idiv r/m8 - [0xF6] = OP_MODRM | OP_FAKERX | OP_8BIT, - [0xF7] = OP_MODRM | OP_FAKERX, - // dec/call/jmp r/m - [0xFE] = OP_MODRM | OP_FAKERX, - [0xFF] = OP_MODRM | OP_FAKERX, - - //////////////////////////////// - // ESCAPE OPS - //////////////////////////////// - // SSE: movu - _0F(0x10) = OP_MODRM | OP_SSE | OP_DIR, - _0F(0x11) = OP_MODRM | OP_SSE, - // nop r/m - _0F(0x1F) = OP_MODRM, - // cmovcc reg, r/m - _0F2(0x40, 0x4F) = OP_MODRM | OP_DIR, - // SSE: add, mul, sub, min, div, max - _0F2(0x51, 0x5F) = OP_MODRM | OP_DIR | OP_SSE, - // movzx reg, r/m - _0F(0xB6) = OP_MODRM | OP_2DT | OP_DIR, - _0F(0xB7) = OP_MODRM | OP_2DT | OP_DIR, - // movsx reg, r/m - _0F(0xBE) = OP_MODRM | OP_2DT | OP_DIR, - _0F(0xBF) = OP_MODRM | OP_2DT | OP_DIR, - // jcc rel32 - _0F2(0x80, 0x8F) = OP_IMM, - // setcc r/m - _0F2(0x90, 0x9F) = OP_MODRM | OP_FAKERX, - }; - #undef NORMIE_BINOP - #undef _0F - - uint32_t props = op_map[op]; - if (props == 0) { - return false; - } - - uint32_t regs = 0; - int32_t disp = 0; - uint8_t scale = 0; - - // in the "default" "type" "system", REX.W is 64bit, certain ops - // will mark they're 8bit and most will just be 32bit (with 16bit on ADDR16) - if (props & OP_SSE) { - // ss REP OPCODE - // sd REPNE OPCODE - // ps __ OPCODE - // pd DATA16 OPCODE - if (flags & TB_X86_INSTR_REPNE) { - inst->dt = TB_X86_TYPE_SSE_SD; - } else if (flags & TB_X86_INSTR_REP) { - inst->dt = TB_X86_TYPE_SSE_SS; - } else if (inst->dt == TB_X86_TYPE_WORD) { - inst->dt = TB_X86_TYPE_SSE_PD; - } else { - inst->dt = TB_X86_TYPE_SSE_PS; - } - - flags &= ~(TB_X86_INSTR_REP | TB_X86_INSTR_REPNE); - } else if (props & OP_64BIT) { - inst->dt = TB_X86_TYPE_QWORD; - } else if (props & OP_8BIT) { - inst->dt = TB_X86_TYPE_BYTE; - } else { - if (rex & 8) inst->dt = TB_X86_TYPE_QWORD; - } - - if (props & OP_MODRM) { - uint8_t mod, rx, rm, modrm = data[current++]; - UNPACK_233(mod, rx, rm, modrm); - - if (props & OP_FAKERX) { - if (op == 0xF6) { - props |= OP_IMM8; - } else if (op == 0xF7) { - props |= OP_IMM; - } - - regs |= 0xFF0000; // no rx since it's reserved - op |= rx << 12; - } else if (props & OP_RCX) { - regs |= RCX << 16; - } else { - // unpack rx - regs |= ((rex&4 ? 8 : 0) | rx) << 16; - } - - if (mod == MOD_DIRECT) { - // unpack base - regs |= (rex&1 ? 8 : 0) | rm; - } else { - flags |= TB_X86_INSTR_INDIRECT; - if (rm == TB_X86_RSP) { // use SIB - uint8_t index, base, sib = data[current++]; - UNPACK_233(scale, index, base, sib); - - if (base == TB_X86_RSP && index == TB_X86_RSP) { - // no index - regs |= 0xFF00; - } else if (mod == 0 && base == TB_X86_RBP) { - // indirect disp32 - regs |= 0xFFFF; - } - - regs |= (rex&1 ? 8 : 0) | base; - regs |= ((rex&2 ? 8 : 0) | index) << 8; - } else { - if (mod == MOD_INDIRECT && rm == TB_X86_RBP) { - // rip-relative - regs |= 0xFFFF; - } else { - // base-only - regs |= 0xFF00; - regs |= (rex&1 ? 8 : 0) | rm; - } - } - - // unpack displacement - if (mod == MOD_INDIRECT_DISP8) { - inst->disp = (int8_t) data[current++]; - } else if (mod == MOD_INDIRECT_DISP32 || (regs & 0xFFFF) == 0xFFFF) { - memcpy(&inst->disp, &data[current], sizeof(disp)); - current += 4; - } else { - inst->disp = 0; - } - } - } else if (props & OP_RAX) { - regs |= 0xFFFF00; - regs |= RAX << 0; - } else if (props & OP_PLUSR) { - regs |= 0xFFFF00; - regs |= (rex&1 ? 8 : 0) | (op & 0x7); - - // discard those reg bits in the opcode - op &= ~0x7; - } else { - regs |= 0xFFFFFF; - } - - if (props & OP_IMM8) { - inst->imm = (int8_t) data[current++]; - flags |= TB_X86_INSTR_IMMEDIATE; - } else if (props & OP_IMM) { - assert(inst->dt >= TB_X86_TYPE_BYTE && inst->dt <= TB_X86_TYPE_QWORD); - int size = 1 << (inst->dt - TB_X86_TYPE_BYTE); - if (size > 4 && (op < 0xB8 || op > 0xBF)) { - // only operation with 8byte immediates is movabs (0xB8 ... 0xBF) - size = 4; - } - - uint64_t x; - memcpy(&x, &data[current], size); - current += size; - - inst->imm = tb__sxt(x, size*8, 64); - flags |= TB_X86_INSTR_IMMEDIATE; - } - - if (props & OP_DIR) { - flags |= TB_X86_INSTR_DIRECTION; - } - - inst->dt2 = inst->dt; - inst->opcode = op; - inst->scale = scale; - inst->regs = regs; - inst->flags = flags; - inst->length = current; - return true; -} - -// INSTRUCTION ::= PREFIX* REX OPCODE[1-4] (MODRM SIB?)? IMMEDIATE? OFFSET? -#if 0 -#define ABC(amt) if (current + (amt) > length) return false -bool tb_x86_disasm(TB_X86_Inst* restrict inst, size_t length, const uint8_t* data) { - *inst = (TB_X86_Inst){ 0 }; - for (int i = 0; i < 4; i++) inst->regs[i] = -1; - - size_t current = 0; - - //////////////////////////////// - // Parse prefixes - //////////////////////////////// - uint8_t rex = 0; // 0x4X - bool addr32 = false; // 0x67 - bool addr16 = false; // 0x66 - bool rep = false; // 0xF3 these are both used - bool repne = false; // 0xF2 to define SSE types - bool ext = false; // 0x0F - - uint8_t op; - while (true) { - ABC(1); - op = data[current++]; - - switch (op) { - case 0x40 ... 0x4F: rex = op; break; - case 0xF0: inst->flags |= TB_X86_INSTR_LOCK; break; - case 0x66: addr16 = true; break; - case 0x67: addr32 = true; break; - case 0xF3: inst->flags |= TB_X86_INSTR_REP; break; - case 0xF2: inst->flags |= TB_X86_INSTR_REPNE; break; - case 0x2E: inst->segment = TB_X86_SEGMENT_CS; break; - case 0x36: inst->segment = TB_X86_SEGMENT_SS; break; - case 0x3E: inst->segment = TB_X86_SEGMENT_DS; break; - case 0x26: inst->segment = TB_X86_SEGMENT_ES; break; - case 0x64: inst->segment = TB_X86_SEGMENT_FS; break; - case 0x65: inst->segment = TB_X86_SEGMENT_GS; break; - default: goto done_prefixing; - } - } - - done_prefixing:; - //////////////////////////////// - // Parse opcode - //////////////////////////////// - enum { - OP_8BIT = 1, - OP_16BIT = 2, - OP_64BIT = 4, - OP_FAKERX = 8, - OP_2DT = 16, - OP_SSE = 32, - }; - - enum { - // encoding - OP_BAD = 0x0000, - OP_MR = 0x1000, - OP_RM = 0x2000, - OP_M = 0x3000, - OP_MI = 0x4000, - OP_MI8 = 0x5000, - OP_MC = 0x6000, - OP_PLUSR = 0x7000, - OP_0ARY = 0x8000, - OP_REL8 = 0x9000, - OP_REL32 = 0xA000, - OP_IMM = 0xB000, - }; - - #define NORMIE_BINOP(op) [op+0] = OP_MR | OP_8BIT, [op+1] = OP_MR, [op+2] = OP_RM | OP_8BIT, [op+3] = OP_RM - static const uint16_t first_table[256] = { - NORMIE_BINOP(0x00), // add - NORMIE_BINOP(0x08), // or - NORMIE_BINOP(0x20), // and - NORMIE_BINOP(0x28), // sub - NORMIE_BINOP(0x30), // xor - NORMIE_BINOP(0x38), // cmp - NORMIE_BINOP(0x88), // mov - - // OP r/m8, imm8 - [0x80] = OP_MI8 | OP_8BIT | OP_FAKERX, - // OP r/m, imm32 - [0x81] = OP_MI | OP_FAKERX, - // OP r/m, imm8 - [0x83] = OP_MI8 | OP_FAKERX, - // TEST r/m, reg - [0x84] = OP_MR | OP_8BIT, - [0x85] = OP_MR, - // push reg - // pop reg - [0x50 ... 0x5F] = OP_PLUSR | OP_64BIT, - // lea reg, r/m - [0x8D] = OP_RM, - // movsxd reg, r/m - [0x63] = OP_RM | OP_2DT, - // push imm32 - [0x68] = OP_IMM, - // imul reg, r/m, imm32 - [0x69] = OP_RM, - // jcc rel8 - [0x70 ... 0x7F] = OP_REL8, - // nop - [0x90] = OP_0ARY, - // cwd/cdq/cqo - [0x99] = OP_0ARY, - // ret - [0xC3] = OP_0ARY, - // int3 - [0xCC] = OP_0ARY, - // movabs - [0xB0 ... 0xB7] = OP_PLUSR | OP_8BIT, - [0xB8 ... 0xBF] = OP_PLUSR, - // mov r/m, imm - [0xC6] = OP_MI | OP_FAKERX | OP_8BIT, - [0xC7] = OP_MI | OP_FAKERX, - // movs - [0xA4 ... 0xA5] = OP_0ARY, - // stos - [0xAA ... 0xAB] = OP_0ARY, - // scas - [0xAE ... 0xAF] = OP_0ARY, - // sar r/m, imm8 - // shl r/m, imm8 - // shr r/m, imm8 - [0xC0] = OP_MI8 | OP_FAKERX | OP_8BIT, - [0xC1] = OP_MI8 | OP_FAKERX, - // sar r/m, CL - // shl r/m, CL - // shr r/m, CL - // rol r/m, CL - // ror r/m, CL - [0xD2] = OP_MC | OP_FAKERX | OP_8BIT, - [0xD3] = OP_MC | OP_FAKERX, - // call rel32 - [0xE8] = OP_REL32, - // jmp rel32 - [0xE9] = OP_REL32, - // jmp rel8 - [0xEB] = OP_REL8, - // idiv r/m8 - [0xF6] = OP_M | OP_FAKERX | OP_8BIT, - [0xF7] = OP_M | OP_FAKERX, - // dec/call/jmp r/m - [0xFE] = OP_M | OP_FAKERX, - [0xFF] = OP_M | OP_FAKERX, - }; - #undef NORMIE_BINOP - - static const uint16_t ext_table[256] = { - // ud2 - [0x0B] = OP_0ARY, - // prefetch r/m - [0x18] = OP_FAKERX, - // SSE: movu - [0x10] = OP_RM | OP_SSE, [0x11] = OP_MR | OP_SSE, - // SSE: add, mul, sub, min, div, max - [0x51 ... 0x5F] = OP_RM | OP_SSE, - // cmovcc - [0x40 ... 0x4F] = OP_RM, - // SSE: cvtsi2sd - [0x2A] = OP_RM | OP_SSE, - [0x2C] = OP_RM | OP_SSE, - // SSE: ucomi - [0x2E] = OP_RM | OP_SSE, - // rdtsc - [0x31] = OP_0ARY, - // nop r/m - [0x1F] = OP_RM, - // imul reg, r/m - [0xAF] = OP_RM, - // movzx - [0xB6 ... 0xB7] = OP_RM | OP_2DT, - // movsx reg, r/m - [0xBE] = OP_RM | OP_2DT, - // movsx reg, r/m - [0xBF] = OP_RM | OP_2DT, - // jcc rel32 - [0x80 ... 0x8F] = OP_REL32, - // setcc r/m - [0x90 ... 0x9F] = OP_M, - }; - - uint16_t first; - if (op == 0x0F) { - op = data[current++]; - first = ext_table[op]; - inst->opcode = 0x0F00 | op; - } else { - first = first_table[op]; - inst->opcode |= op; - } - uint16_t flags = first & 0xFFF; - - #if 1 - if (first == 0) return false; - #else - assert(first != 0 && "unknown op"); - #endif - - // info from table - uint16_t enc = first & 0xF000; - bool uses_imm = enc == OP_MI || enc == OP_MI8; - - // in the "default" "type" "system", REX.W is 64bit, certain ops - // will mark they're 8bit and most will just be 32bit (with 16bit on ADDR16) - inst->dt = TB_X86_TYPE_DWORD; - if (flags & OP_64BIT) { - // basically forced 64bit - inst->dt = TB_X86_TYPE_QWORD; - } else if (flags & OP_SSE) { - // ss REP OPCODE - // sd REPNE OPCODE - // ps __ OPCODE - // pd DATA16 OPCODE - if (inst->flags & TB_X86_INSTR_REPNE) { - inst->dt = TB_X86_TYPE_SSE_SD; - } else if (inst->flags & TB_X86_INSTR_REP) { - inst->dt = TB_X86_TYPE_SSE_SS; - } else if (addr16) { - inst->dt = TB_X86_TYPE_SSE_PD; - } else { - inst->dt = TB_X86_TYPE_SSE_PS; - } - - inst->flags &= ~(TB_X86_INSTR_REP | TB_X86_INSTR_REPNE); - } else { - if (rex & 0x8) inst->dt = TB_X86_TYPE_QWORD; - else if (flags & OP_8BIT) inst->dt = TB_X86_TYPE_BYTE; - else if (addr16) inst->dt = TB_X86_TYPE_WORD; - } - - assert(enc != OP_BAD); - if (enc == OP_IMM) { - ABC(4); - int32_t imm = READ32LE(imm); - inst->flags |= TB_X86_INSTR_IMMEDIATE; - inst->imm = imm; - inst->length = current; - return true; - } else if (enc == OP_REL8) { - inst->flags |= TB_X86_INSTR_USE_RIPMEM; - inst->flags |= TB_X86_INSTR_USE_MEMOP; - inst->base = -1; - inst->index = -1; - - ABC(1); - inst->disp = data[current++]; - inst->length = current; - return true; - } else if (enc == OP_REL32) { - inst->flags |= TB_X86_INSTR_USE_RIPMEM; - inst->flags |= TB_X86_INSTR_USE_MEMOP; - inst->base = -1; - inst->index = -1; - - ABC(4); - int32_t imm = READ32LE(imm); - inst->disp = imm; - inst->length = current; - return true; - } else if (enc == OP_0ARY) { - inst->length = current; - return true; - } else if (enc == OP_PLUSR) { - // bottom 8bits of the opcode are the base reg - inst->regs[0] = (rex & 1 ? 8 : 0) | (inst->opcode & 7); - inst->opcode &= ~7; - - if (op >= 0xB0 && op <= 0xBF) { - // movabs (pre-APX) is the only instruction with a 64bit immediate so i'm finna - // special case it. - uint64_t imm = 0; - switch (inst->data_type) { - case TB_X86_TYPE_BYTE: - ABC(1); - imm = data[current++]; - break; - - case TB_X86_TYPE_WORD: - ABC(2); - imm = READ16LE(imm); - break; - - case TB_X86_TYPE_DWORD: - ABC(4); - imm = READ32LE(imm); - break; - - case TB_X86_TYPE_QWORD: - ABC(8); - imm = READ64LE(imm); - break; - - default: tb_todo(); - } - - inst->flags |= TB_X86_INSTR_IMMEDIATE; - inst->imm = imm; - } - inst->length = current; - return true; - } else { - //////////////////////////////// - // Parse ModRM - //////////////////////////////// - bool rm_slot = enc == OP_RM; - bool rx_slot = !rm_slot; - - ABC(1); - uint8_t mod, rx, rm, modrm = data[current++]; - UNPACK_233(mod, rx, rm, modrm); - - if (flags & OP_FAKERX) { - inst->opcode <<= 4; - inst->opcode |= rx; - - if (inst->opcode == 0xF60) { - flags = OP_MI8; - } else if (inst->opcode == 0xF70) { - flags = OP_MI; - } - } - - if (flags & OP_2DT) { - inst->flags |= TB_X86_INSTR_TWO_DATA_TYPES; - if (inst->opcode == 0x0FB6 || inst->opcode == 0x0FB7 || inst->opcode == 0x0FBE || inst->opcode == 0x0FBF) { - inst->data_type2 = (inst->opcode == 0x0FB6 || inst->opcode == 0x0FBE) ? TB_X86_TYPE_BYTE : TB_X86_TYPE_WORD; - } else { - inst->data_type2 = TB_X86_TYPE_DWORD; - } - } - - if (enc != OP_M && !uses_imm) { - if (enc == OP_MC) { - inst->flags |= TB_X86_INSTR_TWO_DATA_TYPES; - inst->data_type2 = TB_X86_TYPE_BYTE; - inst->regs[rx_slot] = RCX; - } else { - int8_t real_rx = ((rex & 4 ? 8 : 0) | rx); - if (rex == 0 && inst->data_type == TB_X86_TYPE_BYTE && real_rx >= 4) { - // use high registers - real_rx += 8; - } - inst->regs[rx_slot] = real_rx; - } - } - - // writes out RM reg (or base and index) - ptrdiff_t delta = x86_parse_memory_op(inst, length - current, &data[current], rm_slot, mod, rm, rex); - if (delta < 0) { - return false; - } - current += delta; - - // immediates might use RX for an extended opcode - // IMUL's ternary is a special case - if (uses_imm || op == 0x68 || op == 0x69) { - if ((enc == OP_MI && inst->data_type == TB_X86_TYPE_BYTE) || enc == OP_MI8 || op == 0x68) { - ABC(1); - int8_t imm = data[current++]; - inst->flags |= TB_X86_INSTR_IMMEDIATE; - inst->imm = imm; - } else if (enc == OP_MI || op == 0x69) { - ABC(4); - int32_t imm = READ32LE(imm); - inst->flags |= TB_X86_INSTR_IMMEDIATE; - inst->imm = imm; - } else { - return false; - } - } - - inst->length = current; - return true; - } -} -#endif - -const char* tb_x86_mnemonic(TB_X86_Inst* inst) { - if (inst->opcode == 0x99) { - // cwd/cdq/cqo - if (inst->dt == TB_X86_TYPE_WORD) return "cwd"; - if (inst->dt == TB_X86_TYPE_DWORD) return "cdq"; - if (inst->dt == TB_X86_TYPE_QWORD) return "cqo"; - return "??"; - } else if (inst->opcode == 0x98) { - // cbw/cwde/cdqe - if (inst->dt == TB_X86_TYPE_WORD) return "cbw"; - if (inst->dt == TB_X86_TYPE_DWORD) return "cwde"; - if (inst->dt == TB_X86_TYPE_QWORD) return "cdqe"; - return "??"; - } - - #define _0F(op) 0x100+op - #define _0Fx(op, rx) 0x100+op+(rx<<12) - #define _Rx(op, rx) op+(rx<<12) - #define NORMIE_BINOP(op) case 0x80 + (op<<12): case 0x81 + (op<<12): case 0x83 + (op<<12) - switch (inst->opcode) { - case _0F(0x0B): return "ud2"; - case _0F(0x31): return "rdtsc"; - case 0xCC: return "int3"; - - case _0Fx(0x18, 0): return "prefetchnta"; - case _0Fx(0x18, 1): return "prefetch0"; - case _0Fx(0x18, 2): return "prefetch1"; - case _0Fx(0x18, 3): return "prefetch2"; - - case 0x00 ... 0x05: NORMIE_BINOP(0): return "add"; - case 0x08 ... 0x0D: NORMIE_BINOP(1): return "or"; - case 0x20 ... 0x25: NORMIE_BINOP(4): return "and"; - case 0x28 ... 0x2D: NORMIE_BINOP(5): return "sub"; - case 0x30 ... 0x33: NORMIE_BINOP(6): return "xor"; - case 0x38 ... 0x3D: NORMIE_BINOP(7): return "cmp"; - case 0x88 ... 0x8B: case 0x00C6: case 0x00C7: return "mov"; - - case 0xA4: case 0xA5: return "movs"; - case 0xAA: case 0xAB: return "stos"; - case 0xAE: case 0xAF: return "scas"; - - case 0x00C0: case 0x00C1: case 0x00D2: case 0x00D3: return "rol"; - case 0x10C0: case 0x10C1: case 0x10D2: case 0x10D3: return "ror"; - case 0x40C0: case 0x40C1: case 0x40D2: case 0x40D3: return "shl"; - case 0x50C0: case 0x50C1: case 0x50D2: case 0x50D3: return "shr"; - case 0x70C0: case 0x70C1: case 0x70D2: case 0x70D3: return "sar"; - - case 0x00F6: case 0x00F7: return "test"; - case 0x20F6: case 0x20F7: return "not"; - case 0x60F6: case 0x60F7: return "div"; - case 0x70F6: case 0x70F7: return "idiv"; - - case 0x84: case 0x85: return "test"; - - case _0F(0x10): case _0F(0x11): return "mov"; - case _0F(0x58): return "add"; - case _0F(0x59): return "mul"; - case _0F(0x5C): return "sub"; - case _0F(0x5D): return "min"; - case _0F(0x5E): return "div"; - case _0F(0x5F): return "max"; - case _0F(0xC2): return "cmp"; - case _0F(0x2A): return "___"; - case _0F(0x2C): return "___"; - case _0F(0x2E): return "ucomi"; - case _0F(0x51): return "sqrt"; - case _0F(0x52): return "rsqrt"; - case _0F(0x54): return "and"; - case _0F(0x56): return "or"; - case _0F(0x57): return "xor"; - - case 0xB0 ... 0xBF: return "mov"; - case _0F(0xB6): case _0F(0xB7): return "movzx"; - case _0F(0xBE): case _0F(0xBF): return "movsx"; - - case 0x8D: return "lea"; - case 0x90: return "nop"; - case 0xC3: return "ret"; - case 0x63: return "movsxd"; - case 0x50: return "push"; - case 0x58: return "pop"; - - case 0x00FF: return "inc"; - case 0x10FF: return "dec"; - - case 0xE8: case 0x20FF: case 0x30FF: return "call"; - case 0xEB: case 0xE9: case 0x40FF: case 0x50FF: return "jmp"; - - case _0F(0x1F): return "nop"; - case 0x68: return "push"; - case _0F(0xAF): case 0x69: case 0x6B: return "imul"; - - case _0F(0x40): return "cmovo"; - case _0F(0x41): return "cmovno"; - case _0F(0x42): return "cmovb"; - case _0F(0x43): return "cmovnb"; - case _0F(0x44): return "cmove"; - case _0F(0x45): return "cmovne"; - case _0F(0x46): return "cmovbe"; - case _0F(0x47): return "cmova"; - case _0F(0x48): return "cmovs"; - case _0F(0x49): return "cmovns"; - case _0F(0x4A): return "cmovp"; - case _0F(0x4B): return "cmovnp"; - case _0F(0x4C): return "cmovl"; - case _0F(0x4D): return "cmovge"; - case _0F(0x4E): return "cmovle"; - case _0F(0x4F): return "cmovg"; - - case _0F(0x90): return "seto"; - case _0F(0x91): return "setno"; - case _0F(0x92): return "setb"; - case _0F(0x93): return "setnb"; - case _0F(0x94): return "sete"; - case _0F(0x95): return "setne"; - case _0F(0x96): return "setbe"; - case _0F(0x97): return "seta"; - case _0F(0x98): return "sets"; - case _0F(0x99): return "setns"; - case _0F(0x9A): return "setp"; - case _0F(0x9B): return "setnp"; - case _0F(0x9C): return "setl"; - case _0F(0x9D): return "setge"; - case _0F(0x9E): return "setle"; - case _0F(0x9F): return "setg"; - - case _0F(0x80): case 0x70: return "jo"; - case _0F(0x81): case 0x71: return "jno"; - case _0F(0x82): case 0x72: return "jb"; - case _0F(0x83): case 0x73: return "jnb"; - case _0F(0x84): case 0x74: return "je"; - case _0F(0x85): case 0x75: return "jne"; - case _0F(0x86): case 0x76: return "jbe"; - case _0F(0x87): case 0x77: return "ja"; - case _0F(0x88): case 0x78: return "js"; - case _0F(0x89): case 0x79: return "jns"; - case _0F(0x8A): case 0x7A: return "jp"; - case _0F(0x8B): case 0x7B: return "jnp"; - case _0F(0x8C): case 0x7C: return "jl"; - case _0F(0x8D): case 0x7D: return "jge"; - case _0F(0x8E): case 0x7E: return "jle"; - case _0F(0x8F): case 0x7F: return "jg"; - - default: return "??"; - } - #undef NORMIE_BINOP - #undef _0F -} - -const char* tb_x86_reg_name(int8_t reg, TB_X86_DataType dt) { - static const char* X86__GPR_NAMES[4][16] = { - { "al", "cl", "dl", "bl", "spl", "bpl", "sil", "dil", "r8b", "r9b", "r10b", "r11b", "r12b", "r13b", "r14b", "r15b" }, - { "ax", "cx", "dx", "bx", "sp", "bp", "si", "di", "r8w", "r9w", "r10w", "r11w", "r12w", "r13w", "r14w", "r15w" }, - { "eax", "ecx", "edx", "ebx", "esp", "ebp", "esi", "edi", "r8d", "r9d", "r10d", "r11d", "r12d", "r13d", "r14d", "r15d" }, - { "rax", "rcx", "rdx", "rbx", "rsp", "rbp", "rsi", "rdi", "r8", "r9", "r10", "r11", "r12", "r13", "r14", "r15" } - }; - - static const char* X86__HIGH_NAMES[] = { - "ah", "ch", "dh", "bh" - }; - - if (dt >= TB_X86_TYPE_BYTE && dt <= TB_X86_TYPE_QWORD) { - return X86__GPR_NAMES[dt - TB_X86_TYPE_BYTE][reg]; - } else if (dt >= TB_X86_TYPE_SSE_SS && dt <= TB_X86_TYPE_SSE_PD) { - static const char* X86__XMM_NAMES[] = { - "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", "xmm7", - "xmm8", "xmm9", "xmm10", "xmm11", "xmm12", "xmm13", "xmm14", "xmm15", - }; - - return X86__XMM_NAMES[reg]; - } else { - return "??"; - } -} - -const char* tb_x86_type_name(TB_X86_DataType dt) { - switch (dt) { - case TB_X86_TYPE_BYTE: return "byte"; - case TB_X86_TYPE_WORD: return "word"; - case TB_X86_TYPE_DWORD: return "dword"; - case TB_X86_TYPE_QWORD: return "qword"; - case TB_X86_TYPE_SSE_SS:return "dword"; - case TB_X86_TYPE_SSE_SD:return "qword"; - case TB_X86_TYPE_SSE_PS:return "xmmword"; - case TB_X86_TYPE_SSE_PD:return "xmmword"; - - default: return "??"; - } -} - -static const char* x86_fmts[] = { - " + %d", - " - %d", - "%"PRId64, - - // displacement - // " + %#x", - // " - %#x", - // immediates - // "%#"PRIx64, -}; - -static void print_memory_operand(FILE* fp, TB_X86_Inst* restrict inst) { - uint8_t base = inst->regs & 0xFF; - uint8_t index = (inst->regs >> 8) & 0xFF; - - if (inst->flags & TB_X86_INSTR_INDIRECT) { - if ((inst->regs & 0xFFFF) == 0xFFFF) { - fprintf(fp, "[rip"); - } else { - fprintf(fp, "%s [", tb_x86_type_name(inst->dt)); - if (base != 0xFF) { - fprintf(fp, "%s", tb_x86_reg_name(base, TB_X86_TYPE_QWORD)); - } - - if (index != 0xFF) { - fprintf(fp, " + %s*%d", tb_x86_reg_name(index, TB_X86_TYPE_QWORD), 1 << inst->scale); - } - } - - if (inst->disp > 0) { - fprintf(fp, x86_fmts[0], inst->disp); - } else if (inst->disp < 0) { - fprintf(fp, x86_fmts[1], -inst->disp); - } - fprintf(fp, "]"); - } else if (base != 0xFF) { - fprintf(fp, "%s", tb_x86_reg_name(base, inst->dt)); - } -} - -void tb_x86_print_inst(FILE* fp, TB_X86_Inst* inst) { - if (fp == NULL) { fp = stdout; } - - const char* mnemonic = tb_x86_mnemonic(inst); - if (inst->dt >= TB_X86_TYPE_SSE_SS && inst->dt <= TB_X86_TYPE_SSE_PD) { - static const char* strs[] = { "ss", "sd", "ps", "pd" }; - fprintf(fp, "%s", mnemonic); - fprintf(fp, "%-6s", strs[inst->dt - TB_X86_TYPE_SSE_SS]); - } else { - fprintf(fp, "%-8s", mnemonic); - } - - uint8_t rx = (inst->regs >> 16) & 0xFF; - if (inst->flags & TB_X86_INSTR_DIRECTION) { - if (rx != 255) { - fprintf(fp, "%s", tb_x86_reg_name(rx, inst->dt)); - fprintf(fp, ", "); - } - print_memory_operand(fp, inst); - } else { - print_memory_operand(fp, inst); - if (rx != 255) { - fprintf(fp, ", "); - fprintf(fp, "%s", tb_x86_reg_name(rx, inst->dt)); - } - } - - if (inst->flags & TB_X86_INSTR_IMMEDIATE) { - if (inst->regs != 0xFFFFFF) { - fprintf(fp, ", "); - } - - fprintf(fp, x86_fmts[2], inst->imm); - } - - fprintf(fp, "\n"); -} - diff --git a/vendor/tb/src/x64/x64_emitter.h b/vendor/tb/src/x64/x64_emitter.h deleted file mode 100644 index 7aefcdeb..00000000 --- a/vendor/tb/src/x64/x64_emitter.h +++ /dev/null @@ -1,328 +0,0 @@ - -static uint8_t mod_rx_rm(uint8_t mod, uint8_t rx, uint8_t rm) { - return ((mod & 3) << 6) | ((rx & 7) << 3) | (rm & 7); -} - -static uint8_t rex(bool is_64bit, uint8_t rx, uint8_t base, uint8_t index) { - return 0x40 | (is_64bit ? 8 : 0) | (base >> 3) | ((index >> 3) << 1) | ((rx >> 3) << 2); -} - -// emits 0F if (inst->cat == a) is true -#define EXT_OP(a) ((inst->cat == a) ? EMIT1(e, 0x0F) : 0) - -static void jmp(TB_CGEmitter* restrict e, int label) { - EMIT1(e, 0xE9); EMIT4(e, 0); - tb_emit_rel32(e, &e->labels[label], GET_CODE_POS(e) - 4); -} - -static void jcc(TB_CGEmitter* restrict e, Cond cc, int label) { - EMIT1(e, 0x0F); EMIT1(e, 0x80 + cc); EMIT4(e, 0); - tb_emit_rel32(e, &e->labels[label], GET_CODE_POS(e) - 4); -} - -static void emit_memory_operand(TB_CGEmitter* restrict e, uint8_t rx, const Val* a) { - // Operand encoding - if (a->type == VAL_GPR || a->type == VAL_XMM) { - EMIT1(e, mod_rx_rm(MOD_DIRECT, rx, a->reg)); - } else if (a->type == VAL_MEM) { - GPR base = a->reg, index = a->index; - Scale scale = a->scale; - int32_t disp = a->imm; - - bool needs_index = (index != GPR_NONE) || (base & 7) == RSP; - - // If it needs an index, it'll put RSP into the base slot - // and write the real base into the SIB - uint8_t mod = MOD_INDIRECT_DISP32; - if (disp == 0 && (base & 7) != RBP) mod = MOD_INDIRECT; - else if (disp == (int8_t)disp) mod = MOD_INDIRECT_DISP8; - - EMIT1(e, mod_rx_rm(mod, rx, needs_index ? RSP : base)); - if (needs_index) { - EMIT1(e, mod_rx_rm(scale, base == RSP ? RSP : index, base)); - } - - if (mod == MOD_INDIRECT_DISP8) { - EMIT1(e, (int8_t)disp); - } else if (mod == MOD_INDIRECT_DISP32) { - EMIT4(e, disp); - } - } else if (a->type == VAL_GLOBAL) { - EMIT1(e, ((rx & 7) << 3) | RBP); - EMIT4(e, a->imm); - - tb_emit_symbol_patch(e->output, a->symbol, e->count - 4); - } else { - tb_unreachable(); - } -} - -static void inst0(TB_CGEmitter* restrict e, InstType type, TB_X86_DataType dt) { - assert(type < COUNTOF(inst_table)); - const InstDesc* restrict inst = &inst_table[type]; - - if (dt == TB_X86_TYPE_QWORD) EMIT1(e, 0x48); - EXT_OP(INST_BYTE_EXT); - - if (inst->op) { - EMIT1(e, inst->op); - } else { - EMIT1(e, inst->op_i); - EMIT1(e, inst->rx_i); - } -} - -// cannot generate patches with f being NULL -static void inst1(TB_CGEmitter* restrict e, InstType type, const Val* r, TB_X86_DataType dt) { - assert(type < COUNTOF(inst_table)); - const InstDesc* restrict inst = &inst_table[type]; - - bool is_rex = (dt == TB_X86_TYPE_BYTE || dt == TB_X86_TYPE_QWORD); - bool is_rexw = dt == TB_X86_TYPE_QWORD; - - uint8_t op = inst->op_i, rx = inst->rx_i; - if (r->type == VAL_GPR) { - if (is_rex || r->reg >= 8) { - EMIT1(e, rex(is_rexw, 0x00, r->reg, 0x00)); - } - EXT_OP(INST_UNARY_EXT); - EMIT1(e, op ? op : inst->op); - EMIT1(e, mod_rx_rm(MOD_DIRECT, rx, r->reg)); - } else if (r->type == VAL_MEM) { - GPR base = r->reg, index = r->index; - Scale scale = r->scale; - int32_t disp = r->imm; - - bool needs_index = (index != GPR_NONE) || (base & 7) == RSP; - - EMIT1(e, rex(is_rexw, 0x00, base, index != GPR_NONE ? index : 0)); - EXT_OP(INST_UNARY_EXT); - EMIT1(e, op); - - // If it needs an index, it'll put RSP into the base slot - // and write the real base into the SIB - uint8_t mod = MOD_INDIRECT_DISP32; - if (disp == 0) { - mod = MOD_INDIRECT_DISP8; - } else if (disp == (int8_t)disp) { - mod = MOD_INDIRECT_DISP8; - } - - EMIT1(e, mod_rx_rm(mod, rx, needs_index ? RSP : base)); - if (needs_index) { - EMIT1(e, mod_rx_rm(scale, (base & 7) == RSP ? RSP : index, base)); - } - - if (mod == MOD_INDIRECT_DISP8) EMIT1(e, (int8_t)disp); - else if (mod == MOD_INDIRECT_DISP32) EMIT4(e, (int32_t)disp); - } else if (r->type == VAL_GLOBAL) { - if (inst->op) { - // this is a unary instruction with a REL32 variant - EXT_OP(INST_UNARY_EXT); - EMIT1(e, inst->op); - } else { - if (is_rex) EMIT1(e, is_rexw ? 0x48 : 0x40); - EXT_OP(INST_UNARY_EXT); - EMIT1(e, op); - EMIT1(e, ((rx & 7) << 3) | RBP); - } - - EMIT4(e, r->imm); - tb_emit_symbol_patch(e->output, r->symbol, e->count - 4); - } else if (r->type == VAL_LABEL) { - EXT_OP(INST_UNARY_EXT); - EMIT1(e, inst->op); - EMIT4(e, 0); - - assert(r->label >= 0 && r->label < e->label_count); - tb_emit_rel32(e, &e->labels[r->label], GET_CODE_POS(e) - 4); - } else { - tb_unreachable(); - } -} - -static void inst2(TB_CGEmitter* restrict e, InstType type, const Val* a, const Val* b, TB_X86_DataType dt) { - assert(dt >= TB_X86_TYPE_BYTE && dt <= TB_X86_TYPE_QWORD); - assert(type < COUNTOF(inst_table)); - - const InstDesc* restrict inst = &inst_table[type]; - if (type == MOVABS) { - assert(a->type == VAL_GPR && b->type == VAL_ABS); - - EMIT1(e, rex(true, 0, a->reg, 0)); - EMIT1(e, inst->op + (a->reg & 0b111)); - EMIT8(e, b->abs); - return; - } - - bool dir = b->type == VAL_MEM || b->type == VAL_GLOBAL; - if (dir || inst->op == 0x63 || inst->op == 0x69 || inst->op == 0x6E || (type >= CMOVO && type <= CMOVG) || inst->op == 0xAF || inst->cat == INST_BINOP_EXT2) { - SWAP(const Val*, a, b); - } - - // operand size - bool sz = (dt != TB_X86_TYPE_BYTE); - - // uses an immediate value that works as - // a sign extended 8 bit number - bool short_imm = (sz && b->type == VAL_IMM && b->imm == (int8_t)b->imm && inst->op_i == 0x80); - - // the destination can only be a GPR, no direction flag - bool is_gpr_only_dst = (inst->op & 1); - bool dir_flag = (dir != is_gpr_only_dst) && inst->op != 0x69; - - if (inst->cat != INST_BINOP_EXT3) { - // Address size prefix - if (dt == TB_X86_TYPE_WORD && inst->cat != INST_BINOP_EXT2) { - EMIT1(e, 0x66); - } - - assert((b->type == VAL_GPR || b->type == VAL_IMM) && "secondary operand is invalid!"); - } else { - EMIT1(e, 0x66); - } - - // REX PREFIX - // 0 1 0 0 W R X B - // ^ ^ ^ ^ - // | | | 4th bit on base. - // | | 4th bit on index. - // | 4th bit on rx. - // is 64bit? - uint8_t base = 0; - uint8_t rex_prefix = 0x40 | (dt == TB_X86_TYPE_QWORD ? 8 : 0); - if (a->type == VAL_MEM || a->type == VAL_GPR) { - base = a->reg; - } else { - base = RBP; - } - - if (a->type == VAL_MEM && a->index != GPR_NONE) { - rex_prefix |= ((a->index >> 3) << 1); - } - - uint8_t rx = (b->type == VAL_GPR || b->type == VAL_XMM) ? b->reg : inst->rx_i; - if (inst->cat == INST_BINOP_CL) { - assert(b->type == VAL_IMM || (b->type == VAL_GPR && b->reg == RCX)); - - dt = TB_X86_TYPE_BYTE; - rx = inst->rx_i; - } - - rex_prefix |= (base >> 3); - rex_prefix |= (rx >> 3) << 2; - - // if the REX stays as 0x40 then it's default and doesn't need - // to be here. - if (rex_prefix != 0x40 || dt == TB_X86_TYPE_BYTE || type == MOVZXB) { - EMIT1(e, rex_prefix); - } - - if (inst->cat == INST_BINOP_EXT3) { - // movd/movq add the ADDR16 prefix for reasons? - EMIT1(e, 0x0F); - EMIT1(e, inst->op); - } else { - // Opcode - if (inst->cat == INST_BINOP_EXT || inst->cat == INST_BINOP_EXT2) { - // DEF instructions can only be 32bit and 64bit... maybe? - if (type != XADD) sz = 0; - EMIT1(e, 0x0F); - } - - // Immediates have a custom opcode - assert((b->type != VAL_IMM || inst->op_i != 0 || inst->rx_i != 0) && "no immediate variant of instruction"); - uint8_t opcode = b->type == VAL_IMM ? inst->op_i : inst->op; - - // bottom bit usually means size, 0 for 8bit, 1 for everything else. - opcode |= sz; - - // you can't actually be flipped in the immediates because it would mean - // you're storing into an immediate so they reuse that direction bit for size. - opcode |= dir_flag << 1; - opcode |= short_imm << 1; - EMIT1(e, opcode); - } - - emit_memory_operand(e, rx, a); - // BTW memory displacements go before immediates - ptrdiff_t disp_patch = e->count - 4; - if (b->type == VAL_IMM) { - if (dt == TB_X86_TYPE_BYTE || short_imm) { - if (short_imm) { - assert(b->imm == (int8_t)b->imm); - } - - EMIT1(e, (int8_t)b->imm); - } else if (dt == TB_X86_TYPE_WORD) { - uint32_t imm = b->imm; - assert((imm & 0xFFFF0000) == 0xFFFF0000 || (imm & 0xFFFF0000) == 0); - - EMIT2(e, imm); - } else { - EMIT4(e, (int32_t)b->imm); - } - } - - if (a->type == VAL_GLOBAL && disp_patch + 4 != e->count) { - RELOC4(e, disp_patch, (disp_patch + 4) - e->count); - } -} - -static void inst2sse(TB_CGEmitter* restrict e, InstType type, const Val* a, const Val* b, TB_X86_DataType dt) { - assert(type < COUNTOF(inst_table)); - const InstDesc* restrict inst = &inst_table[type]; - - // most SSE instructions (that aren't mov__) are mem src only - bool supports_mem_dst = (type == FP_MOV); - bool dir = is_value_mem(a); - - bool packed = (dt == TB_X86_TYPE_SSE_PS || dt == TB_X86_TYPE_SSE_PD); - bool is_double = (dt == TB_X86_TYPE_SSE_PD || dt == TB_X86_TYPE_SSE_SD); - - if (supports_mem_dst && dir) { - SWAP(const Val*, a, b); - } - - uint8_t rx = a->reg; - uint8_t base, index; - if (b->type == VAL_MEM) { - base = b->reg; - index = b->index != GPR_NONE ? b->index : 0; - } else if (b->type == VAL_XMM || b->type == VAL_GPR) { - base = b->reg; - index = 0; - } else if (b->type == VAL_GLOBAL) { - base = 0; - index = 0; - } else { - tb_todo(); - } - - if (type != FP_XOR && type != FP_AND && type != FP_OR) { - if (!packed && type != FP_UCOMI) { - EMIT1(e, is_double ? 0xF2 : 0xF3); - } else if (is_double) { - // packed double - EMIT1(e, 0x66); - } - } - - if (type == FP_CVT64 || rx >= 8 || base >= 8 || index >= 8) { - EMIT1(e, rex(false, rx, base, index)); - } - - // extension prefix - EMIT1(e, 0x0F); - EMIT1(e, inst->op + (supports_mem_dst ? dir : 0)); - emit_memory_operand(e, rx, b); -} - -static void x86_jmp(TB_CGEmitter* restrict e, Val label) { - inst1(e, JMP, &label, TB_X86_TYPE_QWORD); -} - -static void x86_jcc(TB_CGEmitter* restrict e, int cc, Val label) { - inst1(e, JO + cc, &label, TB_X86_TYPE_QWORD); -} - diff --git a/vendor/tb/src/x64/x64_insts.inc b/vendor/tb/src/x64/x64_insts.inc deleted file mode 100644 index 2e87730e..00000000 --- a/vendor/tb/src/x64/x64_insts.inc +++ /dev/null @@ -1,138 +0,0 @@ -// our precious x86 assembly ops -// -// mnemonic category fields -X(RET, "ret", BYTE, 0xC3) -X(INT3, "int3", BYTE, 0xCC) -X(NOP, "nop", BYTE, 0x90) -X(STOSB, "rep stosb", BYTE, 0xAA) -X(MOVSB, "rep movsb", BYTE, 0xA4) -X(CAST, "cvt", BYTE, 0x99) -X(SYSCALL, "syscall", BYTE_EXT, 0x05) -X(RDTSC, "rdtsc", BYTE_EXT, 0x31) -X(UD2, "ud2", BYTE_EXT, 0x0B) - -X(NOT, "not", UNARY, .op_i = 0xF7, 0x02) -X(NEG, "neg", UNARY, .op_i = 0xF7, 0x03) -X(MUL, "mul", UNARY, .op_i = 0xF7, 0x04) -X(DIV, "div", UNARY, .op_i = 0xF7, 0x06) -X(IDIV, "idiv", UNARY, .op_i = 0xF7, 0x07) -X(CALL, "call", UNARY, 0xE8, 0xFF, 0x02) -X(JMP, "jmp", UNARY, 0xE9, 0xFF, 0x04) - -// prefetching -X(PREFETCHNTA,"prefetchnta",UNARY_EXT, .op_i = 0x18, 0) -X(PREFETCH0, "prefetch0", UNARY_EXT, .op_i = 0x18, 1) -X(PREFETCH1, "prefetch1", UNARY_EXT, .op_i = 0x18, 2) -X(PREFETCH2, "prefetch2", UNARY_EXT, .op_i = 0x18, 3) - -// Jcc -X(JO, "jo", UNARY_EXT, 0x80) -X(JNO, "jno", UNARY_EXT, 0x81) -X(JB, "jb", UNARY_EXT, 0x82) -X(JNB, "jnb", UNARY_EXT, 0x83) -X(JE, "je", UNARY_EXT, 0x84) -X(JNE, "jne", UNARY_EXT, 0x85) -X(JBE, "jbe", UNARY_EXT, 0x86) -X(JA, "ja", UNARY_EXT, 0x87) -X(JS, "js", UNARY_EXT, 0x88) -X(JNS, "jns", UNARY_EXT, 0x89) -X(JP, "jp", UNARY_EXT, 0x8A) -X(JNP, "jnp", UNARY_EXT, 0x8B) -X(JL, "jl", UNARY_EXT, 0x8C) -X(JGE, "jge", UNARY_EXT, 0x8D) -X(JLE, "jle", UNARY_EXT, 0x8E) -X(JG, "jg", UNARY_EXT, 0x8F) - -// SETcc -X(SETO, "seto", UNARY_EXT, 0x90) -X(SETNO, "setno", UNARY_EXT, 0x91) -X(SETB, "setb", UNARY_EXT, 0x92) -X(SETNB, "setnb", UNARY_EXT, 0x93) -X(SETE, "sete", UNARY_EXT, 0x94) -X(SETNE, "setne", UNARY_EXT, 0x95) -X(SETBE, "setbe", UNARY_EXT, 0x96) -X(SETA, "seta", UNARY_EXT, 0x97) -X(SETS, "sets", UNARY_EXT, 0x98) -X(SETNS, "setns", UNARY_EXT, 0x99) -X(SETP, "setp", UNARY_EXT, 0x9A) -X(SETNP, "setnp", UNARY_EXT, 0x9B) -X(SETL, "setl", UNARY_EXT, 0x9C) -X(SETGE, "setge", UNARY_EXT, 0x9D) -X(SETLE, "setle", UNARY_EXT, 0x9E) -X(SETG, "setg", UNARY_EXT, 0x9F) - -// CMOVcc -X(CMOVO, "cmovo", BINOP_EXT, 0x40) -X(CMOVNO, "cmovno", BINOP_EXT, 0x41) -X(CMOVB, "cmovb", BINOP_EXT, 0x42) -X(CMOVNB, "cmovnb", BINOP_EXT, 0x43) -X(CMOVE, "cmove", BINOP_EXT, 0x44) -X(CMOVNE, "cmovne", BINOP_EXT, 0x45) -X(CMOVBE, "cmovbe", BINOP_EXT, 0x46) -X(CMOVA, "cmova", BINOP_EXT, 0x47) -X(CMOVS, "cmovs", BINOP_EXT, 0x48) -X(CMOVNS, "cmovns", BINOP_EXT, 0x49) -X(CMOVP, "cmovp", BINOP_EXT, 0x4A) -X(CMOVNP, "cmovnp", BINOP_EXT, 0x4B) -X(CMOVL, "cmovl", BINOP_EXT, 0x4C) -X(CMOVGE, "cmovge", BINOP_EXT, 0x4D) -X(CMOVLE, "cmovle", BINOP_EXT, 0x4E) -X(CMOVG, "cmovg", BINOP_EXT, 0x4F) - -// bitmagic -X(BSF, "bsf", BINOP_EXT, 0xBC) -X(BSR, "bsr", BINOP_EXT, 0xBD) - -// binary ops but they have an implicit CL on the righthand side -X(SHL, "shl", BINOP_CL, 0xD2, 0xC0, 0x04) -X(SHR, "shr", BINOP_CL, 0xD2, 0xC0, 0x05) -X(ROL, "rol", BINOP_CL, 0xD2, 0xC0, 0x00) -X(ROR, "ror", BINOP_CL, 0xD2, 0xC0, 0x01) -X(SAR, "sar", BINOP_CL, 0xD2, 0xC0, 0x07) - -X(ADD, "add", BINOP, 0x00, 0x80, 0x00) -X(OR, "or", BINOP, 0x08, 0x80, 0x01) -X(AND, "and", BINOP, 0x20, 0x80, 0x04) -X(SUB, "sub", BINOP, 0x28, 0x80, 0x05) -X(XOR, "xor", BINOP, 0x30, 0x80, 0x06) -X(CMP, "cmp", BINOP, 0x38, 0x80, 0x07) -X(MOV, "mov", BINOP, 0x88, 0xC6, 0x00) -X(TEST, "test", BINOP, 0x84, 0xF6, 0x00) - -// misc integer ops -X(MOVABS, "mov", BINOP_PLUS, 0xB8) -X(XCHG, "xchg", BINOP, 0x86) -X(LEA, "lea", BINOP, 0x8D) -X(XADD, "xadd", BINOP_EXT, 0xC0) -X(IMUL, "imul", BINOP_EXT, 0xAF) -X(IMUL3, "imul", BINOP, 0x69) -X(MOVSXB, "movsxb", BINOP_EXT2, 0xBE) -X(MOVSXW, "movsxw", BINOP_EXT2, 0xBF) -X(MOVSXD, "movsxd", BINOP, 0x63) -X(MOVZXB, "movzxb", BINOP_EXT2, 0xB6) -X(MOVZXW, "movzxw", BINOP_EXT2, 0xB7) - -// gpr<->xmm -X(MOV_I2F, "mov", BINOP_EXT3, 0x6E) -X(MOV_F2I, "mov", BINOP_EXT3, 0x7E) - -// SSE binops -X(FP_MOV, "mov", BINOP_SSE, 0x10) -X(FP_ADD, "add", BINOP_SSE, 0x58) -X(FP_MUL, "mul", BINOP_SSE, 0x59) -X(FP_SUB, "sub", BINOP_SSE, 0x5C) -X(FP_MIN, "min", BINOP_SSE, 0x5D) -X(FP_DIV, "div", BINOP_SSE, 0x5E) -X(FP_MAX, "max", BINOP_SSE, 0x5F) -X(FP_CMP, "cmp", BINOP_SSE, 0xC2) -X(FP_UCOMI, "ucomi", BINOP_SSE, 0x2E) -X(FP_CVT32, "cvtsi", BINOP_SSE, 0x2A) -X(FP_CVT64, "cvtsi", BINOP_SSE, 0x2A) -X(FP_CVT, "cvt", BINOP_SSE, 0x5A) -X(FP_CVTT, "cvtt", BINOP_SSE, 0x2C) -X(FP_SQRT, "sqrt", BINOP_SSE, 0x51) -X(FP_RSQRT, "rsqrt", BINOP_SSE, 0x52) -X(FP_AND, "and", BINOP_SSE, 0x54) -X(FP_OR, "or", BINOP_SSE, 0x56) -X(FP_XOR, "xor", BINOP_SSE, 0x57) -#undef X diff --git a/vendor/tb/src/x64/x64_target.c b/vendor/tb/src/x64/x64_target.c deleted file mode 100644 index 3bc20913..00000000 --- a/vendor/tb/src/x64/x64_target.c +++ /dev/null @@ -1,2177 +0,0 @@ -#include "x64.h" -#include -#include "x64_emitter.h" -#include "x64_disasm.c" - -#ifdef TB_HAS_X64 -enum { - // register classes - REG_CLASS_GPR = 1, - REG_CLASS_XMM, - REG_CLASS_COUNT, -}; - -#include "../codegen_impl.h" - -enum { - // OP reg, imm - TILE_HAS_IMM = 1, - - // mov rax, [LOCAL/GLOBAL] - TILE_FOLDED_BASE = 2, - - TILE_INDEXED = 4, - - // cmp a, b - // jcc cond - TILE_FOLDED_CMP = 8, -}; - -typedef struct { - TB_Node* base; - int stride; - int offset; -} AuxAddress; - -typedef struct { - int64_t min, max; - bool if_chain; -} AuxBranch; - -static const struct ParamDesc { - int chkstk_limit; - int gpr_count; - int xmm_count; - uint16_t caller_saved_xmms; // XMM0 - XMMwhatever - uint16_t caller_saved_gprs; // bitfield - - GPR gprs[6]; -} param_descs[] = { - // win64 - { 4096, 4, 4, 6, WIN64_ABI_CALLER_SAVED, { RCX, RDX, R8, R9, 0, 0 } }, - // system v - { INT_MAX, 6, 4, 5, SYSV_ABI_CALLER_SAVED, { RDI, RSI, RDX, RCX, R8, R9 } }, - // syscall - { INT_MAX, 6, 4, 5, SYSCALL_ABI_CALLER_SAVED, { RDI, RSI, RDX, R10, R8, R9 } }, -}; - -enum { - NO_RCX = ~((1 << RCX)), -}; - -// *out_mask of 0 means no mask -static TB_X86_DataType legalize_int(TB_DataType dt, uint64_t* out_mask) { - assert(dt.type == TB_INT || dt.type == TB_PTR); - if (dt.type == TB_PTR) return *out_mask = 0, TB_X86_TYPE_QWORD; - - TB_X86_DataType t = TB_X86_TYPE_NONE; - int bits = 0; - - if (dt.data <= 8) bits = 8, t = TB_X86_TYPE_BYTE; - else if (dt.data <= 16) bits = 16, t = TB_X86_TYPE_WORD; - else if (dt.data <= 32) bits = 32, t = TB_X86_TYPE_DWORD; - else if (dt.data <= 64) bits = 64, t = TB_X86_TYPE_QWORD; - - assert(bits != 0 && "TODO: large int support"); - assert(dt.data != 0); - uint64_t mask = ~UINT64_C(0) >> (64 - dt.data); - - *out_mask = (dt.data == bits) ? 0 : mask; - return t; -} - -static TB_X86_DataType legalize_int2(TB_DataType dt) { - uint64_t m; - return legalize_int(dt, &m); -} - -static TB_X86_DataType legalize_float(TB_DataType dt) { - assert(dt.type == TB_FLOAT); - return (dt.data == TB_FLT_64 ? TB_X86_TYPE_SSE_SD : TB_X86_TYPE_SSE_SS); -} - -static TB_X86_DataType legalize(TB_DataType dt) { - if (dt.type == TB_FLOAT) { - return legalize_float(dt); - } else { - uint64_t m; - return legalize_int(dt, &m); - } -} - -static bool fits_into_int32(uint64_t x) { - uint32_t hi = x >> 32ull; - return hi == 0 || hi == 0xFFFFFFFF; -} - -static bool try_for_imm32(int bits, TB_Node* n, int32_t* out_x) { - if (n->type != TB_INTEGER_CONST) { - return false; - } - - TB_NodeInt* i = TB_NODE_GET_EXTRA(n); - if (bits > 32) { - bool sign = (i->value >> 31ull) & 1; - uint64_t top = i->value >> 32ull; - - // if the sign matches the rest of the top bits, we can sign extend just fine - if (top != (sign ? 0xFFFFFFFF : 0)) { - return false; - } - } - - *out_x = i->value; - return true; -} - -static bool _2addr(TB_Node* n) { - return n->type >= TB_AND && n->type <= TB_CMP_FLE; -} - -static void init_ctx(Ctx* restrict ctx, TB_ABI abi) { - ctx->sched = greedy_scheduler; - ctx->_2addr = _2addr; - // ctx->regalloc = tb__chaitin; - ctx->regalloc = tb__lsra; - - ctx->abi_index = abi == TB_ABI_SYSTEMV ? 1 : 0; - - // currently only using 16 GPRs and 16 XMMs, AVX gives us - // 32 YMMs (which double as XMMs) and later on APX will do - // 32 GPRs. - ctx->num_regs[REG_CLASS_GPR] = 16; - ctx->num_regs[REG_CLASS_XMM] = 16; - - uint16_t all_gprs = 0xFFFF & ~(1 << RSP); - if (ctx->features.gen & TB_FEATURE_FRAME_PTR) { - all_gprs &= ~(1 << RBP); - ctx->stack_header = 16; - } else { - ctx->stack_header = 8; - } - - ctx->normie_mask[REG_CLASS_GPR] = REGMASK(GPR, all_gprs); - ctx->normie_mask[REG_CLASS_XMM] = REGMASK(XMM, (1u << 16) - 1); - - // mark GPR callees (technically includes RSP but since it's - // never conventionally allocated we should never run into issues) - ctx->callee_saved[REG_CLASS_GPR] = ~param_descs[ctx->abi_index].caller_saved_gprs; - - // mark XMM callees - ctx->callee_saved[REG_CLASS_XMM] = 0; - FOREACH_N(i, param_descs[ctx->abi_index].caller_saved_xmms, 16) { - ctx->callee_saved[REG_CLASS_XMM] |= (1ull << i); - } - - TB_FunctionPrototype* proto = ctx->f->prototype; - TB_Node** params = ctx->f->params; - TB_Node* root_ctrl = params[0]; - - FOREACH_N(i, 0, proto->param_count) { - TB_Node* proj = params[3 + i]; - User* use = proj->users; - if (use == NULL || use->next != NULL || use->slot == 0) { - continue; - } - - TB_Node* store_op = use->n; - if (store_op->type != TB_STORE || store_op->inputs[0] != root_ctrl) { - continue; - } - - TB_Node* addr = store_op->inputs[2]; - if (addr->type != TB_LOCAL) { - continue; - } - - int pos = ctx->stack_header + (i * 8); - nl_map_put(ctx->stack_slots, addr, -pos); - - /*if (i >= 4 && ctx->abi_index == 0) { - // marks as visited (stores don't return so we can -1) - - }*/ - } - - ctx->stack_usage += ctx->stack_header + (proto->param_count * 8); - - if (proto->has_varargs) { - const GPR* parameter_gprs = param_descs[ctx->abi_index].gprs; - - // spill the rest of the parameters (assumes they're all in the GPRs) - size_t gpr_count = param_descs[ctx->abi_index].gpr_count; - size_t extra_param_count = proto->param_count > gpr_count ? 0 : gpr_count - proto->param_count; - - ctx->stack_usage += (extra_param_count * 8); - } -} - -static RegMask normie_mask(Ctx* restrict ctx, TB_DataType dt) { - return ctx->normie_mask[dt.type == TB_FLOAT ? REG_CLASS_XMM : REG_CLASS_GPR]; -} - -// returns true if it should split -static bool addr_split_heuristic(int arr_uses, int stride, int scale) { - // doesn't matter if we do *1 *2 *4 *8, all - // basically just an LEA. once we leave LEA - // levels we need to do explicit ops with regs - // which increases pressure. - int cost = 0; - if (stride != 1 << scale || scale >= 4) { - cost = 3; - } else { - cost = 1; - } - - return cost*arr_uses > 10; -} - -// not TLS -static bool simple_symbol(TB_Node* n) { - if (n->type != TB_SYMBOL) return false; - - TB_Symbol* sym = TB_NODE_GET_EXTRA_T(n, TB_NodeSymbol)->sym; - if (sym->tag != TB_SYMBOL_GLOBAL) return true; - - TB_Global* g = (TB_Global*) sym; - return (sym->module->sections[g->parent].flags & TB_MODULE_SECTION_TLS) == 0; -} - -static bool is_tls_symbol(TB_Symbol* sym) { - if (sym->tag == TB_SYMBOL_GLOBAL) { - TB_Global* g = (TB_Global*) sym; - return sym->module->sections[g->parent].flags & TB_MODULE_SECTION_TLS; - } else { - return false; - } -} - -// x86 can do a lot of fancy address computation crap in one operand so we'll -// track that tiling here. -// -// in_count is all the inputs that go alongside this operand -static TileInput* isel_addr(Ctx* restrict ctx, Tile* t, TB_Node* og, TB_Node* n, int extra_cnt) { - int32_t offset = 0; - TB_Node* base = n; - TB_Node* index = NULL; - int64_t stride = 0; - bool has_tmp = 0; - - if (base->type == TB_MEMBER_ACCESS) { - offset = TB_NODE_GET_EXTRA_T(n, TB_NodeMember)->offset; - base = base->inputs[1]; - } - - // we don't wanna - if (base->type == TB_ARRAY_ACCESS) { - stride = TB_NODE_GET_EXTRA_T(base, TB_NodeArray)->stride; - int scale = tb_ffs(stride) - 1; - - if (og == base || !addr_split_heuristic(val_at(ctx, base)->use_count, stride, scale)) { - index = base->inputs[2]; - base = base->inputs[1]; - - if (stride == 1) { - // no scaling required - } else if (stride == 1u << scale) { - // we can only fit a 2bit shift amount in an LEA, after - // that we just defer to an explicit shift op. - if (scale > 3) { - has_tmp = og->type != TB_LOAD || og->dt.type == TB_FLOAT; - } - } else { - // needs a proper multiply (we may wanna invest in a few special patterns - // for reducing simple multiplies into shifts) - // - // a * 24 => (a * 8) * 3 - // b * 3 => b<<1 + b - // - // thus - // - // LEA b, [a * 8] - // LEA dst, [b * 2 + b] - has_tmp = true; - } - } else { - stride = 0; - } - } - - int in_cap = extra_cnt + (index?1:0) + has_tmp; - if (!(base->type == TB_LOCAL || (simple_symbol(base) && index == NULL))) { - in_cap += 1; - } - - // construct tile now - t->ins = tb_arena_alloc(tmp_arena, in_cap * sizeof(TileInput)); - t->in_count = in_cap; - - int in_count = 0; - if (base->type == TB_LOCAL) { - try_init_stack_slot(ctx, base); - t->flags |= TILE_FOLDED_BASE; - } else if (simple_symbol(base) && index == NULL) { - t->flags |= TILE_FOLDED_BASE; - } else { - t->ins[in_count].src = get_interval(ctx, base, 0); - t->ins[in_count].mask = ctx->normie_mask[REG_CLASS_GPR]; - in_count++; - } - - if (index) { - t->ins[in_count].src = get_interval(ctx, index, 0); - t->ins[in_count].mask = ctx->normie_mask[REG_CLASS_GPR]; - t->flags |= TILE_INDEXED; - in_count++; - } - - if (has_tmp) { - t->ins[in_count].src = NULL; - t->ins[in_count].mask = ctx->normie_mask[REG_CLASS_GPR]; - in_count++; - } - - AuxAddress* aux = tb_arena_alloc(tmp_arena, sizeof(AuxAddress)); - aux->base = base; - aux->stride = stride; - aux->offset = offset; - t->aux = aux; - - return &t->ins[in_cap - extra_cnt]; -} - -static int reg_count(Ctx* restrict ctx, TB_Node* n) { - if (n->dt.type == TB_INT) { - return 1; - } else if (n->dt.type == TB_PTR) { - return 1; - } else if (n->dt.type == TB_FLOAT) { - return 1; - } else { - return 0; - } -} - -#define OUT1(m) (dst->outs[0]->dt = n->dt, dst->outs[0]->mask = (m)) -static void isel_node(Ctx* restrict ctx, Tile* dst, TB_Node* n) { - switch (n->type) { - // no inputs - case TB_REGION: - case TB_NATURAL_LOOP: - case TB_AFFINE_LOOP: - case TB_ROOT: - case TB_TRAP: - case TB_CALLGRAPH: - case TB_SPLITMEM: - case TB_MERGEMEM: - case TB_UNREACHABLE: - case TB_DEBUGBREAK: - case TB_INTEGER_CONST: - case TB_FLOAT32_CONST: - case TB_FLOAT64_CONST: - case TB_POISON: - break; - - case TB_SYMBOL: { - TB_Symbol* sym = TB_NODE_GET_EXTRA_T(n, TB_NodeSymbol)->sym; - if (is_tls_symbol(sym)) { - // on windows we'll need one temporary, linux needs none - if (ctx->abi_index == 0) { - dst->ins = tb_arena_alloc(tmp_arena, 1 * sizeof(TileInput)); - dst->in_count = 1; - dst->ins[0].mask = ctx->normie_mask[REG_CLASS_GPR]; - dst->ins[0].src = NULL; - } else { - dst->ins = NULL; - dst->in_count = 0; - } - } - break; - } - - case TB_INLINE_ASM: { - TB_NodeInlineAsm* a = TB_NODE_GET_EXTRA_T(n, TB_NodeInlineAsm); - // a->ra(n, a->ctx, tmp_arena); - tb_todo(); - break; - } - - case TB_LOCAL: { - TB_NodeLocal* local = TB_NODE_GET_EXTRA(n); - isel_addr(ctx, dst, n, n, 0); - break; - } - - case TB_VA_START: { - assert(ctx->module->target_abi == TB_ABI_WIN64 && "How does va_start even work on SysV?"); - - // on Win64 va_start just means whatever is one parameter away from - // the parameter you give it (plus in Win64 the parameters in the stack - // are 8bytes, no fanciness like in SysV): - // void printf(const char* fmt, ...) { - // va_list args; - // va_start(args, fmt); // args = ((char*) &fmt) + 8; - // ... - // } - break; - } - - case TB_LOAD: - case TB_READ: - isel_addr(ctx, dst, n, n->inputs[2], 0); - break; - - case TB_ARRAY_ACCESS: - case TB_MEMBER_ACCESS: - isel_addr(ctx, dst, n, n, 0); - break; - - case TB_CYCLE_COUNTER: { - dst->ins = tb_arena_alloc(tmp_arena, 2 * sizeof(TileInput)); - dst->in_count = 2; - dst->ins[0].mask = REGMASK(GPR, 1 << RAX); - dst->ins[1].mask = REGMASK(GPR, 1 << RDX); - dst->ins[0].src = NULL; - dst->ins[1].src = NULL; - OUT1(REGMASK(GPR, 1 << RAX)); - return; - } - - case TB_WRITE: - case TB_STORE: { - TileInput* ins = isel_addr(ctx, dst, n, n->inputs[2], 1); - ins[0].src = get_interval(ctx, n->inputs[3], 0); - ins[0].mask = normie_mask(ctx, n->inputs[3]->dt); - break; - } - - case TB_SIGN_EXT: - case TB_ZERO_EXT: - if (n->inputs[1]->type == TB_LOAD) { - isel_addr(ctx, dst, n, n->inputs[1]->inputs[2], 0); - } else { - tile_broadcast_ins(ctx, dst, n, 1, 2, normie_mask(ctx, n->inputs[1]->dt)); - } - break; - - case TB_BITCAST: - case TB_TRUNCATE: - case TB_FLOAT_EXT: - case TB_INT2FLOAT: - case TB_FLOAT2INT: - case TB_UINT2FLOAT: - case TB_FLOAT2UINT: - tile_broadcast_ins(ctx, dst, n, 1, 2, normie_mask(ctx, n->inputs[1]->dt)); - break; - - case TB_PHI: - if (n->dt.type == TB_INT || n->dt.type == TB_PTR || n->dt.type == TB_FLOAT) { - RegMask rm = normie_mask(ctx, n->dt); - rm.may_spill = true; - OUT1(rm); - } - return; - - case TB_RETURN: { - static int ret_gprs[2] = { RAX, RDX }; - - int rets = n->input_count - 3; - TileInput* ins = tile_set_ins(ctx, dst, n, 3, n->input_count); - - assert(rets <= 2 && "At most 2 return values :("); - FOREACH_N(i, 0, rets) { - TB_DataType dt = n->inputs[3+i]->dt; - if (dt.type == TB_FLOAT) { - ins[i].mask = REGMASK(XMM, 1 << i); - } else { - ins[i].mask = REGMASK(GPR, 1 << ret_gprs[i]); - } - } - return; - } - - case TB_PROJ: { - if (dst->out_count) { - RegMask rm = { 0 }; - int i = TB_NODE_GET_EXTRA_T(n, TB_NodeProj)->index; - - if (n->inputs[0]->type == TB_ROOT) { - // function params are ABI crap - const struct ParamDesc* params = ¶m_descs[ctx->abi_index]; - if (i == 2) { - assert(0 && "tf are you doing with the RPC?"); - } else if (i >= 3) { - if (n->dt.type == TB_FLOAT) { - rm = REGMASK(XMM, 1u << (i - 3)); - } else { - rm = REGMASK(GPR, 1u << params->gprs[i - 3]); - } - } - } else if (n->inputs[0]->type == TB_CALL || n->inputs[0]->type == TB_SYSCALL) { - if (n->dt.type == TB_FLOAT) { - if (i >= 2) rm = REGMASK(XMM, 1 << (i - 2)); - } else { - if (i == 2) rm = REGMASK(GPR, 1 << RAX); - else if (i == 3) rm = REGMASK(GPR, 1 << RDX); - } - } else { - tb_todo(); - } - - OUT1(rm); - } - return; - } - - // unary ops - case TB_NOT: - tile_broadcast_ins(ctx, dst, n, 1, n->input_count, ctx->normie_mask[REG_CLASS_GPR]); - break; - - case TB_CMP_EQ: - case TB_CMP_NE: - case TB_CMP_SLT: - case TB_CMP_SLE: - case TB_CMP_ULT: - case TB_CMP_ULE: - case TB_CMP_FLT: - case TB_CMP_FLE: { - TB_Node* cmp = n->inputs[1]; - TB_DataType cmp_dt = TB_NODE_GET_EXTRA_T(n->inputs[1], TB_NodeCompare)->cmp_dt; - - int cap = 1; - if (cmp->type >= TB_CMP_EQ && cmp->type <= TB_CMP_FLE) { - dst->flags |= TILE_FOLDED_CMP; - - int32_t x; - if (!try_for_imm32(cmp_dt.type == TB_PTR ? 64 : cmp_dt.data, cmp->inputs[2], &x)) { - cap += 1; - } else { - dst->flags |= TILE_HAS_IMM; - } - } - - RegMask rm = normie_mask(ctx, n->dt); - dst->ins = tb_arena_alloc(tmp_arena, cap * sizeof(TileInput)); - dst->in_count = cap; - - int in_count = 0; - if (dst->flags & TILE_FOLDED_CMP) { - RegMask rm = normie_mask(ctx, cmp_dt); - dst->ins[0].src = get_interval(ctx, cmp->inputs[1], 0); - dst->ins[0].mask = rm; - in_count++; - - if ((dst->flags & TILE_HAS_IMM) == 0) { - dst->ins[1].src = get_interval(ctx, cmp->inputs[2], 0); - dst->ins[1].mask = rm; - in_count++; - } - } else { - dst->ins[0].src = get_interval(ctx, cmp, 0); - dst->ins[0].mask = ctx->normie_mask[REG_CLASS_GPR]; - in_count++; - } - break; - } - - case TB_SELECT: { - TB_Node* cmp = n->inputs[1]; - TB_DataType cmp_dt = TB_NODE_GET_EXTRA_T(n->inputs[1], TB_NodeCompare)->cmp_dt; - - int cap = 3; - if (cmp->type >= TB_CMP_EQ && cmp->type <= TB_CMP_FLE) { - dst->flags |= TILE_FOLDED_CMP; - - int32_t x; - if (!try_for_imm32(cmp_dt.type == TB_PTR ? 64 : cmp_dt.data, cmp->inputs[2], &x)) { - cap += 1; - } else { - dst->flags |= TILE_HAS_IMM; - } - } - - RegMask rm = normie_mask(ctx, n->dt); - dst->ins = tb_arena_alloc(tmp_arena, cap * sizeof(TileInput)); - dst->in_count = cap; - - int in_count = 0; - if (dst->flags & TILE_FOLDED_CMP) { - RegMask rm = normie_mask(ctx, cmp_dt); - dst->ins[0].src = get_interval(ctx, cmp->inputs[1], 0); - dst->ins[0].mask = rm; - in_count++; - - if ((dst->flags & TILE_HAS_IMM) == 0) { - dst->ins[1].src = get_interval(ctx, cmp->inputs[2], 0); - dst->ins[1].mask = rm; - in_count++; - } - } else { - dst->ins[0].src = get_interval(ctx, cmp, 0); - dst->ins[0].mask = ctx->normie_mask[REG_CLASS_GPR]; - in_count++; - } - - dst->ins[in_count].src = get_interval(ctx, n->inputs[2], 0); - dst->ins[in_count].mask = rm; - in_count++; - - dst->ins[in_count].src = get_interval(ctx, n->inputs[3], 0); - dst->ins[in_count].mask = rm; - in_count++; - break; - } - - // binary ops - case TB_AND: - case TB_OR: - case TB_XOR: - case TB_ADD: - case TB_SUB: - case TB_MUL: { - int32_t x; - if (try_for_imm32(n->dt.data, n->inputs[2], &x)) { - tile_broadcast_ins(ctx, dst, n, 1, 2, ctx->normie_mask[REG_CLASS_GPR]); - dst->flags |= TILE_HAS_IMM; - } else { - TileInput* ins = tile_set_ins(ctx, dst, n, 1, 3); - ins[0].mask = ctx->normie_mask[REG_CLASS_GPR]; - ins[1].mask = ctx->normie_mask[REG_CLASS_GPR]; - ins[1].mask.may_spill = true; - } - break; - } - - case TB_SHL: - case TB_SHR: - case TB_ROL: - case TB_ROR: - case TB_SAR: { - int32_t x; - if (try_for_imm32(n->inputs[2]->dt.data, n->inputs[2], &x) && x >= 0 && x < 64) { - tile_broadcast_ins(ctx, dst, n, 1, 2, ctx->normie_mask[REG_CLASS_GPR]); - dst->flags |= TILE_HAS_IMM; - } else { - TileInput* ins = tile_set_ins(ctx, dst, n, 1, 3); - ins[0].mask = REGMASK(GPR, ctx->normie_mask[REG_CLASS_GPR].mask & NO_RCX); - ins[1].mask = REGMASK(GPR, 1 << RCX); - } - break; - } - - case TB_UDIV: - case TB_SDIV: - case TB_UMOD: - case TB_SMOD: { - dst->ins = tb_arena_alloc(tmp_arena, 3 * sizeof(TileInput)); - dst->in_count = 3; - dst->ins[0].mask = REGMASK(GPR, 1 << RAX); - dst->ins[1].mask = ctx->normie_mask[REG_CLASS_GPR]; - dst->ins[2].mask = REGMASK(GPR, 1 << RDX); - dst->ins[0].src = get_interval(ctx, n->inputs[1], 0); - dst->ins[1].src = get_interval(ctx, n->inputs[2], 0); - dst->ins[2].src = NULL; - - if (n->type == TB_UDIV || n->type == TB_SDIV) { - OUT1(REGMASK(GPR, 1 << RAX)); - } else { - OUT1(REGMASK(GPR, 1 << RDX)); - } - break; - } - - case TB_FADD: - case TB_FSUB: - case TB_FMUL: - case TB_FDIV: - case TB_FMIN: - case TB_FMAX: - tile_broadcast_ins(ctx, dst, n, 1, n->input_count, ctx->normie_mask[REG_CLASS_XMM]); - break; - - case TB_BRANCH: { - TB_Node* cmp = n->inputs[1]; - TB_NodeBranch* br = TB_NODE_GET_EXTRA(n); - - AuxBranch* aux = NULL; - int ins = 1, tmps = 0; - if (br->succ_count > 2) { - // try for jump tables or if-chains - // - // check if there's at most only one space between entries - int64_t last = br->keys[0].key; - int64_t min = last, max = last; - - double dist_avg = 0; - double inv_succ_count = 1.0 / (br->succ_count - 2); - - bool large_num = false; - FOREACH_N(i, 2, br->succ_count) { - int64_t key = br->keys[i - 1].key; - if (!fits_into_int32(key)) { - large_num = true; - } - - min = (min > key) ? key : min; - max = (max > key) ? max : key; - - dist_avg += (key - last) * inv_succ_count; - last = key; - } - - // if there's no default case we can skew heuristics around the lack of range check - bool has_default = false; - FOR_USERS(u, n) { - if (u->n->type != TB_PROJ) continue; - int index = TB_NODE_GET_EXTRA_T(u->n, TB_NodeProj)->index; - if (index == 0) { - has_default = cfg_next_control(u->n)->type != TB_UNREACHABLE; - break; - } - } - - int64_t range = (max - min) + 1; - - // if we do if-else chains we'll do 1+2c ops (c is the number of cases). - int64_t if_chain_cost = 1 + 2*range; - // if we do jump table it's 6 ops + a table that's got [max-min] entries but cost - // wise the issue is slots which are missed (go to fallthru). - int64_t jmp_table_cost = has_default ? 6 : 4; - jmp_table_cost += (range - (range / dist_avg)); - - aux = tb_arena_alloc(tmp_arena, sizeof(AuxBranch)); - aux->min = min; - aux->max = max; - aux->if_chain = if_chain_cost < jmp_table_cost; - - if (aux->if_chain) { - // large numbers require a temporary to store the immediate - tmps += large_num; - } else { - // we need tmp for the key (either offset or casted) - tmps += 3; - } - } else { - if (cmp->type >= TB_CMP_EQ && cmp->type <= TB_CMP_FLE) { - TB_DataType cmp_dt = TB_NODE_GET_EXTRA_T(n->inputs[1], TB_NodeCompare)->cmp_dt; - dst->flags |= TILE_FOLDED_CMP; - - int32_t x; - if (!try_for_imm32(cmp_dt.type == TB_PTR ? 64 : cmp_dt.data, cmp->inputs[2], &x)) { - ins += 1; - } else { - dst->flags |= TILE_HAS_IMM; - } - } - } - - dst->ins = tb_arena_alloc(tmp_arena, (ins+tmps) * sizeof(TileInput)); - dst->in_count = ins+tmps; - dst->aux = aux; - - if (dst->flags & TILE_FOLDED_CMP) { - TB_DataType cmp_dt = TB_NODE_GET_EXTRA_T(cmp, TB_NodeCompare)->cmp_dt; - - RegMask rm = normie_mask(ctx, cmp_dt); - dst->ins[0].src = get_interval(ctx, cmp->inputs[1], 0); - dst->ins[0].mask = rm; - - if ((dst->flags & TILE_HAS_IMM) == 0) { - dst->ins[1].src = get_interval(ctx, cmp->inputs[2], 0); - dst->ins[1].mask = rm; - } - } else { - dst->ins[0].src = get_interval(ctx, cmp, 0); - dst->ins[0].mask = normie_mask(ctx, cmp->dt); - } - - FOREACH_N(i, ins, ins+tmps) { - dst->ins[i].src = NULL; - dst->ins[i].mask = ctx->normie_mask[REG_CLASS_GPR]; - } - - break; - } - - case TB_SYSCALL: { - const struct ParamDesc* abi = ¶m_descs[2]; - uint32_t caller_saved_gprs = abi->caller_saved_gprs; - - int param_count = n->input_count - 3; - if (n->type == TB_TAILCALL) { - caller_saved_gprs &= ~(1u << RAX); - } - - FOREACH_N(i, 0, param_count > 4 ? 4 : param_count) { - caller_saved_gprs &= ~(1u << abi->gprs[i]); - } - - size_t clobber_count = tb_popcount(caller_saved_gprs); - size_t input_count = (n->input_count - 2) + clobber_count; - - // SYSCALL - TileInput* ins = dst->ins = tb_arena_alloc(tmp_arena, input_count * sizeof(TileInput)); - dst->in_count = input_count; - - ins[0].src = get_interval(ctx, n->inputs[2], 0); - ins[0].mask = REGMASK(GPR, 1u << RAX); - - assert(param_count < abi->gpr_count); - FOREACH_N(i, 0, param_count) { - ins[i].src = get_interval(ctx, n->inputs[i + 3], 0); - ins[i].mask = REGMASK(GPR, 1u << abi->gprs[i]); - } - - int j = param_count; - FOREACH_N(i, 0, ctx->num_regs[REG_CLASS_GPR]) { - if (caller_saved_gprs & (1u << i)) { - ins[j].src = NULL; - ins[j].mask = REGMASK(GPR, 1u << i); - j++; - } - } - break; - } - - case TB_CALL: - case TB_TAILCALL: { - const struct ParamDesc* abi = ¶m_descs[ctx->abi_index]; - uint32_t caller_saved_gprs = abi->caller_saved_gprs; - uint32_t caller_saved_xmms = ~0ull >> (64 - abi->caller_saved_xmms); - - int param_count = n->input_count - 3; - if (ctx->num_regs[0] < param_count) { - ctx->num_regs[0] = param_count; - ctx->call_usage = param_count; - } - - if (n->type == TB_TAILCALL) { - caller_saved_gprs &= ~(1u << RAX); - } - - FOREACH_N(i, 0, param_count > 4 ? 4 : param_count) { - caller_saved_gprs &= ~(1u << abi->gprs[i]); - } - - size_t clobber_count = tb_popcount(caller_saved_gprs); - size_t input_start = n->inputs[2]->type == TB_SYMBOL ? 3 : 2; - size_t input_count = (n->input_count - input_start) + clobber_count; - - TileInput* ins; - if (n->inputs[2]->type == TB_SYMBOL) { - // CALL symbol - ins = dst->ins = tb_arena_alloc(tmp_arena, input_count * sizeof(TileInput)); - dst->in_count = input_count; - } else { - // CALL r/m - ins = dst->ins = tb_arena_alloc(tmp_arena, input_count * sizeof(TileInput)); - dst->in_count = input_count; - - ins[0].src = get_interval(ctx, n->inputs[2], 0); - if (n->type == TB_TAILCALL) { - ins[0].mask = REGMASK(GPR, 1u << RAX); - } else { - ins[0].mask = ctx->normie_mask[REG_CLASS_GPR]; - } - ins += 1; - } - - FOREACH_N(i, 0, param_count) { - ins[i].src = get_interval(ctx, n->inputs[i + 3], 0); - - if (i < abi->gpr_count) { - if (TB_IS_FLOAT_TYPE(n->inputs[i + 3]->dt)) { - ins[i].mask = REGMASK(XMM, 1u << i); - } else { - ins[i].mask = REGMASK(GPR, 1u << abi->gprs[i]); - } - } else { - // stack slots go into [RSP + 8i] - ins[i].mask = REGMASK(STK, i); - } - } - - int j = param_count; - FOREACH_N(i, 0, 16) { - if (caller_saved_gprs & (1u << i)) { - ins[j].src = NULL; - ins[j].mask = REGMASK(GPR, 1u << i); - j++; - } - } - - assert(j == input_count - (n->inputs[2]->type != TB_SYMBOL)); - return; - } - - default: - tb_todo(); - break; - } - - if (dst->out_count == 1) { - dst->outs[0]->dt = n->dt; - dst->outs[0]->mask = normie_mask(ctx, n->dt); - } else if (dst->out_count != 0) { - tb_todo(); - } -} - -static int stk_offset(Ctx* ctx, int reg) { - int pos = reg*8; - if (reg >= ctx->num_regs[0]) { - return ctx->stack_usage - (pos + 8); - } else { - return pos; - } -} - -static void emit_epilogue(Ctx* restrict ctx, TB_CGEmitter* e, int stack_usage) { - TB_FunctionPrototype* proto = ctx->f->prototype; - bool needs_stack = stack_usage > ctx->stack_header + (proto->param_count * 8); - - FOREACH_REVERSE_N(i, 0, dyn_array_length(ctx->callee_spills)) { - int pos = stk_offset(ctx, ctx->callee_spills[i].stk); - int rc = ctx->callee_spills[i].class; - - Val reg = val_gpr(ctx->callee_spills[i].reg); - reg.type = rc == REG_CLASS_XMM ? VAL_XMM : VAL_GPR; - - Val spill = val_base_disp(RSP, pos); - inst2(e, MOV, ®, &spill, TB_X86_TYPE_QWORD); - } - - // add rsp, N - if (stack_usage) { - if (stack_usage == (int8_t)stack_usage) { - EMIT1(&ctx->emit, rex(true, 0x00, RSP, 0)); - EMIT1(&ctx->emit, 0x83); - EMIT1(&ctx->emit, mod_rx_rm(MOD_DIRECT, 0x00, RSP)); - EMIT1(&ctx->emit, (int8_t) stack_usage); - } else { - EMIT1(&ctx->emit, rex(true, 0x00, RSP, 0)); - EMIT1(&ctx->emit, 0x81); - EMIT1(&ctx->emit, mod_rx_rm(MOD_DIRECT, 0x00, RSP)); - EMIT4(&ctx->emit, stack_usage); - } - } - - // pop rbp (if we even used the frameptr) - if ((ctx->features.gen & TB_FEATURE_FRAME_PTR) && stack_usage > 0) { - EMIT1(&ctx->emit, 0x58 + RBP); - } -} - -static Val op_at(Ctx* ctx, LiveInterval* l) { - if (l->class == REG_CLASS_STK) { - return val_stack(stk_offset(ctx, l->assigned)); - } else { - assert(l->assigned >= 0); - return (Val) { .type = l->class == REG_CLASS_XMM ? VAL_XMM : VAL_GPR, .reg = l->assigned }; - } -} - -static GPR op_gpr_at(LiveInterval* l) { - assert(l->class == REG_CLASS_GPR); - return l->assigned; -} - -static Val parse_memory_op(Ctx* restrict ctx, TB_CGEmitter* e, Tile* t, TB_Node* addr) { - Val ptr; - if (addr->type == TB_LOCAL) { - int pos = get_stack_slot(ctx, addr); - ptr = val_stack(pos); - } else if (addr->type == TB_SYMBOL) { - TB_Symbol* sym = TB_NODE_GET_EXTRA_T(addr, TB_NodeSymbol)->sym; - ptr = val_global(sym, 0); - } else { - tb_todo(); - } - - return ptr; -} - -static void pre_emit(Ctx* restrict ctx, TB_CGEmitter* e, TB_Node* root) { - size_t call_usage = ctx->call_usage; - if (ctx->abi_index == 0 && call_usage > 0 && call_usage < 4) { - call_usage = 4; - } - - ctx->stack_usage -= ctx->initial_spills * 8; - ctx->stack_usage += call_usage * 8; - - TB_FunctionPrototype* proto = ctx->f->prototype; - size_t stack_usage = 0; - if (ctx->stack_usage > ctx->stack_header + (proto->param_count * 8)) { - // Align stack usage to 16bytes + 8 to accommodate for the RIP being pushed by CALL - stack_usage = align_up(ctx->stack_usage + ctx->stack_header, 16) - ctx->stack_header; - } - ctx->stack_usage = stack_usage; - - FOR_USERS(u, root) { - TB_Node* n = u->n; - if (n->type != TB_LOCAL) continue; - - TB_NodeLocal* l = TB_NODE_GET_EXTRA(n); - if (l->type == NULL) continue; - - int pos = get_stack_slot(ctx, n); - - TB_StackSlot s = { - .name = l->name, - .type = l->type, - .storage = { pos }, - }; - dyn_array_put(ctx->debug_stack_slots, s); - } - - // save frame pointer (if applies) - if ((ctx->features.gen & TB_FEATURE_FRAME_PTR) && stack_usage > 0) { - EMIT1(e, 0x50 + RBP); - - // mov rbp, rsp - EMIT1(e, rex(true, RSP, RBP, 0)); - EMIT1(e, 0x89); - EMIT1(e, mod_rx_rm(MOD_DIRECT, RSP, RBP)); - } - - // inserts a chkstk call if we use too much stack - if (stack_usage >= param_descs[ctx->abi_index].chkstk_limit) { - assert(ctx->f->super.module->chkstk_extern); - ctx->f->super.module->uses_chkstk++; - - Val sym = val_global(ctx->f->super.module->chkstk_extern, 0); - Val imm = val_imm(stack_usage); - Val rax = val_gpr(RAX); - Val rsp = val_gpr(RSP); - - inst2(e, MOV, &rax, &imm, TB_X86_TYPE_DWORD); - inst1(e, CALL, &sym, TB_X86_TYPE_QWORD); - inst2(e, SUB, &rsp, &rax, TB_X86_TYPE_QWORD); - } else if (stack_usage) { - if (stack_usage == (int8_t)stack_usage) { - // sub rsp, stack_usage - EMIT1(e, rex(true, 0x00, RSP, 0)); - EMIT1(e, 0x83); - EMIT1(e, mod_rx_rm(MOD_DIRECT, 0x05, RSP)); - EMIT1(e, stack_usage); - } else { - // sub rsp, stack_usage - EMIT1(e, rex(true, 0x00, RSP, 0)); - EMIT1(e, 0x81); - EMIT1(e, mod_rx_rm(MOD_DIRECT, 0x05, RSP)); - EMIT4(e, stack_usage); - } - } - - // we don't want this considered in the prologue because then i'd have to encode shit - // for Win64EH. - FOREACH_N(i, 0, dyn_array_length(ctx->callee_spills)) { - int pos = stk_offset(ctx, ctx->callee_spills[i].stk); - int rc = ctx->callee_spills[i].class; - - Val reg = val_gpr(ctx->callee_spills[i].reg); - reg.type = rc == REG_CLASS_GPR ? VAL_GPR : VAL_XMM; - - Val spill = val_base_disp(RSP, pos); - inst2(e, MOV, &spill, ®, TB_X86_TYPE_QWORD); - } - - // handle unknown parameters (if we have varargs) - if (proto->has_varargs) { - const GPR* parameter_gprs = param_descs[ctx->abi_index].gprs; - - // spill the rest of the parameters (assumes they're all in the GPRs) - size_t gpr_count = param_descs[ctx->abi_index].gpr_count; - size_t extra_param_count = proto->param_count > gpr_count ? 0 : gpr_count - proto->param_count; - - FOREACH_N(i, proto->param_count, gpr_count) { - int dst_pos = ctx->stack_header + (i * 8); - Val src = val_gpr(parameter_gprs[i]); - - Val dst = val_base_disp(RSP, stack_usage + dst_pos); - inst2(e, MOV, &dst, &src, TB_X86_TYPE_QWORD); - } - } - - ctx->prologue_length = e->count; -} - -// compute effective address operand -static Val emit_addr(Ctx* restrict ctx, TB_CGEmitter* e, Tile* t) { - bool use_tmp = t->out_count == 0 || t->outs[0]->mask.class == REG_CLASS_XMM; - - int in_count = 0; - Val ea = { .type = VAL_MEM, .index = GPR_NONE }; - AuxAddress* aux = t->aux; - if (t->flags & TILE_FOLDED_BASE) { - if (aux->base->type == TB_LOCAL) { - int pos = get_stack_slot(ctx, aux->base); - ea.reg = RSP; - ea.imm = pos; - } else { - assert(aux->base->type == TB_SYMBOL); - ea.type = VAL_GLOBAL; - ea.symbol = TB_NODE_GET_EXTRA_T(aux->base, TB_NodeSymbol)->sym; - } - } else { - ea.reg = op_at(ctx, t->ins[in_count].src).reg; - in_count++; - } - - if (t->flags & TILE_INDEXED) { - int index = op_gpr_at(t->ins[in_count++].src); - - int stride = aux->stride; - if (tb_is_power_of_two(stride)) { - int scale = tb_ffs(stride) - 1; - if (scale > 3) { - Val tmp = val_gpr(op_gpr_at(use_tmp ? t->ins[in_count++].src : t->outs[0])); - if (tmp.reg != index) { - Val index_op = val_gpr(index); - inst2(e, MOV, &tmp, &index_op, TB_X86_TYPE_QWORD); - } - - Val imm = val_imm(scale); - inst2(e, SHL, &tmp, &imm, TB_X86_TYPE_QWORD); - index = tmp.reg; - } else { - ea.scale = scale; - } - } else { - tb_todo(); - } - - ea.index = index; - } - - ea.imm += aux->offset; - return ea; -} - -static Cond emit_cmp(Ctx* restrict ctx, TB_CGEmitter* e, TB_Node* cmp, Tile* t, int64_t falsey) { - Val a = op_at(ctx, t->ins[0].src); - if (t->flags & TILE_FOLDED_CMP) { - TB_DataType cmp_dt = TB_NODE_GET_EXTRA_T(cmp, TB_NodeCompare)->cmp_dt; - assert(cmp->type >= TB_CMP_EQ && cmp->type <= TB_CMP_FLE); - assert(falsey == 0 || falsey == 1); - - Cond cc; - if (TB_IS_FLOAT_TYPE(cmp_dt)) { - Val b = op_at(ctx, t->ins[1].src); - inst2sse(e, FP_UCOMI, &a, &b, legalize_float(cmp_dt)); - - switch (cmp->type) { - case TB_CMP_EQ: cc = E; break; - case TB_CMP_NE: cc = NE; break; - case TB_CMP_FLT: cc = B; break; - case TB_CMP_FLE: cc = BE; break; - default: tb_unreachable(); - } - } else { - if (t->flags & TILE_HAS_IMM) { - assert(cmp->inputs[2]->type == TB_INTEGER_CONST); - TB_NodeInt* i = TB_NODE_GET_EXTRA(cmp->inputs[2]); - - Val b = val_imm(i->value); - inst2(e, CMP, &a, &b, legalize_int2(cmp_dt)); - } else { - Val b = op_at(ctx, t->ins[1].src); - inst2(e, CMP, &a, &b, legalize_int2(cmp_dt)); - } - - switch (cmp->type) { - case TB_CMP_EQ: cc = E; break; - case TB_CMP_NE: cc = NE; break; - case TB_CMP_SLT: cc = L; break; - case TB_CMP_SLE: cc = LE; break; - case TB_CMP_ULT: cc = B; break; - case TB_CMP_ULE: cc = BE; break; - default: tb_unreachable(); - } - } - - if (falsey == 1) { cc ^= 1; } - return cc; - } else { - if (falsey == 0) { - inst2(e, TEST, &a, &a, legalize_int2(cmp->dt)); - } else { - assert(fits_into_int32(falsey)); - Val imm = val_imm(falsey); - inst2(e, CMP, &a, &imm, legalize_int2(cmp->dt)); - } - return NE; - } -} - -static void on_basic_block(Ctx* restrict ctx, TB_CGEmitter* e, int bb) { - tb_resolve_rel32(e, &e->labels[bb], e->count); -} - -static void emit_tile(Ctx* restrict ctx, TB_CGEmitter* e, Tile* t) { - if (t->tag == TILE_SPILL_MOVE) { - Val dst = op_at(ctx, t->outs[0]); - Val src = op_at(ctx, t->ins[0].src); - if (!is_value_match(&dst, &src)) { - COMMENT("move v%d -> v%d", t->outs[0]->id, t->ins[0].src->id); - - TB_DataType dt = t->spill_dt; - if (dt.type == TB_FLOAT) { - inst2sse(e, FP_MOV, &dst, &src, legalize_float(dt)); - } else { - inst2(e, MOV, &dst, &src, legalize_int2(dt)); - } - } else { - COMMENT("folded move v%d -> v%d", t->outs[0]->id, t->ins[0].src->id); - } - } else if (t->tag == TILE_GOTO) { - MachineBB* mbb = node_to_bb(ctx, t->succ); - if (ctx->fallthrough != mbb->id) { - EMIT1(e, 0xE9); EMIT4(e, 0); - tb_emit_rel32(e, &e->labels[mbb->id], GET_CODE_POS(e) - 4); - } - } else { - TB_Node* n = t->n; - switch (n->type) { - // epilogue - case TB_RETURN: { - size_t pos = e->count; - emit_epilogue(ctx, e, ctx->stack_usage); - EMIT1(e, 0xC3); - ctx->epilogue_length = e->count - pos; - break; - } - case TB_TRAP: { - EMIT1(e, 0x0F); - EMIT1(e, 0x0B); - break; - } - case TB_DEBUGBREAK: { - EMIT1(e, 0xCC); - break; - } - // projections don't manage their own work, that's the - // TUPLE node's job. - case TB_PROJ: - case TB_REGION: - case TB_NATURAL_LOOP: - case TB_AFFINE_LOOP: - case TB_PHI: - case TB_POISON: - case TB_UNREACHABLE: - case TB_SPLITMEM: - case TB_MERGEMEM: - case TB_CALLGRAPH: - case TB_ROOT: - break; - - case TB_INLINE_ASM: { - TB_NodeInlineAsm* a = TB_NODE_GET_EXTRA_T(n, TB_NodeInlineAsm); - - size_t count = a->emit(n, a->ctx, e->capacity, e->data); - assert(e->count + count < e->capacity); - e->count += count; - break; - } - - // rdtsc - // shl rdx, 32 - // or rax, rdx - case TB_CYCLE_COUNTER: { - Val rax = val_gpr(RAX); - Val rdx = val_gpr(RDX); - Val imm = val_imm(32); - EMIT1(e, 0x0F); EMIT1(e, 0x31); - inst2(e, SHL, &rdx, &imm, TB_X86_TYPE_QWORD); - inst2(e, OR, &rax, &rdx, TB_X86_TYPE_QWORD); - break; - } - - case TB_READ: { - TB_Node* proj1 = proj_with_index(n, 1)->n; - - Val dst = op_at(ctx, val_at(ctx, proj1)->tile->outs[0]); - Val ea = emit_addr(ctx, e, t); - inst2(e, MOV, &dst, &ea, legalize_int2(proj1->dt)); - break; - } - - case TB_LOAD: { - Val dst = op_at(ctx, t->outs[0]); - Val ea = emit_addr(ctx, e, t); - if (n->dt.type == TB_FLOAT) { - inst2sse(e, FP_MOV, &dst, &ea, legalize_float(n->dt)); - } else { - inst2(e, MOV, &dst, &ea, legalize_int2(n->dt)); - } - break; - } - case TB_WRITE: - case TB_STORE: { - TB_Node* val = n->inputs[3]; - - Val ea = emit_addr(ctx, e, t); - Val src; - if (val->dt.type == TB_FLOAT) { - src = op_at(ctx, t->ins[t->in_count - 1].src); - inst2sse(e, FP_MOV, &ea, &src, legalize_float(val->dt)); - } else { - if (t->flags & TILE_HAS_IMM) { - assert(val->type == TB_INTEGER_CONST); - TB_NodeInt* i = TB_NODE_GET_EXTRA(val); - src = val_imm(i->value); - } else { - src = op_at(ctx, t->ins[t->in_count - 1].src); - } - - inst2(e, MOV, &ea, &src, legalize_int2(val->dt)); - } - break; - } - case TB_LOCAL: - case TB_MEMBER_ACCESS: - case TB_ARRAY_ACCESS: { - Val dst = op_at(ctx, t->outs[0]); - Val ea = emit_addr(ctx, e, t); - inst2(e, LEA, &dst, &ea, TB_X86_TYPE_QWORD); - break; - } - case TB_VA_START: { - TB_FunctionPrototype* proto = ctx->f->prototype; - - Val dst = op_at(ctx, t->outs[0]); - Val ea = val_stack(ctx->stack_usage + ctx->stack_header + proto->param_count*8); - inst2(e, LEA, &dst, &ea, TB_X86_TYPE_QWORD); - break; - } - - case TB_INTEGER_CONST: { - uint64_t x = TB_NODE_GET_EXTRA_T(n, TB_NodeInt)->value; - uint32_t hi = x >> 32ull; - - TB_X86_DataType dt = legalize_int2(n->dt); - Val dst = op_at(ctx, t->outs[0]); - if (x == 0) { - // xor reg, reg - inst2(e, XOR, &dst, &dst, dt); - } else if (hi == 0 || dt == TB_X86_TYPE_QWORD) { - Val src = val_abs(x); - inst2(e, MOVABS, &dst, &src, dt); - } else { - Val src = val_imm(x); - inst2(e, MOV, &dst, &src, dt); - } - break; - } - case TB_FLOAT32_CONST: { - uint32_t imm = (Cvt_F32U32){ .f = TB_NODE_GET_EXTRA_T(n, TB_NodeFloat32)->value }.i; - Val dst = op_at(ctx, t->outs[0]); - - if (imm == 0) { - inst2sse(e, FP_XOR, &dst, &dst, TB_X86_TYPE_SSE_PS); - } else { - TB_Symbol* sym = &tb__small_data_intern(ctx->module, sizeof(float), &imm)->super; - Val src = val_global(sym, 0); - inst2sse(e, FP_MOV, &dst, &src, TB_X86_TYPE_SSE_PS); - } - break; - } - case TB_FLOAT64_CONST: { - uint64_t imm = (Cvt_F64U64){ .f = TB_NODE_GET_EXTRA_T(n, TB_NodeFloat64)->value }.i; - Val dst = op_at(ctx, t->outs[0]); - - if (imm == 0) { - inst2sse(e, FP_XOR, &dst, &dst, TB_X86_TYPE_SSE_PS); - } else { - TB_Symbol* sym = &tb__small_data_intern(ctx->module, sizeof(double), &imm)->super; - Val src = val_global(sym, 0); - inst2sse(e, FP_MOV, &dst, &src, TB_X86_TYPE_SSE_PS); - } - break; - } - case TB_FADD: - case TB_FSUB: - case TB_FMUL: - case TB_FDIV: - case TB_FMIN: - case TB_FMAX: { - const static InstType ops[] = { FP_ADD, FP_SUB, FP_MUL, FP_DIV, FP_MIN, FP_MAX }; - TB_X86_DataType dt = legalize_float(n->dt); - - Val dst = op_at(ctx, t->outs[0]); - Val lhs = op_at(ctx, t->ins[0].src); - if (!is_value_match(&dst, &lhs)) { - inst2sse(e, FP_MOV, &dst, &lhs, dt); - } - - Val rhs = op_at(ctx, t->ins[1].src); - inst2sse(e, ops[n->type - TB_FADD], &dst, &rhs, dt); - break; - } - case TB_SIGN_EXT: - case TB_ZERO_EXT: { - bool is_signed = n->type == TB_SIGN_EXT; - TB_DataType src_dt = n->inputs[1]->dt; - int bits_in_type = src_dt.type == TB_PTR ? 64 : src_dt.data; - - int op = 0; - TB_X86_DataType dt = legalize_int2(n->dt); - switch (bits_in_type) { - case 8: op = is_signed ? MOVSXB : MOVZXB; break; - case 16: op = is_signed ? MOVSXB : MOVZXW; break; - case 32: if (is_signed) { - op = MOVSXD; - } else { - op = MOV, dt = TB_X86_TYPE_DWORD; - } - break; - case 64: op = MOV; break; - } - - Val dst = op_at(ctx, t->outs[0]); - if (is_signed && dt <= TB_X86_TYPE_DWORD) { - dt = TB_X86_TYPE_DWORD; - } - - if (n->inputs[1]->type == TB_LOAD) { - Val ea = emit_addr(ctx, e, t); - inst2(e, op ? op : MOV, &dst, &ea, dt); - } else { - Val lhs = op_at(ctx, t->ins[0].src); - inst2(e, op ? op : MOV, &dst, &lhs, dt); - } - - if (op == 0) { - if (!is_signed && bits_in_type < 32) { - // chop bits with a mask - Val imm = val_imm(UINT64_MAX >> (64 - bits_in_type)); - inst2(e, AND, &dst, &imm, dt); - } else { - // unconventional sizes do: - // SHL dst, x - // SAR dst, x (or SHR if zero ext) - // - // where x is 'reg_width - val_width' - int dst_bits = dt == TB_X86_TYPE_QWORD ? 64 : 32; - int ext = is_signed ? SAR : SHR; - Val imm = val_imm(dst_bits - bits_in_type); - inst2(e, SHL, &dst, &imm, dt); - inst2(e, ext, &dst, &imm, dt); - } - } - break; - } - case TB_TRUNCATE: { - if (TB_IS_FLOAT_TYPE(n->dt)) { - Val dst = op_at(ctx, t->outs[0]); - Val lhs = op_at(ctx, t->ins[0].src); - inst2sse(e, FP_CVT, &dst, &lhs, legalize_float(n->inputs[1]->dt)); - } else { - TB_X86_DataType dt = legalize_int2(n->dt); - - Val dst = op_at(ctx, t->outs[0]); - Val lhs = op_at(ctx, t->ins[0].src); - if (!is_value_match(&dst, &lhs)) { - inst2(e, MOV, &dst, &lhs, dt); - } - } - break; - } - case TB_FLOAT_EXT: { - TB_X86_DataType src_dt = legalize_float(n->inputs[1]->dt); - Val dst = op_at(ctx, t->outs[0]); - Val lhs = op_at(ctx, t->ins[0].src); - inst2sse(e, FP_CVT, &dst, &lhs, src_dt); - break; - } - case TB_UINT2FLOAT: - case TB_INT2FLOAT: { - TB_DataType src_dt = n->inputs[1]->dt; - assert(src_dt.type == TB_INT); - - // it's either 32bit or 64bit conversion - // CVTSI2SS r/m32, xmm1 - // CVTSI2SD r/m64, xmm1 - bool is_64bit = src_dt.data > 32; - - TB_X86_DataType dt = legalize_float(n->dt); - Val dst = op_at(ctx, t->outs[0]); - Val lhs = op_at(ctx, t->ins[0].src); - inst2sse(e, is_64bit ? FP_CVT64 : FP_CVT32, &dst, &lhs, dt); - break; - } - - case TB_FLOAT2INT: - case TB_FLOAT2UINT: { - TB_DataType src_dt = n->inputs[1]->dt; - assert(src_dt.type == TB_FLOAT); - - // it's either 32bit or 64bit conversion - // F3 0F 2C /r CVTTSS2SI xmm1, r/m32 - // F3 REX.W 0F 2C /r CVTTSS2SI xmm1, r/m64 - // F2 0F 2C /r CVTTSD2SI xmm1, r/m32 - // F2 REX.W 0F 2C /r CVTTSD2SI xmm1, r/m64 - Val dst = op_at(ctx, t->outs[0]); - Val lhs = op_at(ctx, t->ins[0].src); - inst2sse(e, FP_CVTT, &dst, &lhs, legalize_float(src_dt)); - break; - } - case TB_BITCAST: { - TB_X86_DataType dst_dt = legalize_int2(n->dt); - TB_X86_DataType src_dt = legalize_int2(n->inputs[1]->dt); - - Val dst = op_at(ctx, t->outs[0]); - Val src = op_at(ctx, t->ins[0].src); - - if (dst_dt >= TB_X86_TYPE_BYTE && dst_dt <= TB_X86_TYPE_QWORD && - src_dt >= TB_X86_TYPE_BYTE && src_dt <= TB_X86_TYPE_QWORD) { - if (dst_dt != src_dt || !is_value_match(&dst, &src)) { - inst2(e, MOV, &dst, &src, dst_dt); - } - } else { - tb_todo(); - } - break; - } - case TB_SYMBOL: { - TB_Symbol* sym = TB_NODE_GET_EXTRA_T(n, TB_NodeSymbol)->sym; - Val dst = op_at(ctx, t->outs[0]); - - assert(sym); - if (is_tls_symbol(sym)) { - if (ctx->abi_index == 0) { - Val tmp = op_at(ctx, t->ins[0].src); - Val tls_index = val_global(ctx->module->tls_index_extern, 0); - - // mov tmp, dword [_tls_index] - inst2(e, MOV, &tmp, &tls_index, TB_X86_TYPE_DWORD); - // mov dst, qword gs:[58h] - EMIT1(e, 0x65); - EMIT1(e, tmp.reg >= 8 ? 0x4C : 0x48); - EMIT1(e, 0x8B); - EMIT1(e, mod_rx_rm(MOD_INDIRECT, tmp.reg, RSP)); - EMIT1(e, mod_rx_rm(SCALE_X1, RSP, RBP)); - EMIT4(e, 0x58); - // mov dst, qword [dst+tmp*8] - Val mem = val_base_index_disp(dst.reg, tmp.reg, SCALE_X8, 0); - INST2(MOV, &dst, &mem, TB_X86_TYPE_QWORD); - // add dst, relocation - EMIT1(e, rex(true, 0, dst.reg, 0)), EMIT1(e, 0x81); - EMIT1(e, mod_rx_rm(MOD_DIRECT, 0, dst.reg)); - EMIT4(e, 0); - tb_emit_symbol_patch(e->output, sym, e->count - 4); - } else { - tb_todo(); - } - } else { - Val src = val_global(sym, 0); - inst2(e, LEA, &dst, &src, TB_X86_TYPE_QWORD); - } - break; - } - case TB_NOT: { - TB_X86_DataType dt = legalize_int2(n->dt); - Val dst = op_at(ctx, t->outs[0]); - Val src = op_at(ctx, t->ins[0].src); - if (!is_value_match(&dst, &src)) { - inst2(e, MOV, &dst, &src, dt); - } - - inst1(e, NOT, &dst, dt); - break; - } - case TB_AND: - case TB_OR: - case TB_XOR: - case TB_ADD: - case TB_SUB: { - const static InstType ops[] = { AND, OR, XOR, ADD, SUB }; - InstType op = ops[n->type - TB_AND]; - TB_X86_DataType dt = legalize_int2(n->dt); - - Val dst = op_at(ctx, t->outs[0]); - Val lhs = op_at(ctx, t->ins[0].src); - - if (!is_value_match(&dst, &lhs)) { - // we'd rather do LEA addition than mov+add, but if it's add by itself it's fine - if (n->type == TB_ADD && (dt == TB_X86_TYPE_DWORD || dt == TB_X86_TYPE_QWORD)) { - if (t->flags & TILE_HAS_IMM) { - assert(n->inputs[2]->type == TB_INTEGER_CONST); - TB_NodeInt* i = TB_NODE_GET_EXTRA(n->inputs[2]); - - // lea dst, [lhs + imm] - Val ea = val_base_disp(lhs.reg, i->value); - inst2(e, LEA, &dst, &ea, dt); - break; - } - } - - inst2(e, MOV, &dst, &lhs, dt); - } - - if (t->flags & TILE_HAS_IMM) { - assert(n->inputs[2]->type == TB_INTEGER_CONST); - TB_NodeInt* i = TB_NODE_GET_EXTRA(n->inputs[2]); - - Val rhs = val_imm(i->value); - inst2(e, op, &dst, &rhs, dt); - } else { - Val rhs = op_at(ctx, t->ins[1].src); - inst2(e, op, &dst, &rhs, dt); - } - break; - } - case TB_MUL: { - TB_X86_DataType dt = legalize_int2(n->dt); - - Val dst = op_at(ctx, t->outs[0]); - Val lhs = op_at(ctx, t->ins[0].src); - - if (t->flags & TILE_HAS_IMM) { - assert(n->inputs[2]->type == TB_INTEGER_CONST); - TB_NodeInt* i = TB_NODE_GET_EXTRA(n->inputs[2]); - - inst2(e, IMUL3, &dst, &lhs, dt); - if (dt == TB_X86_TYPE_WORD) { - EMIT2(e, i->value); - } else { - EMIT4(e, i->value); - } - } else { - if (!is_value_match(&dst, &lhs)) { - inst2(e, MOV, &dst, &lhs, dt); - } - - Val rhs = op_at(ctx, t->ins[1].src); - inst2(e, IMUL, &dst, &rhs, dt); - } - break; - } - case TB_SHL: - case TB_SHR: - case TB_ROL: - case TB_ROR: - case TB_SAR: { - TB_X86_DataType dt = legalize_int2(n->dt); - - Val dst = op_at(ctx, t->outs[0]); - Val lhs = op_at(ctx, t->ins[0].src); - if (!is_value_match(&dst, &lhs)) { - inst2(e, MOV, &dst, &lhs, dt); - } - - InstType op; - switch (n->type) { - case TB_SHL: op = SHL; break; - case TB_SHR: op = SHR; break; - case TB_ROL: op = ROL; break; - case TB_ROR: op = ROR; break; - case TB_SAR: op = SAR; break; - default: tb_todo(); - } - - if (t->flags & TILE_HAS_IMM) { - assert(n->inputs[2]->type == TB_INTEGER_CONST); - TB_NodeInt* i = TB_NODE_GET_EXTRA(n->inputs[2]); - - Val rhs = val_imm(i->value); - inst2(e, op, &dst, &rhs, dt); - } else { - Val rcx = val_gpr(RCX); - inst2(e, op, &dst, &rcx, TB_X86_TYPE_DWORD); - } - break; - } - case TB_UDIV: - case TB_SDIV: - case TB_UMOD: - case TB_SMOD: { - bool is_signed = (n->type == TB_SDIV || n->type == TB_SMOD); - bool is_div = (n->type == TB_UDIV || n->type == TB_SDIV); - - TB_DataType dt = n->dt; - - // if signed: - // cqo/cdq (sign extend RAX into RDX) - // else: - // xor rdx, rdx - if (is_signed) { - if (n->dt.data > 32) { - EMIT1(e, 0x48); - } - EMIT1(e, 0x99); - } else { - Val rdx = val_gpr(RDX); - inst2(e, XOR, &rdx, &rdx, TB_X86_TYPE_DWORD); - } - - Val rhs = op_at(ctx, t->ins[1].src); - inst1(e, is_signed ? IDIV : DIV, &rhs, legalize_int2(dt)); - break; - } - case TB_SYSCALL: { - inst0(e, SYSCALL, TB_X86_TYPE_QWORD); - break; - } - case TB_CALL: - case TB_TAILCALL: { - int op = CALL; - if (n->type == TB_TAILCALL) { - op = JMP; - emit_epilogue(ctx, e, ctx->stack_usage); - } - - if (n->inputs[2]->type == TB_SYMBOL) { - TB_Symbol* sym = TB_NODE_GET_EXTRA_T(n->inputs[2], TB_NodeSymbol)->sym; - - Val target = val_global(sym, 0); - inst1(e, op, &target, TB_X86_TYPE_QWORD); - } else { - Val target = op_at(ctx, t->ins[0].src); - inst1(e, op, &target, TB_X86_TYPE_QWORD); - } - break; - } - case TB_CMP_EQ: - case TB_CMP_NE: - case TB_CMP_SLT: - case TB_CMP_SLE: - case TB_CMP_ULT: - case TB_CMP_ULE: - case TB_CMP_FLT: - case TB_CMP_FLE: { - TB_X86_DataType dt = legalize_int2(n->dt); - Val dst = op_at(ctx, t->outs[0]); - - Cond cc = emit_cmp(ctx, e, n, t, 0); - inst1(e, SETO+(cc^1), &dst, dt); - break; - } - case TB_SELECT: { - TB_X86_DataType dt = legalize_int2(n->dt); - Val dst = op_at(ctx, t->outs[0]); - - Cond cc = emit_cmp(ctx, e, n->inputs[1], t, 0); - - int ops = 1; - if ((t->flags & TILE_HAS_IMM) == 0) { - ops += 1; - } - - Val a = op_at(ctx, t->ins[ops+0].src); - if (!is_value_match(&dst, &a)) { - inst2(e, MOV, &dst, &a, dt); - } - - Val b = op_at(ctx, t->ins[ops+1].src); - inst2(e, CMOVO+(cc^1), &dst, &b, dt); - break; - } - case TB_BRANCH: { - TB_NodeBranch* br = TB_NODE_GET_EXTRA(n); - - // the arena on the function should also be available at this time, we're - // in the TB_Passes - TB_Arena* arena = ctx->f->arena; - TB_ArenaSavepoint sp = tb_arena_save(arena); - int* succ = tb_arena_alloc(arena, br->succ_count * sizeof(int)); - - // fill successors - bool has_default = false; - FOR_USERS(u, n) { - if (u->n->type == TB_PROJ) { - int index = TB_NODE_GET_EXTRA_T(u->n, TB_NodeProj)->index; - TB_Node* succ_n = cfg_next_bb_after_cproj(u->n); - - if (index == 0) { - has_default = !cfg_is_unreachable(succ_n); - } - - MachineBB* mbb = node_to_bb(ctx, succ_n); - succ[index] = mbb->id; - } - } - - TB_DataType dt = n->inputs[1]->dt; - if (br->succ_count == 1) { - assert(0 && "degenerate branch? that's odd"); - } else if (br->succ_count == 2) { - Val naw = val_label(succ[1]); - Val yea = val_label(succ[0]); - Cond cc = emit_cmp(ctx, e, n->inputs[1], t, br->keys[0].key); - - // if flipping avoids a jmp, do that - if (ctx->fallthrough == yea.label) { - x86_jcc(e, cc ^ 1, naw); - } else { - x86_jcc(e, cc, yea); - if (ctx->fallthrough != naw.label) { - x86_jmp(e, naw); - } - } - } else { - AuxBranch* aux = t->aux; - TB_X86_DataType cmp_dt = legalize_int2(dt); - Val key = op_at(ctx, t->ins[0].src); - - if (aux->if_chain) { - // Basic if-else chain - FOREACH_N(i, 1, br->succ_count) { - uint64_t curr_key = br->keys[i-1].key; - - if (fits_into_int32(curr_key)) { - Val imm = val_imm(curr_key); - inst2(e, CMP, &key, &imm, cmp_dt); - } else { - Val tmp = op_at(ctx, t->ins[1].src); - Val imm = val_abs(curr_key); - - inst2(e, MOV, &key, &imm, cmp_dt); - inst2(e, CMP, &key, &imm, cmp_dt); - } - x86_jcc(e, E, val_label(succ[i])); - } - x86_jmp(e, val_label(succ[0])); - } else { - int64_t min = aux->min; - int64_t max = aux->max; - int64_t range = (aux->max - aux->min) + 1; - - // make a jump table with 4 byte relative pointers for each target - TB_Function* f = ctx->f; - TB_Global* jump_table = tb_global_create(f->super.module, -1, "jumptbl", NULL, TB_LINKAGE_PRIVATE); - tb_global_set_storage(f->super.module, tb_module_get_rdata(f->super.module), jump_table, range*4, 4, 1); - - // generate patches for later - uint32_t* jump_entries = tb_global_add_region(f->super.module, jump_table, 0, range*4); - - Set entries_set = set_create_in_arena(arena, range); - FOREACH_N(i, 1, br->succ_count) { - uint64_t key_idx = br->keys[i - 1].key - min; - assert(key_idx < range); - - JumpTablePatch p; - p.pos = &jump_entries[key_idx]; - p.target = succ[i]; - dyn_array_put(ctx->jump_table_patches, p); - set_put(&entries_set, key_idx); - } - - // handle default cases - FOREACH_N(i, 0, range) { - if (!set_get(&entries_set, i)) { - JumpTablePatch p; - p.pos = &jump_entries[i]; - p.target = succ[0]; - dyn_array_put(ctx->jump_table_patches, p); - } - } - - /*int tmp = DEF(NULL, dt); - hint_reg(ctx, tmp, key); - if (dt.data >= 32) { - SUBMIT(inst_move(dt, tmp, key)); - } else if (dt.data == 16) { - dt = TB_TYPE_I32; - SUBMIT(inst_op_rr(MOVZXW, dt, tmp, key)); - } else if (dt.data == 8) { - dt = TB_TYPE_I32; - SUBMIT(inst_op_rr(MOVZXB, dt, tmp, key)); - } else { - dt = TB_TYPE_I32; - uint64_t mask = tb__mask(dt.data); - - SUBMIT(inst_move(dt, tmp, key)); - SUBMIT(inst_op_rri(AND, dt, tmp, tmp, mask)); - }*/ - - // copy key into temporary - { - Val tmp = op_at(ctx, t->ins[1].src); - inst2(e, MOV, &tmp, &key, TB_X86_TYPE_QWORD); - key = tmp; - } - - int ins = 1; - Val target = op_at(ctx, t->ins[2].src); - Val table = op_at(ctx, t->ins[3].src); - - // Simple range check: - // if ((key - min) >= (max - min)) goto default - if (has_default) { - if (min != 0) { - Val imm = val_imm(min); - inst2(e, SUB, &key, &imm, cmp_dt); - } - // cmp key, range - Val imm = val_imm(range); - inst2(e, CMP, &key, &imm, cmp_dt); - // jnb fallthru - jcc(e, NB, succ[0]); - } - // lea target, [rip + f] - Val fn_sym = val_global((TB_Symbol*) f, 0); - inst2(e, LEA, &target, &fn_sym, TB_X86_TYPE_QWORD); - // lea table, [rip + JUMP_TABLE] - Val table_sym = val_global((TB_Symbol*) jump_table, 0); - inst2(e, LEA, &table, &table_sym, TB_X86_TYPE_QWORD); - // movsxd table, [table + key*4] - Val addr = val_base_index_disp(table.reg, key.reg, SCALE_X4, 0); - inst2(e, MOVSXD, &table, &addr, TB_X86_TYPE_QWORD); - // add target, table - inst2(e, ADD, &target, &table, TB_X86_TYPE_QWORD); - // jmp target - __(jmp, target); - } - } - - tb_arena_restore(arena, sp); - break; - } - - default: tb_todo(); - } - } -} - -static void post_emit(Ctx* restrict ctx, TB_CGEmitter* e) { - // pad to 16bytes - static const uint8_t nops[8][8] = { - { 0x90 }, - { 0x66, 0x90 }, - { 0x0F, 0x1F, 0x00 }, - { 0x0F, 0x1F, 0x40, 0x00 }, - { 0x0F, 0x1F, 0x44, 0x00, 0x00 }, - { 0x66, 0x0F, 0x1F, 0x44, 0x00, 0x00 }, - { 0x0F, 0x1F, 0x80, 0x00, 0x00, 0x00, 0x00 }, - { 0x0F, 0x1F, 0x84, 0x00, 0x00, 0x00, 0x00, 0x00 }, - }; - - size_t pad = 16 - (ctx->emit.count & 15); - if (pad < 16) { - ctx->nop_pads = pad; - - uint8_t* dst = tb_cgemit_reserve(&ctx->emit, pad); - tb_cgemit_commit(&ctx->emit, pad); - - if (pad > 8) { - size_t rem = pad - 8; - memset(dst, 0x66, rem); - pad -= rem, dst += rem; - } - memcpy(dst, nops[pad - 1], pad); - } -} - -static void emit_win64eh_unwind_info(TB_Emitter* e, TB_FunctionOutput* out_f, uint64_t stack_usage) { - size_t patch_pos = e->count; - UnwindInfo unwind = { - .version = 1, - .flags = 0, // UNWIND_FLAG_EHANDLER, - .prolog_length = out_f->prologue_length, - .code_count = 0, - }; - tb_outs(e, sizeof(UnwindInfo), &unwind); - - size_t code_count = 0; - if (stack_usage > 0) { - UnwindCode codes[] = { - // sub rsp, stack_usage - { .code_offset = 4, .unwind_op = UNWIND_OP_ALLOC_SMALL, .op_info = (stack_usage / 8) - 1 }, - }; - tb_outs(e, sizeof(codes), codes); - code_count += 1; - } - - tb_patch1b(e, patch_pos + offsetof(UnwindInfo, code_count), code_count); -} - -#define E(fmt, ...) tb_asm_print(e, fmt, ## __VA_ARGS__) -// #define E(fmt, ...) printf(fmt, ## __VA_ARGS__) -static void our_print_memory_operand(TB_CGEmitter* e, Disasm* restrict d, TB_X86_Inst* restrict inst, size_t pos) { - uint8_t base = inst->regs & 0xFF; - uint8_t index = (inst->regs >> 8) & 0xFF; - - if (inst->flags & TB_X86_INSTR_INDIRECT) { - if ((inst->regs & 0xFFFF) == 0xFFFF) { - E("[rip"); - } else { - E("%s [", tb_x86_type_name(inst->dt)); - if (base != 0xFF) { - E("%s", tb_x86_reg_name(base, TB_X86_TYPE_QWORD)); - } - - if (index != 0xFF) { - E(" + %s*%d", tb_x86_reg_name(index, TB_X86_TYPE_QWORD), 1 << inst->scale); - } - } - - if (inst->disp > 0) { - E(" + %#x", inst->disp); - } else if (inst->disp < 0) { - E(" - %#x", -inst->disp); - } - E("]"); - } else if (base != 0xFF) { - E("%s", tb_x86_reg_name(base, inst->dt)); - } -} - -static void our_print_rip32(TB_CGEmitter* e, Disasm* restrict d, TB_X86_Inst* restrict inst, size_t pos, int64_t imm) { - if (d->patch && d->patch->pos == pos - 4) { - const TB_Symbol* target = d->patch->target; - - if (target->name[0] == 0) { - E("sym%p", target); - } else { - E("%s", target->name); - } - - if (imm > 0) { - E(" + %"PRId64, imm); - } else if (imm < 0) { - E(" - %"PRId64, imm); - } - - d->patch = d->patch->next; - } else { - uint32_t target = pos + imm; - int bb = tb_emit_get_label(e, target); - uint32_t landed = e->labels[bb] & 0x7FFFFFFF; - - if (landed != target) { - E(".bb%d + %d", bb, (int)target - (int)landed); - } else { - E(".bb%d", bb); - } - } -} - -static void disassemble(TB_CGEmitter* e, Disasm* restrict d, int bb, size_t pos, size_t end) { - if (bb >= 0) { - E(".bb%d:\n", bb); - } - - while (pos < end) { - while (d->loc != d->end && d->loc->pos == pos) { - E(" // %s : line %d\n", d->loc->file->path, d->loc->line); - d->loc++; - } - - TB_X86_Inst inst; - if (!tb_x86_disasm(&inst, end - pos, &e->data[pos])) { - E(" ERROR\n"); - pos += 1; // skip ahead once... cry - continue; - } - - uint64_t line_start = e->total_asm; - const char* mnemonic = tb_x86_mnemonic(&inst); - E(" "); - if (inst.flags & TB_X86_INSTR_REP) { - E("rep "); - } - if (inst.flags & TB_X86_INSTR_LOCK) { - E("lock "); - } - E("%s", mnemonic); - if (inst.dt >= TB_X86_TYPE_SSE_SS && inst.dt <= TB_X86_TYPE_SSE_PD) { - static const char* strs[] = { "ss", "sd", "ps", "pd" }; - E("%s", strs[inst.dt - TB_X86_TYPE_SSE_SS]); - } - E(" "); - - uint8_t rx = (inst.regs >> 16) & 0xFF; - if (inst.flags & TB_X86_INSTR_DIRECTION) { - if (rx != 255) { - E("%s", tb_x86_reg_name(rx, inst.dt2)); - E(", "); - } - our_print_memory_operand(e, d, &inst, pos); - } else { - our_print_memory_operand(e, d, &inst, pos); - if (rx != 255) { - E(", "); - E("%s", tb_x86_reg_name(rx, inst.dt2)); - } - } - - if (inst.flags & TB_X86_INSTR_IMMEDIATE) { - if (inst.regs != 0xFFFFFF) { - E(", "); - } - - if (inst.opcode == 0xE8 || inst.opcode == 0xE9 || inst.opcode == 0xEB || (inst.opcode >= 0x180 && inst.opcode <= 0x18F)) { - our_print_rip32(e, d, &inst, pos + inst.length, inst.imm); - } else { - E("%#"PRIx64, inst.imm); - } - } - - int offset = e->total_asm - line_start; - if (d->comment && d->comment->pos == pos) { - TB_OPTDEBUG(ANSI)(E("\x1b[32m")); - E(" // "); - bool out_of_line = false; - do { - if (out_of_line) { - // tack on a newline - E("%*s // ", offset, ""); - } - - E("%.*s\n", d->comment->line_len, d->comment->line); - d->comment = d->comment->next; - out_of_line = true; - } while (d->comment && d->comment->pos == pos); - TB_OPTDEBUG(ANSI)(E("\x1b[0m")); - } else { - E("\n"); - } - - pos += inst.length; - } -} -#undef E - -static size_t emit_call_patches(TB_Module* restrict m, TB_FunctionOutput* out_f) { - size_t r = 0; - uint32_t src_section = out_f->section; - - for (TB_SymbolPatch* patch = out_f->first_patch; patch; patch = patch->next) { - if (patch->target->tag == TB_SYMBOL_FUNCTION) { - uint32_t dst_section = ((TB_Function*) patch->target)->output->section; - - // you can't do relocations across sections - if (src_section == dst_section) { - assert(patch->pos < out_f->code_size); - - // x64 thinks of relative addresses as being relative - // to the end of the instruction or in this case just - // 4 bytes ahead hence the +4. - size_t actual_pos = out_f->code_pos + patch->pos + 4; - - uint32_t p = ((TB_Function*) patch->target)->output->code_pos - actual_pos; - memcpy(&out_f->code[patch->pos], &p, sizeof(uint32_t)); - - r += 1; - patch->internal = true; - } - } - } - - return out_f->patch_count - r; -} - -ICodeGen tb__x64_codegen = { - .minimum_addressable_size = 8, - .pointer_size = 64, - .emit_win64eh_unwind_info = emit_win64eh_unwind_info, - .emit_call_patches = emit_call_patches, - .get_data_type_size = get_data_type_size, - .compile_function = compile_function, -}; -#else -ICodeGen tb__x64_codegen; -#endif diff --git a/vendor/tb/tests/cg_test.c b/vendor/tb/tests/cg_test.c deleted file mode 100644 index aead14dd..00000000 --- a/vendor/tb/tests/cg_test.c +++ /dev/null @@ -1,28 +0,0 @@ -// Just some code generation tests for TB -#include - -int main() { - TB_FeatureSet features = { 0 }; - TB_Module* mod = tb_module_create_for_host(&features, true); - - TB_PrototypeParam ret = { TB_TYPE_I32 }; - TB_FunctionPrototype* proto = tb_prototype_create(mod, TB_CDECL, 1, &ret, 1, &ret, false); - - TB_Function* f = tb_function_create(mod, -1, "numbers", TB_LINKAGE_PUBLIC); - tb_function_set_prototype(f, tb_module_get_text(mod), proto, NULL); - - TB_Node* v = tb_inst_uint(f, TB_TYPE_I32, 42); - v = tb_inst_mul(f, v, tb_inst_param(f, 0), 0); - - tb_inst_ret(f, 1, &v); - - TB_Passes* passes = tb_pass_enter(f, NULL); - TB_FunctionOutput* asm_out = tb_pass_codegen(passes, true); - if (asm_out) { - printf("\n"); - tb_output_print_asm(asm_out, stdout); - } - tb_pass_exit(passes); - tb_module_destroy(mod); - return 0; -} diff --git a/vendor/tb/unittests/tb_test_exit_status.inc b/vendor/tb/unittests/tb_test_exit_status.inc deleted file mode 100644 index eac737bb..00000000 --- a/vendor/tb/unittests/tb_test_exit_status.inc +++ /dev/null @@ -1,11 +0,0 @@ -#include "util.inc" - -static int test_exit_status(void) { - TB_TEST_MODULE_BEGIN_; - - TB_Node *exit_value = tb_inst_sint(f_main, TB_TYPE_I32, 42); - EXIT_WITH_(exit_value); - - TB_TEST_MODULE_END_(test_exit_status, 42, 1); - return status; -} diff --git a/vendor/tb/unittests/tb_test_int_arith.inc b/vendor/tb/unittests/tb_test_int_arith.inc deleted file mode 100644 index e44c6307..00000000 --- a/vendor/tb/unittests/tb_test_int_arith.inc +++ /dev/null @@ -1,67 +0,0 @@ -#include "util.inc" - -#define TEST_INT_ARITH_(prefix_, type_, inst_type_, inst_op_, arg0_, \ - arg1_, res_) \ - static int test_##prefix_##_##inst_op_(void) { \ - TB_TEST_MODULE_BEGIN_; \ - \ - TB_Node *foo = tb_inst_##inst_type_(f_main, TB_TYPE_##type_, \ - (arg0_)); \ - TB_Node *bar = tb_inst_##inst_type_(f_main, TB_TYPE_##type_, \ - (arg1_)); \ - TB_Node *sum = tb_inst_##inst_op_(f_main, foo, bar, \ - TB_ARITHMATIC_NONE); \ - \ - EXIT_WITH_(sum); \ - \ - TB_TEST_MODULE_END_(test_##prefix_##_##inst_op_, (res_), 0); \ - return status; \ - } - -TEST_INT_ARITH_(i8, I8, sint, add, 50, -8, 42) -TEST_INT_ARITH_(i8, I8, sint, sub, 20, -10, 30) -TEST_INT_ARITH_(i8, I8, sint, mul, 7, 9, 63) -TEST_INT_ARITH_(i8, I8, sint, div, 100, 11, 9) -TEST_INT_ARITH_(i8, I8, sint, mod, 100, 11, 1) - -TEST_INT_ARITH_(i16, I16, sint, add, 300, -240, 60) -TEST_INT_ARITH_(i16, I16, sint, sub, 1000, 934, 66) -TEST_INT_ARITH_(i16, I16, sint, mul, 9, 8, 72) -TEST_INT_ARITH_(i16, I16, sint, div, 999, 112, 8) -TEST_INT_ARITH_(i16, I16, sint, mod, 999, 112, 103) - -TEST_INT_ARITH_(i32, I32, sint, add, 300, -240, 60) -TEST_INT_ARITH_(i32, I32, sint, sub, 1000, 934, 66) -TEST_INT_ARITH_(i32, I32, sint, mul, 9, 8, 72) -TEST_INT_ARITH_(i32, I32, sint, div, 999, 112, 8) -TEST_INT_ARITH_(i32, I32, sint, mod, 999, 112, 103) - -TEST_INT_ARITH_(i64, I64, sint, add, 300, -240, 60) -TEST_INT_ARITH_(i64, I64, sint, sub, 1000, 934, 66) -TEST_INT_ARITH_(i64, I64, sint, mul, 9, 8, 72) -TEST_INT_ARITH_(i64, I64, sint, div, 999, 112, 8) -TEST_INT_ARITH_(i64, I64, sint, mod, 999, 112, 103) - -TEST_INT_ARITH_(u8, I8, uint, add, 50, 8, 58) -TEST_INT_ARITH_(u8, I8, uint, sub, 30, 10, 20) -TEST_INT_ARITH_(u8, I8, uint, mul, 7, 9, 63) -TEST_INT_ARITH_(u8, I8, uint, div, 100, 11, 9) -TEST_INT_ARITH_(u8, I8, uint, mod, 100, 11, 1) - -TEST_INT_ARITH_(u16, I16, uint, add, 30, 50, 80) -TEST_INT_ARITH_(u16, I16, uint, sub, 1000, 934, 66) -TEST_INT_ARITH_(u16, I16, uint, mul, 9, 8, 72) -TEST_INT_ARITH_(u16, I16, uint, div, 999, 112, 8) -TEST_INT_ARITH_(u16, I16, uint, mod, 999, 112, 103) - -TEST_INT_ARITH_(u32, I32, uint, add, 50, 40, 90) -TEST_INT_ARITH_(u32, I32, uint, sub, 1000, 934, 66) -TEST_INT_ARITH_(u32, I32, uint, mul, 9, 8, 72) -TEST_INT_ARITH_(u32, I32, uint, div, 999, 112, 8) -TEST_INT_ARITH_(u32, I32, uint, mod, 999, 112, 103) - -TEST_INT_ARITH_(u64, I64, uint, add, 20, 25, 45) -TEST_INT_ARITH_(u64, I64, uint, sub, 1000, 934, 66) -TEST_INT_ARITH_(u64, I64, uint, mul, 9, 8, 72) -TEST_INT_ARITH_(u64, I64, uint, div, 999, 112, 8) -TEST_INT_ARITH_(u64, I64, uint, mod, 999, 112, 103) diff --git a/vendor/tb/unittests/tb_test_regressions.inc b/vendor/tb/unittests/tb_test_regressions.inc deleted file mode 100644 index f21bbcf9..00000000 --- a/vendor/tb/unittests/tb_test_regressions.inc +++ /dev/null @@ -1,26 +0,0 @@ -#include "util.inc" - -static int test_regression_module_arena(void) { - tb_module_destroy(tb_module_create(tb_test_arch, tb_test_system, - &tb_test_feature_set, 0)); - tb_module_destroy(tb_module_create(tb_test_arch, tb_test_system, - &tb_test_feature_set, 0)); - - // We're testing for segfault. - return 1; -} - -static int test_regression_link_global(void) { - TB_Module *module = tb_module_create(tb_test_arch, tb_test_system, - &tb_test_feature_set, 0); - TB_Global *global = tb_global_create(module, -1, "global", NULL, - TB_LINKAGE_PRIVATE); - tb_global_set_storage(module, tb_module_get_rdata(module), global, - 8, 8, 1); - TB_Linker *linker = tb_linker_create(tb_test_exe_type, - tb_test_arch); - tb_linker_append_module(linker, module); - tb_module_destroy(module); - tb_linker_destroy(linker); - return 1; -} diff --git a/vendor/tb/unittests/tb_unittests.c b/vendor/tb/unittests/tb_unittests.c deleted file mode 100644 index 272f8cf6..00000000 --- a/vendor/tb/unittests/tb_unittests.c +++ /dev/null @@ -1,110 +0,0 @@ -#include - -typedef struct { - TB_Module* m; - TB_JIT* jit; -} RunTest; - -RunTest create_jit() { - TB_FeatureSet features = { 0 }; - TB_Module* mod = tb_module_create_for_host(&features, true); - TB_JIT* jit = tb_jit_begin(mod, 0); - - return (RunTest){ mod, jit }; -} - -#include "tb_test_regressions.inc" -#include "tb_test_exit_status.inc" -#include "tb_test_int_arith.inc" - -typedef int (*TestFn)(void); - -static int failed = 0, total = 0; - -static void run_test(const char* name, TestFn fn) { - size_t len = strlen(name); - - fflush(stdout); - printf("%s\r", #proc_); - fflush(stdout); - int status_ = test_##proc_(); - fflush(stdout); - - printf("%s%.*s ", #proc_, (int) (41 - sizeof(#proc_)), - " ........................................" - ); - - if (status_) { - printf("OK\n"); - } else { - printf("FAILED\n"); - failed++; - } - total++; - fflush(stdout); - -} - -int main(int argc, char **argv) { - TEST(regression_module_arena); - TEST(regression_link_global); - TEST(exit_status); - - TEST(i8_add); - TEST(i8_sub); - TEST(i8_mul); - TEST(i8_div); - TEST(i8_mod); - - TEST(i16_add); - TEST(i16_sub); - TEST(i16_mul); - TEST(i16_div); - TEST(i16_mod); - - TEST(i32_add); - TEST(i32_sub); - TEST(i32_mul); - TEST(i32_div); - TEST(i32_mod); - - TEST(i64_add); - TEST(i64_sub); - TEST(i64_mul); - TEST(i64_div); - TEST(i64_mod); - - TEST(u8_add); - TEST(u8_sub); - TEST(u8_mul); - TEST(u8_div); - TEST(u8_mod); - - TEST(u16_add); - TEST(u16_sub); - TEST(u16_mul); - TEST(u16_div); - TEST(u16_mod); - - TEST(u32_add); - TEST(u32_sub); - TEST(u32_mul); - TEST(u32_div); - TEST(u32_mod); - - TEST(u64_add); - TEST(u64_sub); - TEST(u64_mul); - TEST(u64_div); - TEST(u64_mod); - - fflush(stdout); - if (failed > 0) { - printf("\n%d of %d tests failed.\n", failed, total); - } else { - printf("\nAll %d tests succeeded.\n", total); - } - fflush(stdout); - - return failed; -} diff --git a/vendor/tb/unittests/util.inc b/vendor/tb/unittests/util.inc deleted file mode 100644 index 232d9d63..00000000 --- a/vendor/tb/unittests/util.inc +++ /dev/null @@ -1,226 +0,0 @@ -#ifndef TB_TEST_UTIL_INC -#define TB_TEST_UTIL_INC - -#if defined(__GCC__) || defined(__clang__) -# pragma GCC diagnostic ignored "-Wunknown-pragmas" -# pragma GCC diagnostic ignored "-Wunused-function" -#endif - -#include "../include/tb.h" - -#include -#include - -#if defined(_WIN32) && !defined(__CYGWIN__) -# define TB_TEST_IS_WINDOWS_ 1 -# define WEXITSTATUS(x) x -# define EXEC_EXTENSION ".exe" - -static TB_System tb_test_system = TB_SYSTEM_WINDOWS; -static TB_ExecutableType tb_test_exe_type = TB_EXECUTABLE_PE; -#else -# define TB_TEST_IS_WINDOWS_ 0 -# define EXEC_EXTENSION "" - -# include - -static TB_System tb_test_system = TB_SYSTEM_LINUX; -static TB_ExecutableType tb_test_exe_type = TB_EXECUTABLE_ELF; -#endif - -static TB_Arch tb_test_arch = TB_ARCH_X86_64; -static TB_FeatureSet tb_test_feature_set = { .x64 = 0 }; - -static void tb_test_link_library(TB_Linker *linker, - char const *name) { -#if TB_TEST_IS_WINDOWS_ - // FIXME - // Find and link library on Windows. - // - - char full_path[200] = "W:\\MSVC\\Windows Kits\\10\\Lib\\10.0.22621.0" - "\\um\\x64\\"; - - ptrdiff_t folder_len = strlen(full_path); - ptrdiff_t name_len = strlen(name); - - assert(folder_len + name_len + 1 < sizeof full_path); - memcpy(full_path + folder_len, name, name_len); - full_path[folder_len + name_len] = '\0'; - - FILE *f = fopen(full_path, "rb"); - - assert(f != NULL); - if (f == NULL) - return; - - ptrdiff_t chunk_size = 100000; - ptrdiff_t data_size = 0; - - uint8_t *data = NULL; - uint8_t *p = data; - - while (!feof(f)) { - data = realloc(data, data_size + chunk_size); - assert(data != NULL); - if (data == NULL) - return; - - ptrdiff_t n = fread(data + data_size, 1, chunk_size, f); - if (n <= 0) - break; - data_size += n; - } - - fclose(f); - - TB_Slice sl_name = { .length = name_len, - .data = (uint8_t const *) name }; - TB_Slice sl_data = { .length = data_size, .data = data }; - tb_linker_append_library(linker, sl_name, sl_data); - - free(data); -#endif - - // NOTE - // We don't need to link libraries for unit-testing on Linux yet. - // -} - -#define ERROR(x) \ - do { \ - printf("Error in %s (\"%s\" line %d): " #x "\n", __FUNCTION__, \ - __FILE__, (int) __LINE__); \ - status = 0; \ - goto _final; \ - } while (0) - -#if TB_TEST_IS_WINDOWS_ - -# define EXIT_WITH_(node_status_) \ - do { \ - TB_PrototypeParam param0 = { .dt = TB_TYPE_I32, \ - .name = "uExitCode" }; \ - TB_FunctionPrototype *fp_ExitProcess = tb_prototype_create( \ - module, TB_STDCALL, 1, ¶m0, 0, NULL, 0); \ - \ - TB_Node *addr_ExitProcess = tb_inst_get_symbol_address( \ - f_main, \ - (TB_Symbol *) tb_extern_create(module, -1, "ExitProcess", \ - TB_EXTERNAL_SO_LOCAL)); \ - \ - tb_inst_call(f_main, fp_ExitProcess, addr_ExitProcess, 1, \ - &(node_status_)); \ - tb_inst_ret(f_main, 0, NULL); \ - } while (0) - -#else - -# define EXIT_WITH_(node_status_) \ - do { \ - TB_Node *num_ = tb_inst_sint(f_main, TB_TYPE_I32, 60); \ - tb_inst_syscall(f_main, TB_TYPE_I64, num_, 1, \ - &(node_status_)); \ - tb_inst_ret(f_main, 0, NULL); \ - } while (0) - -#endif - -#define TB_TEST_MODULE_BEGIN_ \ - int status = 1; \ - int ret = 0; \ - \ - TB_Module *module = NULL; \ - TB_Linker *linker = NULL; \ - \ - module = tb_module_create(tb_test_arch, tb_test_system, \ - &tb_test_feature_set, 0); \ - \ - if (module == NULL) \ - ERROR("tb_module_create failed."); \ - \ - TB_FunctionPrototype *fp_main = tb_prototype_create( \ - module, TB_CDECL, 0, NULL, 0, NULL, false); \ - \ - TB_Function *f_main = tb_function_create( \ - module, -1, "main", TB_LINKAGE_PUBLIC); \ - \ - TB_ModuleSectionHandle text = tb_module_get_text(module); \ - tb_function_set_prototype(f_main, text, fp_main, NULL); \ - \ - if (f_main == NULL) \ - ERROR("tb_function_create failed."); - -#define TB_TEST_MODULE_END_(name_, result_, print_asm_) \ - { \ -<<<<<<< HEAD - TB_Passes *passes = tb_pass_enter(f_main, NULL); \ - \ - if (passes == NULL) \ - ERROR("tb_pass_enter failed."); \ - \ - TB_FunctionOutput *asm_out = tb_pass_codegen(passes, 1); \ - \ - if ((print_asm_) && asm_out != NULL) { \ - printf("\n"); \ - tb_output_print_asm(asm_out, stdout); \ -======= - TB_SymbolIter it = tb_symbol_iter(module); \ - TB_Symbol* sym; \ - while (sym = tb_symbol_iter_next(&it), sym) { \ - if (sym->tag == TB_SYMBOL_FUNCTION) { \ - TB_Function *f = (TB_Function*) sym; \ - TB_Passes *passes = tb_pass_enter(f, NULL); \ - \ - if (passes == NULL) \ - ERROR("tb_pass_enter failed."); \ - \ - TB_FunctionOutput *asm_out = tb_pass_codegen(passes, 1); \ - \ - if ((print_asm_) && asm_out != NULL) { \ - printf("\n"); \ - tb_output_print_asm(asm_out, stdout); \ - } \ - \ - tb_pass_exit(passes); \ - } \ ->>>>>>> cfg - } \ - \ - tb_pass_exit(passes); \ - } \ - \ - linker = tb_linker_create(tb_test_exe_type, tb_test_arch); \ - \ - if (linker == NULL) \ - ERROR("tb_linker_create failed."); \ - \ - tb_linker_append_module(linker, module); \ - \ - tb_linker_set_entrypoint(linker, "main"); \ - \ - if (TB_TEST_IS_WINDOWS_) \ - tb_test_link_library(linker, "kernel32.lib"); \ - \ - TB_ExportBuffer buf = tb_linker_export(linker); \ - tb_export_buffer_to_file(buf, "bin/" #name_ EXEC_EXTENSION); \ - tb_export_buffer_free(buf); \ - \ - if (!TB_TEST_IS_WINDOWS_) { \ - (void) system("chmod a+x bin/" #name_); \ - ret = WEXITSTATUS(system("./bin/" #name_)); \ - } else \ - ret = WEXITSTATUS(system("start bin\\" #name_)); \ - \ - if (ret != (result_)) { \ - printf("Got %d, expected %d\n", (int) ret, (int) (result_)); \ - status = 0; \ - } \ - \ -_final: \ - if (module != NULL) \ - tb_module_destroy(module); \ - if (linker != NULL) \ - tb_linker_destroy(linker); - -#endif diff --git a/vendor/tcc b/vendor/tcc new file mode 160000 index 00000000..7d1bbc80 --- /dev/null +++ b/vendor/tcc @@ -0,0 +1 @@ +Subproject commit 7d1bbc80d4978c128b8ebead42485d7a79624dcd diff --git a/vendor/tcc/coff.h b/vendor/tcc/coff.h deleted file mode 100644 index e8e6185a..00000000 --- a/vendor/tcc/coff.h +++ /dev/null @@ -1,446 +0,0 @@ -/**************************************************************************/ -/* COFF.H */ -/* COFF data structures and related definitions used by the linker */ -/**************************************************************************/ - -/*------------------------------------------------------------------------*/ -/* COFF FILE HEADER */ -/*------------------------------------------------------------------------*/ -struct filehdr { - unsigned short f_magic; /* magic number */ - unsigned short f_nscns; /* number of sections */ - long f_timdat; /* time & date stamp */ - long f_symptr; /* file pointer to symtab */ - long f_nsyms; /* number of symtab entries */ - unsigned short f_opthdr; /* sizeof(optional hdr) */ - unsigned short f_flags; /* flags */ - unsigned short f_TargetID; /* for C6x = 0x0099 */ - }; - -/*------------------------------------------------------------------------*/ -/* File header flags */ -/*------------------------------------------------------------------------*/ -#define F_RELFLG 0x01 /* relocation info stripped from file */ -#define F_EXEC 0x02 /* file is executable (no unresolved refs) */ -#define F_LNNO 0x04 /* line numbers stripped from file */ -#define F_LSYMS 0x08 /* local symbols stripped from file */ -#define F_GSP10 0x10 /* 34010 version */ -#define F_GSP20 0x20 /* 34020 version */ -#define F_SWABD 0x40 /* bytes swabbed (in names) */ -#define F_AR16WR 0x80 /* byte ordering of an AR16WR (PDP-11) */ -#define F_LITTLE 0x100 /* byte ordering of an AR32WR (vax) */ -#define F_BIG 0x200 /* byte ordering of an AR32W (3B, maxi) */ -#define F_PATCH 0x400 /* contains "patch" list in optional header */ -#define F_NODF 0x400 - -#define F_VERSION (F_GSP10 | F_GSP20) -#define F_BYTE_ORDER (F_LITTLE | F_BIG) -#define FILHDR struct filehdr - -/* #define FILHSZ sizeof(FILHDR) */ -#define FILHSZ 22 /* above rounds to align on 4 bytes which causes problems */ - -#define COFF_C67_MAGIC 0x00c2 - -/*------------------------------------------------------------------------*/ -/* Macros to recognize magic numbers */ -/*------------------------------------------------------------------------*/ -#define ISMAGIC(x) (((unsigned short)(x))==(unsigned short)magic) -#define ISARCHIVE(x) ((((unsigned short)(x))==(unsigned short)ARTYPE)) -#define BADMAGIC(x) (((unsigned short)(x) & 0x8080) && !ISMAGIC(x)) - - -/*------------------------------------------------------------------------*/ -/* OPTIONAL FILE HEADER */ -/*------------------------------------------------------------------------*/ -typedef struct aouthdr { - short magic; /* see magic.h */ - short vstamp; /* version stamp */ - long tsize; /* text size in bytes, padded to FW bdry*/ - long dsize; /* initialized data " " */ - long bsize; /* uninitialized data " " */ - long entrypt; /* entry pt. */ - long text_start; /* base of text used for this file */ - long data_start; /* base of data used for this file */ -} AOUTHDR; - -#define AOUTSZ sizeof(AOUTHDR) - -/*----------------------------------------------------------------------*/ -/* When a UNIX aout header is to be built in the optional header, */ -/* the following magic numbers can appear in that header: */ -/* */ -/* AOUT1MAGIC : default : readonly sharable text segment */ -/* AOUT2MAGIC: : writable text segment */ -/* PAGEMAGIC : : configured for paging */ -/*----------------------------------------------------------------------*/ -#define AOUT1MAGIC 0410 -#define AOUT2MAGIC 0407 -#define PAGEMAGIC 0413 - - -/*------------------------------------------------------------------------*/ -/* COMMON ARCHIVE FILE STRUCTURES */ -/* */ -/* ARCHIVE File Organization: */ -/* _______________________________________________ */ -/* |__________ARCHIVE_MAGIC_STRING_______________| */ -/* |__________ARCHIVE_FILE_MEMBER_1______________| */ -/* | | */ -/* | Archive File Header "ar_hdr" | */ -/* |.............................................| */ -/* | Member Contents | */ -/* | 1. External symbol directory | */ -/* | 2. Text file | */ -/* |_____________________________________________| */ -/* |________ARCHIVE_FILE_MEMBER_2________________| */ -/* | "ar_hdr" | */ -/* |.............................................| */ -/* | Member Contents (.o or text file) | */ -/* |_____________________________________________| */ -/* | . . . | */ -/* | . . . | */ -/* | . . . | */ -/* |_____________________________________________| */ -/* |________ARCHIVE_FILE_MEMBER_n________________| */ -/* | "ar_hdr" | */ -/* |.............................................| */ -/* | Member Contents | */ -/* |_____________________________________________| */ -/* */ -/*------------------------------------------------------------------------*/ - -#define COFF_ARMAG "!\n" -#define SARMAG 8 -#define ARFMAG "`\n" - -struct ar_hdr /* archive file member header - printable ascii */ -{ - char ar_name[16]; /* file member name - `/' terminated */ - char ar_date[12]; /* file member date - decimal */ - char ar_uid[6]; /* file member user id - decimal */ - char ar_gid[6]; /* file member group id - decimal */ - char ar_mode[8]; /* file member mode - octal */ - char ar_size[10]; /* file member size - decimal */ - char ar_fmag[2]; /* ARFMAG - string to end header */ -}; - - -/*------------------------------------------------------------------------*/ -/* SECTION HEADER */ -/*------------------------------------------------------------------------*/ -struct scnhdr { - char s_name[8]; /* section name */ - long s_paddr; /* physical address */ - long s_vaddr; /* virtual address */ - long s_size; /* section size */ - long s_scnptr; /* file ptr to raw data for section */ - long s_relptr; /* file ptr to relocation */ - long s_lnnoptr; /* file ptr to line numbers */ - unsigned int s_nreloc; /* number of relocation entries */ - unsigned int s_nlnno; /* number of line number entries */ - unsigned int s_flags; /* flags */ - unsigned short s_reserved; /* reserved byte */ - unsigned short s_page; /* memory page id */ - }; - -#define SCNHDR struct scnhdr -#define SCNHSZ sizeof(SCNHDR) - -/*------------------------------------------------------------------------*/ -/* Define constants for names of "special" sections */ -/*------------------------------------------------------------------------*/ -/* #define _TEXT ".text" */ -#define _DATA ".data" -#define _BSS ".bss" -#define _CINIT ".cinit" -#define _TV ".tv" - -/*------------------------------------------------------------------------*/ -/* The low 4 bits of s_flags is used as a section "type" */ -/*------------------------------------------------------------------------*/ -#define STYP_REG 0x00 /* "regular" : allocated, relocated, loaded */ -#define STYP_DSECT 0x01 /* "dummy" : not allocated, relocated, not loaded */ -#define STYP_NOLOAD 0x02 /* "noload" : allocated, relocated, not loaded */ -#define STYP_GROUP 0x04 /* "grouped" : formed of input sections */ -#define STYP_PAD 0x08 /* "padding" : not allocated, not relocated, loaded */ -#define STYP_COPY 0x10 /* "copy" : used for C init tables - - not allocated, relocated, - loaded; reloc & lineno - entries processed normally */ -#define STYP_TEXT 0x20 /* section contains text only */ -#define STYP_DATA 0x40 /* section contains data only */ -#define STYP_BSS 0x80 /* section contains bss only */ - -#define STYP_ALIGN 0x100 /* align flag passed by old version assemblers */ -#define ALIGN_MASK 0x0F00 /* part of s_flags that is used for align vals */ -#define ALIGNSIZE(x) (1 << ((x & ALIGN_MASK) >> 8)) - - -/*------------------------------------------------------------------------*/ -/* RELOCATION ENTRIES */ -/*------------------------------------------------------------------------*/ -struct reloc -{ - long r_vaddr; /* (virtual) address of reference */ - short r_symndx; /* index into symbol table */ - unsigned short r_disp; /* additional bits for address calculation */ - unsigned short r_type; /* relocation type */ -}; - -#define RELOC struct reloc -#define RELSZ 10 /* sizeof(RELOC) */ - -/*--------------------------------------------------------------------------*/ -/* define all relocation types */ -/*--------------------------------------------------------------------------*/ - -#define R_ABS 0 /* absolute address - no relocation */ -#define R_DIR16 01 /* UNUSED */ -#define R_REL16 02 /* UNUSED */ -#define R_DIR24 04 /* UNUSED */ -#define R_REL24 05 /* 24 bits, direct */ -#define R_DIR32 06 /* UNUSED */ -#define R_RELBYTE 017 /* 8 bits, direct */ -#define R_RELWORD 020 /* 16 bits, direct */ -#define R_RELLONG 021 /* 32 bits, direct */ -#define R_PCRBYTE 022 /* 8 bits, PC-relative */ -#define R_PCRWORD 023 /* 16 bits, PC-relative */ -#define R_PCRLONG 024 /* 32 bits, PC-relative */ -#define R_OCRLONG 030 /* GSP: 32 bits, one's complement direct */ -#define R_GSPPCR16 031 /* GSP: 16 bits, PC relative (in words) */ -#define R_GSPOPR32 032 /* GSP: 32 bits, direct big-endian */ -#define R_PARTLS16 040 /* Brahma: 16 bit offset of 24 bit address*/ -#define R_PARTMS8 041 /* Brahma: 8 bit page of 24 bit address */ -#define R_PARTLS7 050 /* DSP: 7 bit offset of 16 bit address */ -#define R_PARTMS9 051 /* DSP: 9 bit page of 16 bit address */ -#define R_REL13 052 /* DSP: 13 bits, direct */ - - -/*------------------------------------------------------------------------*/ -/* LINE NUMBER ENTRIES */ -/*------------------------------------------------------------------------*/ -struct lineno -{ - union - { - long l_symndx ; /* sym. table index of function name - iff l_lnno == 0 */ - long l_paddr ; /* (physical) address of line number */ - } l_addr ; - unsigned short l_lnno ; /* line number */ -}; - -#define LINENO struct lineno -#define LINESZ 6 /* sizeof(LINENO) */ - - -/*------------------------------------------------------------------------*/ -/* STORAGE CLASSES */ -/*------------------------------------------------------------------------*/ -#define C_EFCN -1 /* physical end of function */ -#define C_NULL 0 -#define C_AUTO 1 /* automatic variable */ -#define C_EXT 2 /* external symbol */ -#define C_STAT 3 /* static */ -#define C_REG 4 /* register variable */ -#define C_EXTDEF 5 /* external definition */ -#define C_LABEL 6 /* label */ -#define C_ULABEL 7 /* undefined label */ -#define C_MOS 8 /* member of structure */ -#define C_ARG 9 /* function argument */ -#define C_STRTAG 10 /* structure tag */ -#define C_MOU 11 /* member of union */ -#define C_UNTAG 12 /* union tag */ -#define C_TPDEF 13 /* type definition */ -#define C_USTATIC 14 /* undefined static */ -#define C_ENTAG 15 /* enumeration tag */ -#define C_MOE 16 /* member of enumeration */ -#define C_REGPARM 17 /* register parameter */ -#define C_FIELD 18 /* bit field */ - -#define C_BLOCK 100 /* ".bb" or ".eb" */ -#define C_FCN 101 /* ".bf" or ".ef" */ -#define C_EOS 102 /* end of structure */ -#define C_FILE 103 /* file name */ -#define C_LINE 104 /* dummy sclass for line number entry */ -#define C_ALIAS 105 /* duplicate tag */ -#define C_HIDDEN 106 /* special storage class for external */ - /* symbols in dmert public libraries */ - -/*------------------------------------------------------------------------*/ -/* SYMBOL TABLE ENTRIES */ -/*------------------------------------------------------------------------*/ - -#define SYMNMLEN 8 /* Number of characters in a symbol name */ -#define FILNMLEN 14 /* Number of characters in a file name */ -#define DIMNUM 4 /* Number of array dimensions in auxiliary entry */ - - -struct syment -{ - union - { - char _n_name[SYMNMLEN]; /* old COFF version */ - struct - { - long _n_zeroes; /* new == 0 */ - long _n_offset; /* offset into string table */ - } _n_n; - char *_n_nptr[2]; /* allows for overlaying */ - } _n; - long n_value; /* value of symbol */ - short n_scnum; /* section number */ - unsigned short n_type; /* type and derived type */ - char n_sclass; /* storage class */ - char n_numaux; /* number of aux. entries */ -}; - -#define n_name _n._n_name -#define n_nptr _n._n_nptr[1] -#define n_zeroes _n._n_n._n_zeroes -#define n_offset _n._n_n._n_offset - -/*------------------------------------------------------------------------*/ -/* Relocatable symbols have a section number of the */ -/* section in which they are defined. Otherwise, section */ -/* numbers have the following meanings: */ -/*------------------------------------------------------------------------*/ -#define N_UNDEF 0 /* undefined symbol */ -#define N_ABS -1 /* value of symbol is absolute */ -#define N_DEBUG -2 /* special debugging symbol */ -#define N_TV (unsigned short)-3 /* needs transfer vector (preload) */ -#define P_TV (unsigned short)-4 /* needs transfer vector (postload) */ - - -/*------------------------------------------------------------------------*/ -/* The fundamental type of a symbol packed into the low */ -/* 4 bits of the word. */ -/*------------------------------------------------------------------------*/ -#define _EF ".ef" - -#define T_NULL 0 /* no type info */ -#define T_ARG 1 /* function argument (only used by compiler) */ -#define T_CHAR 2 /* character */ -#define T_SHORT 3 /* short integer */ -#define T_INT 4 /* integer */ -#define T_LONG 5 /* long integer */ -#define T_FLOAT 6 /* floating point */ -#define T_DOUBLE 7 /* double word */ -#define T_STRUCT 8 /* structure */ -#define T_UNION 9 /* union */ -#define T_ENUM 10 /* enumeration */ -#define T_MOE 11 /* member of enumeration */ -#define T_UCHAR 12 /* unsigned character */ -#define T_USHORT 13 /* unsigned short */ -#define T_UINT 14 /* unsigned integer */ -#define T_ULONG 15 /* unsigned long */ - -/*------------------------------------------------------------------------*/ -/* derived types are: */ -/*------------------------------------------------------------------------*/ -#define DT_NON 0 /* no derived type */ -#define DT_PTR 1 /* pointer */ -#define DT_FCN 2 /* function */ -#define DT_ARY 3 /* array */ - -#define MKTYPE(basic, d1,d2,d3,d4,d5,d6) \ - ((basic) | ((d1) << 4) | ((d2) << 6) | ((d3) << 8) |\ - ((d4) << 10) | ((d5) << 12) | ((d6) << 14)) - -/*------------------------------------------------------------------------*/ -/* type packing constants and macros */ -/*------------------------------------------------------------------------*/ -#define N_BTMASK_COFF 017 -#define N_TMASK_COFF 060 -#define N_TMASK1_COFF 0300 -#define N_TMASK2_COFF 0360 -#define N_BTSHFT_COFF 4 -#define N_TSHIFT_COFF 2 - -#define BTYPE_COFF(x) ((x) & N_BTMASK_COFF) -#define ISINT(x) (((x) >= T_CHAR && (x) <= T_LONG) || \ - ((x) >= T_UCHAR && (x) <= T_ULONG) || (x) == T_ENUM) -#define ISFLT_COFF(x) ((x) == T_DOUBLE || (x) == T_FLOAT) -#define ISPTR_COFF(x) (((x) & N_TMASK_COFF) == (DT_PTR << N_BTSHFT_COFF)) -#define ISFCN_COFF(x) (((x) & N_TMASK_COFF) == (DT_FCN << N_BTSHFT_COFF)) -#define ISARY_COFF(x) (((x) & N_TMASK_COFF) == (DT_ARY << N_BTSHFT_COFF)) -#define ISTAG_COFF(x) ((x)==C_STRTAG || (x)==C_UNTAG || (x)==C_ENTAG) - -#define INCREF_COFF(x) ((((x)&~N_BTMASK_COFF)<>N_TSHIFT_COFF)&~N_BTMASK_COFF)|((x)&N_BTMASK_COFF)) - - -/*------------------------------------------------------------------------*/ -/* AUXILIARY SYMBOL ENTRY */ -/*------------------------------------------------------------------------*/ -union auxent -{ - struct - { - long x_tagndx; /* str, un, or enum tag indx */ - union - { - struct - { - unsigned short x_lnno; /* declaration line number */ - unsigned short x_size; /* str, union, array size */ - } x_lnsz; - long x_fsize; /* size of function */ - } x_misc; - union - { - struct /* if ISFCN, tag, or .bb */ - { - long x_lnnoptr; /* ptr to fcn line # */ - long x_endndx; /* entry ndx past block end */ - } x_fcn; - struct /* if ISARY, up to 4 dimen. */ - { - unsigned short x_dimen[DIMNUM]; - } x_ary; - } x_fcnary; - unsigned short x_regcount; /* number of registers used by func */ - } x_sym; - struct - { - char x_fname[FILNMLEN]; - } x_file; - struct - { - long x_scnlen; /* section length */ - unsigned short x_nreloc; /* number of relocation entries */ - unsigned short x_nlinno; /* number of line numbers */ - } x_scn; -}; - -#define SYMENT struct syment -#define SYMESZ 18 /* sizeof(SYMENT) */ - -#define AUXENT union auxent -#define AUXESZ 18 /* sizeof(AUXENT) */ - -/*------------------------------------------------------------------------*/ -/* NAMES OF "SPECIAL" SYMBOLS */ -/*------------------------------------------------------------------------*/ -#define _STEXT ".text" -#define _ETEXT "etext" -#define _SDATA ".data" -#define _EDATA "edata" -#define _SBSS ".bss" -#define _END "end" -#define _CINITPTR "cinit" - -/*--------------------------------------------------------------------------*/ -/* ENTRY POINT SYMBOLS */ -/*--------------------------------------------------------------------------*/ -#define _START "_start" -#define _MAIN "_main" - /* _CSTART "_c_int00" (defined in params.h) */ - - -#define _TVORIG "_tvorig" -#define _TORIGIN "_torigin" -#define _DORIGIN "_dorigin" - -#define _SORIGIN "_sorigin" diff --git a/vendor/tcc/config.h b/vendor/tcc/config.h deleted file mode 100644 index 35754b08..00000000 --- a/vendor/tcc/config.h +++ /dev/null @@ -1,11 +0,0 @@ -/* Automatically generated by configure - do not modify */ -#ifndef CONFIG_TCCDIR -# define CONFIG_TCCDIR "/usr/local/lib/tcc" -#endif -#ifndef CONFIG_TCC_PREDEFS -# define CONFIG_TCC_PREDEFS 1 -#endif -#define TCC_VERSION "0.9.28rc" -#define GCC_MAJOR 12 -#define GCC_MINOR 2 -#define CC_NAME CC_gcc diff --git a/vendor/tcc/dwarf.h b/vendor/tcc/dwarf.h deleted file mode 100644 index c961bc36..00000000 --- a/vendor/tcc/dwarf.h +++ /dev/null @@ -1,1046 +0,0 @@ -/* This file defines standard DWARF types, structures, and macros. - Copyright (C) 2000-2011, 2014, 2016, 2017, 2018 Red Hat, Inc. - This file is part of elfutils. - - This file is free software; you can redistribute it and/or modify - it under the terms of either - - * the GNU Lesser General Public License as published by the Free - Software Foundation; either version 3 of the License, or (at - your option) any later version - - or - - * the GNU General Public License as published by the Free - Software Foundation; either version 2 of the License, or (at - your option) any later version - - or both in parallel, as here. - - elfutils is distributed in the hope that it will be useful, but - WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - General Public License for more details. - - You should have received copies of the GNU General Public License and - the GNU Lesser General Public License along with this program. If - not, see . */ - -#ifndef _DWARF_H -#define _DWARF_H 1 - -/* DWARF Unit Header Types. */ -enum - { - DW_UT_compile = 0x01, - DW_UT_type = 0x02, - DW_UT_partial = 0x03, - DW_UT_skeleton = 0x04, - DW_UT_split_compile = 0x05, - DW_UT_split_type = 0x06, - - DW_UT_lo_user = 0x80, - DW_UT_hi_user = 0xff - }; - -/* DWARF tags. */ -enum - { - DW_TAG_array_type = 0x01, - DW_TAG_class_type = 0x02, - DW_TAG_entry_point = 0x03, - DW_TAG_enumeration_type = 0x04, - DW_TAG_formal_parameter = 0x05, - /* 0x06 reserved. */ - /* 0x07 reserved. */ - DW_TAG_imported_declaration = 0x08, - /* 0x09 reserved. */ - DW_TAG_label = 0x0a, - DW_TAG_lexical_block = 0x0b, - /* 0x0c reserved. */ - DW_TAG_member = 0x0d, - /* 0x0e reserved. */ - DW_TAG_pointer_type = 0x0f, - DW_TAG_reference_type = 0x10, - DW_TAG_compile_unit = 0x11, - DW_TAG_string_type = 0x12, - DW_TAG_structure_type = 0x13, - /* 0x14 reserved. */ - DW_TAG_subroutine_type = 0x15, - DW_TAG_typedef = 0x16, - DW_TAG_union_type = 0x17, - DW_TAG_unspecified_parameters = 0x18, - DW_TAG_variant = 0x19, - DW_TAG_common_block = 0x1a, - DW_TAG_common_inclusion = 0x1b, - DW_TAG_inheritance = 0x1c, - DW_TAG_inlined_subroutine = 0x1d, - DW_TAG_module = 0x1e, - DW_TAG_ptr_to_member_type = 0x1f, - DW_TAG_set_type = 0x20, - DW_TAG_subrange_type = 0x21, - DW_TAG_with_stmt = 0x22, - DW_TAG_access_declaration = 0x23, - DW_TAG_base_type = 0x24, - DW_TAG_catch_block = 0x25, - DW_TAG_const_type = 0x26, - DW_TAG_constant = 0x27, - DW_TAG_enumerator = 0x28, - DW_TAG_file_type = 0x29, - DW_TAG_friend = 0x2a, - DW_TAG_namelist = 0x2b, - DW_TAG_namelist_item = 0x2c, - DW_TAG_packed_type = 0x2d, - DW_TAG_subprogram = 0x2e, - DW_TAG_template_type_parameter = 0x2f, - DW_TAG_template_value_parameter = 0x30, - DW_TAG_thrown_type = 0x31, - DW_TAG_try_block = 0x32, - DW_TAG_variant_part = 0x33, - DW_TAG_variable = 0x34, - DW_TAG_volatile_type = 0x35, - DW_TAG_dwarf_procedure = 0x36, - DW_TAG_restrict_type = 0x37, - DW_TAG_interface_type = 0x38, - DW_TAG_namespace = 0x39, - DW_TAG_imported_module = 0x3a, - DW_TAG_unspecified_type = 0x3b, - DW_TAG_partial_unit = 0x3c, - DW_TAG_imported_unit = 0x3d, - /* 0x3e reserved. Was DW_TAG_mutable_type. */ - DW_TAG_condition = 0x3f, - DW_TAG_shared_type = 0x40, - DW_TAG_type_unit = 0x41, - DW_TAG_rvalue_reference_type = 0x42, - DW_TAG_template_alias = 0x43, - DW_TAG_coarray_type = 0x44, - DW_TAG_generic_subrange = 0x45, - DW_TAG_dynamic_type = 0x46, - DW_TAG_atomic_type = 0x47, - DW_TAG_call_site = 0x48, - DW_TAG_call_site_parameter = 0x49, - DW_TAG_skeleton_unit = 0x4a, - DW_TAG_immutable_type = 0x4b, - - DW_TAG_lo_user = 0x4080, - - DW_TAG_MIPS_loop = 0x4081, - DW_TAG_format_label = 0x4101, - DW_TAG_function_template = 0x4102, - DW_TAG_class_template = 0x4103, - - DW_TAG_GNU_BINCL = 0x4104, - DW_TAG_GNU_EINCL = 0x4105, - - DW_TAG_GNU_template_template_param = 0x4106, - DW_TAG_GNU_template_parameter_pack = 0x4107, - DW_TAG_GNU_formal_parameter_pack = 0x4108, - DW_TAG_GNU_call_site = 0x4109, - DW_TAG_GNU_call_site_parameter = 0x410a, - - DW_TAG_hi_user = 0xffff - }; - - -/* Children determination encodings. */ -enum - { - DW_CHILDREN_no = 0, - DW_CHILDREN_yes = 1 - }; - - -/* DWARF attributes encodings. */ -enum - { - DW_AT_sibling = 0x01, - DW_AT_location = 0x02, - DW_AT_name = 0x03, - /* 0x04 reserved. */ - /* 0x05 reserved. */ - /* 0x06 reserved. */ - /* 0x07 reserved. */ - /* 0x08 reserved. */ - DW_AT_ordering = 0x09, - /* 0x0a reserved. */ - DW_AT_byte_size = 0x0b, - DW_AT_bit_offset = 0x0c, /* Deprecated in DWARF4. */ - DW_AT_bit_size = 0x0d, - /* 0x0e reserved. */ - /* 0x0f reserved. */ - DW_AT_stmt_list = 0x10, - DW_AT_low_pc = 0x11, - DW_AT_high_pc = 0x12, - DW_AT_language = 0x13, - /* 0x14 reserved. */ - DW_AT_discr = 0x15, - DW_AT_discr_value = 0x16, - DW_AT_visibility = 0x17, - DW_AT_import = 0x18, - DW_AT_string_length = 0x19, - DW_AT_common_reference = 0x1a, - DW_AT_comp_dir = 0x1b, - DW_AT_const_value = 0x1c, - DW_AT_containing_type = 0x1d, - DW_AT_default_value = 0x1e, - /* 0x1f reserved. */ - DW_AT_inline = 0x20, - DW_AT_is_optional = 0x21, - DW_AT_lower_bound = 0x22, - /* 0x23 reserved. */ - /* 0x24 reserved. */ - DW_AT_producer = 0x25, - /* 0x26 reserved. */ - DW_AT_prototyped = 0x27, - /* 0x28 reserved. */ - /* 0x29 reserved. */ - DW_AT_return_addr = 0x2a, - /* 0x2b reserved. */ - DW_AT_start_scope = 0x2c, - /* 0x2d reserved. */ - DW_AT_bit_stride = 0x2e, - DW_AT_upper_bound = 0x2f, - /* 0x30 reserved. */ - DW_AT_abstract_origin = 0x31, - DW_AT_accessibility = 0x32, - DW_AT_address_class = 0x33, - DW_AT_artificial = 0x34, - DW_AT_base_types = 0x35, - DW_AT_calling_convention = 0x36, - DW_AT_count = 0x37, - DW_AT_data_member_location = 0x38, - DW_AT_decl_column = 0x39, - DW_AT_decl_file = 0x3a, - DW_AT_decl_line = 0x3b, - DW_AT_declaration = 0x3c, - DW_AT_discr_list = 0x3d, - DW_AT_encoding = 0x3e, - DW_AT_external = 0x3f, - DW_AT_frame_base = 0x40, - DW_AT_friend = 0x41, - DW_AT_identifier_case = 0x42, - DW_AT_macro_info = 0x43, /* Deprecated in DWARF5. */ - DW_AT_namelist_item = 0x44, - DW_AT_priority = 0x45, - DW_AT_segment = 0x46, - DW_AT_specification = 0x47, - DW_AT_static_link = 0x48, - DW_AT_type = 0x49, - DW_AT_use_location = 0x4a, - DW_AT_variable_parameter = 0x4b, - DW_AT_virtuality = 0x4c, - DW_AT_vtable_elem_location = 0x4d, - DW_AT_allocated = 0x4e, - DW_AT_associated = 0x4f, - DW_AT_data_location = 0x50, - DW_AT_byte_stride = 0x51, - DW_AT_entry_pc = 0x52, - DW_AT_use_UTF8 = 0x53, - DW_AT_extension = 0x54, - DW_AT_ranges = 0x55, - DW_AT_trampoline = 0x56, - DW_AT_call_column = 0x57, - DW_AT_call_file = 0x58, - DW_AT_call_line = 0x59, - DW_AT_description = 0x5a, - DW_AT_binary_scale = 0x5b, - DW_AT_decimal_scale = 0x5c, - DW_AT_small = 0x5d, - DW_AT_decimal_sign = 0x5e, - DW_AT_digit_count = 0x5f, - DW_AT_picture_string = 0x60, - DW_AT_mutable = 0x61, - DW_AT_threads_scaled = 0x62, - DW_AT_explicit = 0x63, - DW_AT_object_pointer = 0x64, - DW_AT_endianity = 0x65, - DW_AT_elemental = 0x66, - DW_AT_pure = 0x67, - DW_AT_recursive = 0x68, - DW_AT_signature = 0x69, - DW_AT_main_subprogram = 0x6a, - DW_AT_data_bit_offset = 0x6b, - DW_AT_const_expr = 0x6c, - DW_AT_enum_class = 0x6d, - DW_AT_linkage_name = 0x6e, - DW_AT_string_length_bit_size = 0x6f, - DW_AT_string_length_byte_size = 0x70, - DW_AT_rank = 0x71, - DW_AT_str_offsets_base = 0x72, - DW_AT_addr_base = 0x73, - DW_AT_rnglists_base = 0x74, - /* 0x75 reserved. */ - DW_AT_dwo_name = 0x76, - DW_AT_reference = 0x77, - DW_AT_rvalue_reference = 0x78, - DW_AT_macros = 0x79, - DW_AT_call_all_calls = 0x7a, - DW_AT_call_all_source_calls = 0x7b, - DW_AT_call_all_tail_calls = 0x7c, - DW_AT_call_return_pc = 0x7d, - DW_AT_call_value = 0x7e, - DW_AT_call_origin = 0x7f, - DW_AT_call_parameter = 0x80, - DW_AT_call_pc = 0x81, - DW_AT_call_tail_call = 0x82, - DW_AT_call_target = 0x83, - DW_AT_call_target_clobbered = 0x84, - DW_AT_call_data_location = 0x85, - DW_AT_call_data_value = 0x86, - DW_AT_noreturn = 0x87, - DW_AT_alignment = 0x88, - DW_AT_export_symbols = 0x89, - DW_AT_deleted = 0x8a, - DW_AT_defaulted = 0x8b, - DW_AT_loclists_base = 0x8c, - - DW_AT_lo_user = 0x2000, - - DW_AT_MIPS_fde = 0x2001, - DW_AT_MIPS_loop_begin = 0x2002, - DW_AT_MIPS_tail_loop_begin = 0x2003, - DW_AT_MIPS_epilog_begin = 0x2004, - DW_AT_MIPS_loop_unroll_factor = 0x2005, - DW_AT_MIPS_software_pipeline_depth = 0x2006, - DW_AT_MIPS_linkage_name = 0x2007, - DW_AT_MIPS_stride = 0x2008, - DW_AT_MIPS_abstract_name = 0x2009, - DW_AT_MIPS_clone_origin = 0x200a, - DW_AT_MIPS_has_inlines = 0x200b, - DW_AT_MIPS_stride_byte = 0x200c, - DW_AT_MIPS_stride_elem = 0x200d, - DW_AT_MIPS_ptr_dopetype = 0x200e, - DW_AT_MIPS_allocatable_dopetype = 0x200f, - DW_AT_MIPS_assumed_shape_dopetype = 0x2010, - DW_AT_MIPS_assumed_size = 0x2011, - - /* GNU extensions. */ - DW_AT_sf_names = 0x2101, - DW_AT_src_info = 0x2102, - DW_AT_mac_info = 0x2103, - DW_AT_src_coords = 0x2104, - DW_AT_body_begin = 0x2105, - DW_AT_body_end = 0x2106, - DW_AT_GNU_vector = 0x2107, - DW_AT_GNU_guarded_by = 0x2108, - DW_AT_GNU_pt_guarded_by = 0x2109, - DW_AT_GNU_guarded = 0x210a, - DW_AT_GNU_pt_guarded = 0x210b, - DW_AT_GNU_locks_excluded = 0x210c, - DW_AT_GNU_exclusive_locks_required = 0x210d, - DW_AT_GNU_shared_locks_required = 0x210e, - DW_AT_GNU_odr_signature = 0x210f, - DW_AT_GNU_template_name = 0x2110, - DW_AT_GNU_call_site_value = 0x2111, - DW_AT_GNU_call_site_data_value = 0x2112, - DW_AT_GNU_call_site_target = 0x2113, - DW_AT_GNU_call_site_target_clobbered = 0x2114, - DW_AT_GNU_tail_call = 0x2115, - DW_AT_GNU_all_tail_call_sites = 0x2116, - DW_AT_GNU_all_call_sites = 0x2117, - DW_AT_GNU_all_source_call_sites = 0x2118, - DW_AT_GNU_locviews = 0x2137, - DW_AT_GNU_entry_view = 0x2138, - DW_AT_GNU_macros = 0x2119, - DW_AT_GNU_deleted = 0x211a, - /* GNU Debug Fission extensions. */ - DW_AT_GNU_dwo_name = 0x2130, - DW_AT_GNU_dwo_id = 0x2131, - DW_AT_GNU_ranges_base = 0x2132, - DW_AT_GNU_addr_base = 0x2133, - DW_AT_GNU_pubnames = 0x2134, - DW_AT_GNU_pubtypes = 0x2135, - - /* https://gcc.gnu.org/wiki/DW_AT_GNU_numerator_denominator */ - DW_AT_GNU_numerator = 0x2303, - DW_AT_GNU_denominator = 0x2304, - /* https://gcc.gnu.org/wiki/DW_AT_GNU_bias */ - DW_AT_GNU_bias = 0x2305, - - DW_AT_hi_user = 0x3fff - }; - -/* Old unofficially attribute names. Should not be used. - Will not appear in known-dwarf.h */ - -/* DWARF1 array subscripts and element data types. */ -#define DW_AT_subscr_data 0x0a -/* DWARF1 enumeration literals. */ -#define DW_AT_element_list 0x0f -/* DWARF1 reference for variable to member structure, class or union. */ -#define DW_AT_member 0x14 - -/* DWARF form encodings. */ -enum - { - DW_FORM_addr = 0x01, - DW_FORM_block2 = 0x03, - DW_FORM_block4 = 0x04, - DW_FORM_data2 = 0x05, - DW_FORM_data4 = 0x06, - DW_FORM_data8 = 0x07, - DW_FORM_string = 0x08, - DW_FORM_block = 0x09, - DW_FORM_block1 = 0x0a, - DW_FORM_data1 = 0x0b, - DW_FORM_flag = 0x0c, - DW_FORM_sdata = 0x0d, - DW_FORM_strp = 0x0e, - DW_FORM_udata = 0x0f, - DW_FORM_ref_addr = 0x10, - DW_FORM_ref1 = 0x11, - DW_FORM_ref2 = 0x12, - DW_FORM_ref4 = 0x13, - DW_FORM_ref8 = 0x14, - DW_FORM_ref_udata = 0x15, - DW_FORM_indirect = 0x16, - DW_FORM_sec_offset = 0x17, - DW_FORM_exprloc = 0x18, - DW_FORM_flag_present = 0x19, - DW_FORM_strx = 0x1a, - DW_FORM_addrx = 0x1b, - DW_FORM_ref_sup4 = 0x1c, - DW_FORM_strp_sup = 0x1d, - DW_FORM_data16 = 0x1e, - DW_FORM_line_strp = 0x1f, - DW_FORM_ref_sig8 = 0x20, - DW_FORM_implicit_const = 0x21, - DW_FORM_loclistx = 0x22, - DW_FORM_rnglistx = 0x23, - DW_FORM_ref_sup8 = 0x24, - DW_FORM_strx1 = 0x25, - DW_FORM_strx2 = 0x26, - DW_FORM_strx3 = 0x27, - DW_FORM_strx4 = 0x28, - DW_FORM_addrx1 = 0x29, - DW_FORM_addrx2 = 0x2a, - DW_FORM_addrx3 = 0x2b, - DW_FORM_addrx4 = 0x2c, - - /* GNU Debug Fission extensions. */ - DW_FORM_GNU_addr_index = 0x1f01, - DW_FORM_GNU_str_index = 0x1f02, - - DW_FORM_GNU_ref_alt = 0x1f20, /* offset in alternate .debuginfo. */ - DW_FORM_GNU_strp_alt = 0x1f21 /* offset in alternate .debug_str. */ - }; - - -/* DWARF location operation encodings. */ -enum - { - DW_OP_addr = 0x03, /* Constant address. */ - DW_OP_deref = 0x06, - DW_OP_const1u = 0x08, /* Unsigned 1-byte constant. */ - DW_OP_const1s = 0x09, /* Signed 1-byte constant. */ - DW_OP_const2u = 0x0a, /* Unsigned 2-byte constant. */ - DW_OP_const2s = 0x0b, /* Signed 2-byte constant. */ - DW_OP_const4u = 0x0c, /* Unsigned 4-byte constant. */ - DW_OP_const4s = 0x0d, /* Signed 4-byte constant. */ - DW_OP_const8u = 0x0e, /* Unsigned 8-byte constant. */ - DW_OP_const8s = 0x0f, /* Signed 8-byte constant. */ - DW_OP_constu = 0x10, /* Unsigned LEB128 constant. */ - DW_OP_consts = 0x11, /* Signed LEB128 constant. */ - DW_OP_dup = 0x12, - DW_OP_drop = 0x13, - DW_OP_over = 0x14, - DW_OP_pick = 0x15, /* 1-byte stack index. */ - DW_OP_swap = 0x16, - DW_OP_rot = 0x17, - DW_OP_xderef = 0x18, - DW_OP_abs = 0x19, - DW_OP_and = 0x1a, - DW_OP_div = 0x1b, - DW_OP_minus = 0x1c, - DW_OP_mod = 0x1d, - DW_OP_mul = 0x1e, - DW_OP_neg = 0x1f, - DW_OP_not = 0x20, - DW_OP_or = 0x21, - DW_OP_plus = 0x22, - DW_OP_plus_uconst = 0x23, /* Unsigned LEB128 addend. */ - DW_OP_shl = 0x24, - DW_OP_shr = 0x25, - DW_OP_shra = 0x26, - DW_OP_xor = 0x27, - DW_OP_bra = 0x28, /* Signed 2-byte constant. */ - DW_OP_eq = 0x29, - DW_OP_ge = 0x2a, - DW_OP_gt = 0x2b, - DW_OP_le = 0x2c, - DW_OP_lt = 0x2d, - DW_OP_ne = 0x2e, - DW_OP_skip = 0x2f, /* Signed 2-byte constant. */ - DW_OP_lit0 = 0x30, /* Literal 0. */ - DW_OP_lit1 = 0x31, /* Literal 1. */ - DW_OP_lit2 = 0x32, /* Literal 2. */ - DW_OP_lit3 = 0x33, /* Literal 3. */ - DW_OP_lit4 = 0x34, /* Literal 4. */ - DW_OP_lit5 = 0x35, /* Literal 5. */ - DW_OP_lit6 = 0x36, /* Literal 6. */ - DW_OP_lit7 = 0x37, /* Literal 7. */ - DW_OP_lit8 = 0x38, /* Literal 8. */ - DW_OP_lit9 = 0x39, /* Literal 9. */ - DW_OP_lit10 = 0x3a, /* Literal 10. */ - DW_OP_lit11 = 0x3b, /* Literal 11. */ - DW_OP_lit12 = 0x3c, /* Literal 12. */ - DW_OP_lit13 = 0x3d, /* Literal 13. */ - DW_OP_lit14 = 0x3e, /* Literal 14. */ - DW_OP_lit15 = 0x3f, /* Literal 15. */ - DW_OP_lit16 = 0x40, /* Literal 16. */ - DW_OP_lit17 = 0x41, /* Literal 17. */ - DW_OP_lit18 = 0x42, /* Literal 18. */ - DW_OP_lit19 = 0x43, /* Literal 19. */ - DW_OP_lit20 = 0x44, /* Literal 20. */ - DW_OP_lit21 = 0x45, /* Literal 21. */ - DW_OP_lit22 = 0x46, /* Literal 22. */ - DW_OP_lit23 = 0x47, /* Literal 23. */ - DW_OP_lit24 = 0x48, /* Literal 24. */ - DW_OP_lit25 = 0x49, /* Literal 25. */ - DW_OP_lit26 = 0x4a, /* Literal 26. */ - DW_OP_lit27 = 0x4b, /* Literal 27. */ - DW_OP_lit28 = 0x4c, /* Literal 28. */ - DW_OP_lit29 = 0x4d, /* Literal 29. */ - DW_OP_lit30 = 0x4e, /* Literal 30. */ - DW_OP_lit31 = 0x4f, /* Literal 31. */ - DW_OP_reg0 = 0x50, /* Register 0. */ - DW_OP_reg1 = 0x51, /* Register 1. */ - DW_OP_reg2 = 0x52, /* Register 2. */ - DW_OP_reg3 = 0x53, /* Register 3. */ - DW_OP_reg4 = 0x54, /* Register 4. */ - DW_OP_reg5 = 0x55, /* Register 5. */ - DW_OP_reg6 = 0x56, /* Register 6. */ - DW_OP_reg7 = 0x57, /* Register 7. */ - DW_OP_reg8 = 0x58, /* Register 8. */ - DW_OP_reg9 = 0x59, /* Register 9. */ - DW_OP_reg10 = 0x5a, /* Register 10. */ - DW_OP_reg11 = 0x5b, /* Register 11. */ - DW_OP_reg12 = 0x5c, /* Register 12. */ - DW_OP_reg13 = 0x5d, /* Register 13. */ - DW_OP_reg14 = 0x5e, /* Register 14. */ - DW_OP_reg15 = 0x5f, /* Register 15. */ - DW_OP_reg16 = 0x60, /* Register 16. */ - DW_OP_reg17 = 0x61, /* Register 17. */ - DW_OP_reg18 = 0x62, /* Register 18. */ - DW_OP_reg19 = 0x63, /* Register 19. */ - DW_OP_reg20 = 0x64, /* Register 20. */ - DW_OP_reg21 = 0x65, /* Register 21. */ - DW_OP_reg22 = 0x66, /* Register 22. */ - DW_OP_reg23 = 0x67, /* Register 24. */ - DW_OP_reg24 = 0x68, /* Register 24. */ - DW_OP_reg25 = 0x69, /* Register 25. */ - DW_OP_reg26 = 0x6a, /* Register 26. */ - DW_OP_reg27 = 0x6b, /* Register 27. */ - DW_OP_reg28 = 0x6c, /* Register 28. */ - DW_OP_reg29 = 0x6d, /* Register 29. */ - DW_OP_reg30 = 0x6e, /* Register 30. */ - DW_OP_reg31 = 0x6f, /* Register 31. */ - DW_OP_breg0 = 0x70, /* Base register 0. */ - DW_OP_breg1 = 0x71, /* Base register 1. */ - DW_OP_breg2 = 0x72, /* Base register 2. */ - DW_OP_breg3 = 0x73, /* Base register 3. */ - DW_OP_breg4 = 0x74, /* Base register 4. */ - DW_OP_breg5 = 0x75, /* Base register 5. */ - DW_OP_breg6 = 0x76, /* Base register 6. */ - DW_OP_breg7 = 0x77, /* Base register 7. */ - DW_OP_breg8 = 0x78, /* Base register 8. */ - DW_OP_breg9 = 0x79, /* Base register 9. */ - DW_OP_breg10 = 0x7a, /* Base register 10. */ - DW_OP_breg11 = 0x7b, /* Base register 11. */ - DW_OP_breg12 = 0x7c, /* Base register 12. */ - DW_OP_breg13 = 0x7d, /* Base register 13. */ - DW_OP_breg14 = 0x7e, /* Base register 14. */ - DW_OP_breg15 = 0x7f, /* Base register 15. */ - DW_OP_breg16 = 0x80, /* Base register 16. */ - DW_OP_breg17 = 0x81, /* Base register 17. */ - DW_OP_breg18 = 0x82, /* Base register 18. */ - DW_OP_breg19 = 0x83, /* Base register 19. */ - DW_OP_breg20 = 0x84, /* Base register 20. */ - DW_OP_breg21 = 0x85, /* Base register 21. */ - DW_OP_breg22 = 0x86, /* Base register 22. */ - DW_OP_breg23 = 0x87, /* Base register 23. */ - DW_OP_breg24 = 0x88, /* Base register 24. */ - DW_OP_breg25 = 0x89, /* Base register 25. */ - DW_OP_breg26 = 0x8a, /* Base register 26. */ - DW_OP_breg27 = 0x8b, /* Base register 27. */ - DW_OP_breg28 = 0x8c, /* Base register 28. */ - DW_OP_breg29 = 0x8d, /* Base register 29. */ - DW_OP_breg30 = 0x8e, /* Base register 30. */ - DW_OP_breg31 = 0x8f, /* Base register 31. */ - DW_OP_regx = 0x90, /* Unsigned LEB128 register. */ - DW_OP_fbreg = 0x91, /* Signed LEB128 offset. */ - DW_OP_bregx = 0x92, /* ULEB128 register followed by SLEB128 off. */ - DW_OP_piece = 0x93, /* ULEB128 size of piece addressed. */ - DW_OP_deref_size = 0x94, /* 1-byte size of data retrieved. */ - DW_OP_xderef_size = 0x95, /* 1-byte size of data retrieved. */ - DW_OP_nop = 0x96, - DW_OP_push_object_address = 0x97, - DW_OP_call2 = 0x98, - DW_OP_call4 = 0x99, - DW_OP_call_ref = 0x9a, - DW_OP_form_tls_address = 0x9b,/* TLS offset to address in current thread */ - DW_OP_call_frame_cfa = 0x9c,/* CFA as determined by CFI. */ - DW_OP_bit_piece = 0x9d, /* ULEB128 size and ULEB128 offset in bits. */ - DW_OP_implicit_value = 0x9e, /* DW_FORM_block follows opcode. */ - DW_OP_stack_value = 0x9f, /* No operands, special like DW_OP_piece. */ - - DW_OP_implicit_pointer = 0xa0, - DW_OP_addrx = 0xa1, - DW_OP_constx = 0xa2, - DW_OP_entry_value = 0xa3, - DW_OP_const_type = 0xa4, - DW_OP_regval_type = 0xa5, - DW_OP_deref_type = 0xa6, - DW_OP_xderef_type = 0xa7, - DW_OP_convert = 0xa8, - DW_OP_reinterpret = 0xa9, - - /* GNU extensions. */ - DW_OP_GNU_push_tls_address = 0xe0, - DW_OP_GNU_uninit = 0xf0, - DW_OP_GNU_encoded_addr = 0xf1, - DW_OP_GNU_implicit_pointer = 0xf2, - DW_OP_GNU_entry_value = 0xf3, - DW_OP_GNU_const_type = 0xf4, - DW_OP_GNU_regval_type = 0xf5, - DW_OP_GNU_deref_type = 0xf6, - DW_OP_GNU_convert = 0xf7, - DW_OP_GNU_reinterpret = 0xf9, - DW_OP_GNU_parameter_ref = 0xfa, - - /* GNU Debug Fission extensions. */ - DW_OP_GNU_addr_index = 0xfb, - DW_OP_GNU_const_index = 0xfc, - - DW_OP_GNU_variable_value = 0xfd, - - DW_OP_lo_user = 0xe0, /* Implementation-defined range start. */ - DW_OP_hi_user = 0xff /* Implementation-defined range end. */ - }; - - -/* DWARF base type encodings. */ -enum - { - DW_ATE_void = 0x0, - DW_ATE_address = 0x1, - DW_ATE_boolean = 0x2, - DW_ATE_complex_float = 0x3, - DW_ATE_float = 0x4, - DW_ATE_signed = 0x5, - DW_ATE_signed_char = 0x6, - DW_ATE_unsigned = 0x7, - DW_ATE_unsigned_char = 0x8, - DW_ATE_imaginary_float = 0x9, - DW_ATE_packed_decimal = 0xa, - DW_ATE_numeric_string = 0xb, - DW_ATE_edited = 0xc, - DW_ATE_signed_fixed = 0xd, - DW_ATE_unsigned_fixed = 0xe, - DW_ATE_decimal_float = 0xf, - DW_ATE_UTF = 0x10, - DW_ATE_UCS = 0x11, - DW_ATE_ASCII = 0x12, - - DW_ATE_lo_user = 0x80, - DW_ATE_hi_user = 0xff - }; - - -/* DWARF decimal sign encodings. */ -enum - { - DW_DS_unsigned = 1, - DW_DS_leading_overpunch = 2, - DW_DS_trailing_overpunch = 3, - DW_DS_leading_separate = 4, - DW_DS_trailing_separate = 5, - }; - - -/* DWARF endianity encodings. */ -enum - { - DW_END_default = 0, - DW_END_big = 1, - DW_END_little = 2, - - DW_END_lo_user = 0x40, - DW_END_hi_user = 0xff - }; - - -/* DWARF accessibility encodings. */ -enum - { - DW_ACCESS_public = 1, - DW_ACCESS_protected = 2, - DW_ACCESS_private = 3 - }; - - -/* DWARF visibility encodings. */ -enum - { - DW_VIS_local = 1, - DW_VIS_exported = 2, - DW_VIS_qualified = 3 - }; - - -/* DWARF virtuality encodings. */ -enum - { - DW_VIRTUALITY_none = 0, - DW_VIRTUALITY_virtual = 1, - DW_VIRTUALITY_pure_virtual = 2 - }; - - -/* DWARF language encodings. */ -enum - { - DW_LANG_C89 = 0x0001, /* ISO C:1989 */ - DW_LANG_C = 0x0002, /* C */ - DW_LANG_Ada83 = 0x0003, /* ISO Ada:1983 */ - DW_LANG_C_plus_plus = 0x0004, /* ISO C++:1998 */ - DW_LANG_Cobol74 = 0x0005, /* ISO Cobol:1974 */ - DW_LANG_Cobol85 = 0x0006, /* ISO Cobol:1985 */ - DW_LANG_Fortran77 = 0x0007, /* ISO FORTRAN 77 */ - DW_LANG_Fortran90 = 0x0008, /* ISO Fortran 90 */ - DW_LANG_Pascal83 = 0x0009, /* ISO Pascal:1983 */ - DW_LANG_Modula2 = 0x000a, /* ISO Modula-2:1996 */ - DW_LANG_Java = 0x000b, /* Java */ - DW_LANG_C99 = 0x000c, /* ISO C:1999 */ - DW_LANG_Ada95 = 0x000d, /* ISO Ada:1995 */ - DW_LANG_Fortran95 = 0x000e, /* ISO Fortran 95 */ - DW_LANG_PLI = 0x000f, /* ISO PL/1:1976 */ - DW_LANG_ObjC = 0x0010, /* Objective-C */ - DW_LANG_ObjC_plus_plus = 0x0011, /* Objective-C++ */ - DW_LANG_UPC = 0x0012, /* Unified Parallel C */ - DW_LANG_D = 0x0013, /* D */ - DW_LANG_Python = 0x0014, /* Python */ - DW_LANG_OpenCL = 0x0015, /* OpenCL */ - DW_LANG_Go = 0x0016, /* Go */ - DW_LANG_Modula3 = 0x0017, /* Modula-3 */ - DW_LANG_Haskell = 0x0018, /* Haskell */ - DW_LANG_C_plus_plus_03 = 0x0019, /* ISO C++:2003 */ - DW_LANG_C_plus_plus_11 = 0x001a, /* ISO C++:2011 */ - DW_LANG_OCaml = 0x001b, /* OCaml */ - DW_LANG_Rust = 0x001c, /* Rust */ - DW_LANG_C11 = 0x001d, /* ISO C:2011 */ - DW_LANG_Swift = 0x001e, /* Swift */ - DW_LANG_Julia = 0x001f, /* Julia */ - DW_LANG_Dylan = 0x0020, /* Dylan */ - DW_LANG_C_plus_plus_14 = 0x0021, /* ISO C++:2014 */ - DW_LANG_Fortran03 = 0x0022, /* ISO/IEC 1539-1:2004 */ - DW_LANG_Fortran08 = 0x0023, /* ISO/IEC 1539-1:2010 */ - DW_LANG_RenderScript = 0x0024, /* RenderScript Kernal Language */ - DW_LANG_BLISS = 0x0025, /* BLISS */ - - DW_LANG_lo_user = 0x8000, - DW_LANG_Mips_Assembler = 0x8001, /* Assembler */ - DW_LANG_hi_user = 0xffff - }; - -/* Old (typo) '1' != 'I'. */ -#define DW_LANG_PL1 DW_LANG_PLI - -/* DWARF identifier case encodings. */ -enum - { - DW_ID_case_sensitive = 0, - DW_ID_up_case = 1, - DW_ID_down_case = 2, - DW_ID_case_insensitive = 3 - }; - - -/* DWARF calling conventions encodings. - Used as values of DW_AT_calling_convention for subroutines - (normal, program or nocall) or structures, unions and class types - (normal, reference or value). */ -enum - { - DW_CC_normal = 0x1, - DW_CC_program = 0x2, - DW_CC_nocall = 0x3, - DW_CC_pass_by_reference = 0x4, - DW_CC_pass_by_value = 0x5, - DW_CC_lo_user = 0x40, - DW_CC_hi_user = 0xff - }; - - -/* DWARF inline encodings. */ -enum - { - DW_INL_not_inlined = 0, - DW_INL_inlined = 1, - DW_INL_declared_not_inlined = 2, - DW_INL_declared_inlined = 3 - }; - - -/* DWARF ordering encodings. */ -enum - { - DW_ORD_row_major = 0, - DW_ORD_col_major = 1 - }; - - -/* DWARF discriminant descriptor encodings. */ -enum - { - DW_DSC_label = 0, - DW_DSC_range = 1 - }; - -/* DWARF defaulted member function encodings. */ -enum - { - DW_DEFAULTED_no = 0, - DW_DEFAULTED_in_class = 1, - DW_DEFAULTED_out_of_class = 2 - }; - -/* DWARF line content descriptions. */ -enum - { - DW_LNCT_path = 0x1, - DW_LNCT_directory_index = 0x2, - DW_LNCT_timestamp = 0x3, - DW_LNCT_size = 0x4, - DW_LNCT_MD5 = 0x5, - DW_LNCT_lo_user = 0x2000, - DW_LNCT_hi_user = 0x3fff - }; - -/* DWARF standard opcode encodings. */ -enum - { - DW_LNS_copy = 1, - DW_LNS_advance_pc = 2, - DW_LNS_advance_line = 3, - DW_LNS_set_file = 4, - DW_LNS_set_column = 5, - DW_LNS_negate_stmt = 6, - DW_LNS_set_basic_block = 7, - DW_LNS_const_add_pc = 8, - DW_LNS_fixed_advance_pc = 9, - DW_LNS_set_prologue_end = 10, - DW_LNS_set_epilogue_begin = 11, - DW_LNS_set_isa = 12 - }; - - -/* DWARF extended opcode encodings. */ -enum - { - DW_LNE_end_sequence = 1, - DW_LNE_set_address = 2, - DW_LNE_define_file = 3, - DW_LNE_set_discriminator = 4, - - DW_LNE_lo_user = 128, - - DW_LNE_NVIDIA_inlined_call = 144, - DW_LNE_NVIDIA_set_function_name = 145, - - DW_LNE_hi_user = 255 - }; - - -/* DWARF macinfo type encodings. */ -enum - { - DW_MACINFO_define = 1, - DW_MACINFO_undef = 2, - DW_MACINFO_start_file = 3, - DW_MACINFO_end_file = 4, - DW_MACINFO_vendor_ext = 255 - }; - - -/* DWARF debug_macro type encodings. */ -enum - { - DW_MACRO_define = 0x01, - DW_MACRO_undef = 0x02, - DW_MACRO_start_file = 0x03, - DW_MACRO_end_file = 0x04, - DW_MACRO_define_strp = 0x05, - DW_MACRO_undef_strp = 0x06, - DW_MACRO_import = 0x07, - DW_MACRO_define_sup = 0x08, - DW_MACRO_undef_sup = 0x09, - DW_MACRO_import_sup = 0x0a, - DW_MACRO_define_strx = 0x0b, - DW_MACRO_undef_strx = 0x0c, - DW_MACRO_lo_user = 0xe0, - DW_MACRO_hi_user = 0xff - }; - -/* Old GNU extension names for DWARF5 debug_macro type encodings. - There are no equivalents for the supplementary object file (sup) - and indirect string references (strx). */ -#define DW_MACRO_GNU_define DW_MACRO_define -#define DW_MACRO_GNU_undef DW_MACRO_undef -#define DW_MACRO_GNU_start_file DW_MACRO_start_file -#define DW_MACRO_GNU_end_file DW_MACRO_end_file -#define DW_MACRO_GNU_define_indirect DW_MACRO_define_strp -#define DW_MACRO_GNU_undef_indirect DW_MACRO_undef_strp -#define DW_MACRO_GNU_transparent_include DW_MACRO_import -#define DW_MACRO_GNU_lo_user DW_MACRO_lo_user -#define DW_MACRO_GNU_hi_user DW_MACRO_hi_user - - -/* Range list entry encoding. */ -enum - { - DW_RLE_end_of_list = 0x0, - DW_RLE_base_addressx = 0x1, - DW_RLE_startx_endx = 0x2, - DW_RLE_startx_length = 0x3, - DW_RLE_offset_pair = 0x4, - DW_RLE_base_address = 0x5, - DW_RLE_start_end = 0x6, - DW_RLE_start_length = 0x7 - }; - - -/* Location list entry encoding. */ -enum - { - DW_LLE_end_of_list = 0x0, - DW_LLE_base_addressx = 0x1, - DW_LLE_startx_endx = 0x2, - DW_LLE_startx_length = 0x3, - DW_LLE_offset_pair = 0x4, - DW_LLE_default_location = 0x5, - DW_LLE_base_address = 0x6, - DW_LLE_start_end = 0x7, - DW_LLE_start_length = 0x8 - }; - - -/* GNU DebugFission list entry encodings (.debug_loc.dwo). */ -enum - { - DW_LLE_GNU_end_of_list_entry = 0x0, - DW_LLE_GNU_base_address_selection_entry = 0x1, - DW_LLE_GNU_start_end_entry = 0x2, - DW_LLE_GNU_start_length_entry = 0x3 - }; - -/* DWARF5 package file section identifiers. */ -enum - { - DW_SECT_INFO = 1, - /* Reserved = 2, */ - DW_SECT_ABBREV = 3, - DW_SECT_LINE = 4, - DW_SECT_LOCLISTS = 5, - DW_SECT_STR_OFFSETS = 6, - DW_SECT_MACRO = 7, - DW_SECT_RNGLISTS = 8, - }; - - -/* DWARF call frame instruction encodings. */ -enum - { - DW_CFA_advance_loc = 0x40, - DW_CFA_offset = 0x80, - DW_CFA_restore = 0xc0, - DW_CFA_extended = 0, - - DW_CFA_nop = 0x00, - DW_CFA_set_loc = 0x01, - DW_CFA_advance_loc1 = 0x02, - DW_CFA_advance_loc2 = 0x03, - DW_CFA_advance_loc4 = 0x04, - DW_CFA_offset_extended = 0x05, - DW_CFA_restore_extended = 0x06, - DW_CFA_undefined = 0x07, - DW_CFA_same_value = 0x08, - DW_CFA_register = 0x09, - DW_CFA_remember_state = 0x0a, - DW_CFA_restore_state = 0x0b, - DW_CFA_def_cfa = 0x0c, - DW_CFA_def_cfa_register = 0x0d, - DW_CFA_def_cfa_offset = 0x0e, - DW_CFA_def_cfa_expression = 0x0f, - DW_CFA_expression = 0x10, - DW_CFA_offset_extended_sf = 0x11, - DW_CFA_def_cfa_sf = 0x12, - DW_CFA_def_cfa_offset_sf = 0x13, - DW_CFA_val_offset = 0x14, - DW_CFA_val_offset_sf = 0x15, - DW_CFA_val_expression = 0x16, - - DW_CFA_low_user = 0x1c, - DW_CFA_MIPS_advance_loc8 = 0x1d, - DW_CFA_GNU_window_save = 0x2d, - DW_CFA_AARCH64_negate_ra_state = 0x2d, - DW_CFA_GNU_args_size = 0x2e, - DW_CFA_GNU_negative_offset_extended = 0x2f, - DW_CFA_high_user = 0x3f - }; - -/* ID indicating CIE as opposed to FDE in .debug_frame. */ -enum - { - DW_CIE_ID_32 = 0xffffffffU, /* In 32-bit format CIE header. */ - DW_CIE_ID_64 = 0xffffffffffffffffULL /* In 64-bit format CIE header. */ - }; - - -/* Information for GNU unwind information. */ -enum - { - DW_EH_PE_absptr = 0x00, - DW_EH_PE_omit = 0xff, - - /* FDE data encoding. */ - DW_EH_PE_uleb128 = 0x01, - DW_EH_PE_udata2 = 0x02, - DW_EH_PE_udata4 = 0x03, - DW_EH_PE_udata8 = 0x04, - DW_EH_PE_sleb128 = 0x09, - DW_EH_PE_sdata2 = 0x0a, - DW_EH_PE_sdata4 = 0x0b, - DW_EH_PE_sdata8 = 0x0c, - DW_EH_PE_signed = 0x08, - - /* FDE flags. */ - DW_EH_PE_pcrel = 0x10, - DW_EH_PE_textrel = 0x20, - DW_EH_PE_datarel = 0x30, - DW_EH_PE_funcrel = 0x40, - DW_EH_PE_aligned = 0x50, - - DW_EH_PE_indirect = 0x80 - }; - - -/* DWARF XXX. */ -#define DW_ADDR_none 0 - -/* Section 7.2.2 of the DWARF3 specification defines a range of escape - codes that can appear in the length field of certain DWARF structures. - - These defines enumerate the minimum and maximum values of this range. - Currently only the maximum value is used (to indicate that 64-bit - values are going to be used in the dwarf data that accompanies the - structure). The other values are reserved. - - Note: There is a typo in DWARF3 spec (published Dec 20, 2005). In - sections 7.4, 7.5.1, 7.19, 7.20 the minimum escape code is referred to - as 0xffffff00 whereas in fact it should be 0xfffffff0. */ -#define DWARF3_LENGTH_MIN_ESCAPE_CODE 0xfffffff0u -#define DWARF3_LENGTH_MAX_ESCAPE_CODE 0xffffffffu -#define DWARF3_LENGTH_64_BIT DWARF3_LENGTH_MAX_ESCAPE_CODE - -#endif /* dwarf.h */ diff --git a/vendor/tcc/i386-asm.c b/vendor/tcc/i386-asm.c deleted file mode 100644 index 3cc8d18b..00000000 --- a/vendor/tcc/i386-asm.c +++ /dev/null @@ -1,1744 +0,0 @@ -/* - * i386 specific functions for TCC assembler - * - * Copyright (c) 2001, 2002 Fabrice Bellard - * Copyright (c) 2009 Frédéric Feret (x86_64 support) - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 2 of the License, or (at your option) any later version. - * - * This library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public - * License along with this library; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA - */ - -#define USING_GLOBALS -#include "tcc.h" - -#define MAX_OPERANDS 3 - -#define TOK_ASM_first TOK_ASM_clc -#define TOK_ASM_last TOK_ASM_emms -#define TOK_ASM_alllast TOK_ASM_subps - -#define OPC_B 0x01 /* only used with OPC_WL */ -#define OPC_WL 0x02 /* accepts w, l or no suffix */ -#define OPC_BWL (OPC_B | OPC_WL) /* accepts b, w, l or no suffix */ -#define OPC_REG 0x04 /* register is added to opcode */ -#define OPC_MODRM 0x08 /* modrm encoding */ - -#define OPCT_MASK 0x70 -#define OPC_FWAIT 0x10 /* add fwait opcode */ -#define OPC_SHIFT 0x20 /* shift opcodes */ -#define OPC_ARITH 0x30 /* arithmetic opcodes */ -#define OPC_FARITH 0x40 /* FPU arithmetic opcodes */ -#define OPC_TEST 0x50 /* test opcodes */ -#define OPC_0F01 0x60 /* 0x0f01XX (group 7, XX is 2nd opcode, - no operands and unstructured mod/rm) */ -#define OPCT_IS(v,i) (((v) & OPCT_MASK) == (i)) - -#define OPC_0F 0x100 /* Is secondary map (0x0f prefix) */ -#define OPC_48 0x200 /* Always has REX prefix */ -#ifdef TCC_TARGET_X86_64 -# define OPC_WLQ 0x1000 /* accepts w, l, q or no suffix */ -# define OPC_BWLQ (OPC_B | OPC_WLQ) /* accepts b, w, l, q or no suffix */ -# define OPC_WLX OPC_WLQ -# define OPC_BWLX OPC_BWLQ -#else -# define OPC_WLX OPC_WL -# define OPC_BWLX OPC_BWL -#endif - -#define OPC_GROUP_SHIFT 13 - -/* in order to compress the operand type, we use specific operands and - we or only with EA */ -enum { - OPT_REG8=0, /* warning: value is hardcoded from TOK_ASM_xxx */ - OPT_REG16, /* warning: value is hardcoded from TOK_ASM_xxx */ - OPT_REG32, /* warning: value is hardcoded from TOK_ASM_xxx */ -#ifdef TCC_TARGET_X86_64 - OPT_REG64, /* warning: value is hardcoded from TOK_ASM_xxx */ -#endif - OPT_MMX, /* warning: value is hardcoded from TOK_ASM_xxx */ - OPT_SSE, /* warning: value is hardcoded from TOK_ASM_xxx */ - OPT_CR, /* warning: value is hardcoded from TOK_ASM_xxx */ - OPT_TR, /* warning: value is hardcoded from TOK_ASM_xxx */ - OPT_DB, /* warning: value is hardcoded from TOK_ASM_xxx */ - OPT_SEG, - OPT_ST, -#ifdef TCC_TARGET_X86_64 - OPT_REG8_LOW, /* %spl,%bpl,%sil,%dil, encoded like ah,ch,dh,bh, but - with REX prefix, not used in insn templates */ -#endif - OPT_IM8, - OPT_IM8S, - OPT_IM16, - OPT_IM32, -#ifdef TCC_TARGET_X86_64 - OPT_IM64, -#endif - OPT_EAX, /* %al, %ax, %eax or %rax register */ - OPT_ST0, /* %st(0) register */ - OPT_CL, /* %cl register */ - OPT_DX, /* %dx register */ - OPT_ADDR, /* OP_EA with only offset */ - OPT_INDIR, /* *(expr) */ - /* composite types */ - OPT_COMPOSITE_FIRST, - OPT_IM, /* IM8 | IM16 | IM32 */ - OPT_REG, /* REG8 | REG16 | REG32 | REG64 */ - OPT_REGW, /* REG16 | REG32 | REG64 */ - OPT_IMW, /* IM16 | IM32 */ - OPT_MMXSSE, /* MMX | SSE */ - OPT_DISP, /* Like OPT_ADDR, but emitted as displacement (for jumps) */ - OPT_DISP8, /* Like OPT_ADDR, but only 8bit (short jumps) */ - /* can be ored with any OPT_xxx */ - OPT_EA = 0x80 -}; - -#define OP_REG8 (1 << OPT_REG8) -#define OP_REG16 (1 << OPT_REG16) -#define OP_REG32 (1 << OPT_REG32) -#define OP_MMX (1 << OPT_MMX) -#define OP_SSE (1 << OPT_SSE) -#define OP_CR (1 << OPT_CR) -#define OP_TR (1 << OPT_TR) -#define OP_DB (1 << OPT_DB) -#define OP_SEG (1 << OPT_SEG) -#define OP_ST (1 << OPT_ST) -#define OP_IM8 (1 << OPT_IM8) -#define OP_IM8S (1 << OPT_IM8S) -#define OP_IM16 (1 << OPT_IM16) -#define OP_IM32 (1 << OPT_IM32) -#define OP_EAX (1 << OPT_EAX) -#define OP_ST0 (1 << OPT_ST0) -#define OP_CL (1 << OPT_CL) -#define OP_DX (1 << OPT_DX) -#define OP_ADDR (1 << OPT_ADDR) -#define OP_INDIR (1 << OPT_INDIR) -#ifdef TCC_TARGET_X86_64 -# define OP_REG64 (1 << OPT_REG64) -# define OP_REG8_LOW (1 << OPT_REG8_LOW) -# define OP_IM64 (1 << OPT_IM64) -# define OP_EA32 (OP_EA << 1) -#else -# define OP_REG64 0 -# define OP_REG8_LOW 0 -# define OP_IM64 0 -# define OP_EA32 0 -#endif - -#define OP_EA 0x40000000 -#define OP_REG (OP_REG8 | OP_REG16 | OP_REG32 | OP_REG64) - -#ifdef TCC_TARGET_X86_64 -# define TREG_XAX TREG_RAX -# define TREG_XCX TREG_RCX -# define TREG_XDX TREG_RDX -#else -# define TREG_XAX TREG_EAX -# define TREG_XCX TREG_ECX -# define TREG_XDX TREG_EDX -#endif - -typedef struct ASMInstr { - uint16_t sym; - uint16_t opcode; - uint16_t instr_type; - uint8_t nb_ops; - uint8_t op_type[MAX_OPERANDS]; /* see OP_xxx */ -} ASMInstr; - -typedef struct Operand { - uint32_t type; - int8_t reg; /* register, -1 if none */ - int8_t reg2; /* second register, -1 if none */ - uint8_t shift; - ExprValue e; -} Operand; - -static const uint8_t reg_to_size[9] = { -/* - [OP_REG8] = 0, - [OP_REG16] = 1, - [OP_REG32] = 2, -#ifdef TCC_TARGET_X86_64 - [OP_REG64] = 3, -#endif -*/ - 0, 0, 1, 0, 2, 0, 0, 0, 3 -}; - -#define NB_TEST_OPCODES 30 - -static const uint8_t test_bits[NB_TEST_OPCODES] = { - 0x00, /* o */ - 0x01, /* no */ - 0x02, /* b */ - 0x02, /* c */ - 0x02, /* nae */ - 0x03, /* nb */ - 0x03, /* nc */ - 0x03, /* ae */ - 0x04, /* e */ - 0x04, /* z */ - 0x05, /* ne */ - 0x05, /* nz */ - 0x06, /* be */ - 0x06, /* na */ - 0x07, /* nbe */ - 0x07, /* a */ - 0x08, /* s */ - 0x09, /* ns */ - 0x0a, /* p */ - 0x0a, /* pe */ - 0x0b, /* np */ - 0x0b, /* po */ - 0x0c, /* l */ - 0x0c, /* nge */ - 0x0d, /* nl */ - 0x0d, /* ge */ - 0x0e, /* le */ - 0x0e, /* ng */ - 0x0f, /* nle */ - 0x0f, /* g */ -}; - -static const uint8_t segment_prefixes[] = { - 0x26, /* es */ - 0x2e, /* cs */ - 0x36, /* ss */ - 0x3e, /* ds */ - 0x64, /* fs */ - 0x65 /* gs */ -}; - -static const ASMInstr asm_instrs[] = { -#define ALT(x) x -/* This removes a 0x0f in the second byte */ -#define O(o) ((uint64_t) ((((o) & 0xff00) == 0x0f00) ? ((((o) >> 8) & ~0xff) | ((o) & 0xff)) : (o))) -/* This constructs instr_type from opcode, type and group. */ -#define T(o,i,g) ((i) | ((g) << OPC_GROUP_SHIFT) | ((((o) & 0xff00) == 0x0f00) ? OPC_0F : 0)) -#define DEF_ASM_OP0(name, opcode) -#define DEF_ASM_OP0L(name, opcode, group, instr_type) { TOK_ASM_ ## name, O(opcode), T(opcode, instr_type, group), 0, { 0 } }, -#define DEF_ASM_OP1(name, opcode, group, instr_type, op0) { TOK_ASM_ ## name, O(opcode), T(opcode, instr_type, group), 1, { op0 }}, -#define DEF_ASM_OP2(name, opcode, group, instr_type, op0, op1) { TOK_ASM_ ## name, O(opcode), T(opcode, instr_type, group), 2, { op0, op1 }}, -#define DEF_ASM_OP3(name, opcode, group, instr_type, op0, op1, op2) { TOK_ASM_ ## name, O(opcode), T(opcode, instr_type, group), 3, { op0, op1, op2 }}, -#ifdef TCC_TARGET_X86_64 -# include "x86_64-asm.h" -#else -# include "i386-asm.h" -#endif - /* last operation */ - { 0, }, -}; - -static const uint16_t op0_codes[] = { -#define ALT(x) -#define DEF_ASM_OP0(x, opcode) opcode, -#define DEF_ASM_OP0L(name, opcode, group, instr_type) -#define DEF_ASM_OP1(name, opcode, group, instr_type, op0) -#define DEF_ASM_OP2(name, opcode, group, instr_type, op0, op1) -#define DEF_ASM_OP3(name, opcode, group, instr_type, op0, op1, op2) -#ifdef TCC_TARGET_X86_64 -# include "x86_64-asm.h" -#else -# include "i386-asm.h" -#endif -}; - -static inline int get_reg_shift(TCCState *s1) -{ - int shift, v; - v = asm_int_expr(s1); - switch(v) { - case 1: - shift = 0; - break; - case 2: - shift = 1; - break; - case 4: - shift = 2; - break; - case 8: - shift = 3; - break; - default: - expect("1, 2, 4 or 8 constant"); - shift = 0; - break; - } - return shift; -} - -#ifdef TCC_TARGET_X86_64 -static int asm_parse_numeric_reg(int t, unsigned int *type) -{ - int reg = -1; - if (t >= TOK_IDENT && t < tok_ident) { - const char *s = table_ident[t - TOK_IDENT]->str; - char c; - *type = OP_REG64; - if (*s == 'c') { - s++; - *type = OP_CR; - } - if (*s++ != 'r') - return -1; - /* Don't allow leading '0'. */ - if ((c = *s++) >= '1' && c <= '9') - reg = c - '0'; - else - return -1; - if ((c = *s) >= '0' && c <= '5') - s++, reg = reg * 10 + c - '0'; - if (reg > 15) - return -1; - if ((c = *s) == 0) - ; - else if (*type != OP_REG64) - return -1; - else if (c == 'b' && !s[1]) - *type = OP_REG8; - else if (c == 'w' && !s[1]) - *type = OP_REG16; - else if (c == 'd' && !s[1]) - *type = OP_REG32; - else - return -1; - } - return reg; -} -#endif - -static int asm_parse_reg(unsigned int *type) -{ - int reg = 0; - *type = 0; - if (tok != '%') - goto error_32; - next(); - if (tok >= TOK_ASM_eax && tok <= TOK_ASM_edi) { - reg = tok - TOK_ASM_eax; - *type = OP_REG32; -#ifdef TCC_TARGET_X86_64 - } else if (tok >= TOK_ASM_rax && tok <= TOK_ASM_rdi) { - reg = tok - TOK_ASM_rax; - *type = OP_REG64; - } else if (tok == TOK_ASM_rip) { - reg = -2; /* Probably should use different escape code. */ - *type = OP_REG64; - } else if ((reg = asm_parse_numeric_reg(tok, type)) >= 0 - && (*type == OP_REG32 || *type == OP_REG64)) { - ; -#endif - } else { - error_32: - expect("register"); - } - next(); - return reg; -} - -static void parse_operand(TCCState *s1, Operand *op) -{ - ExprValue e; - int reg, indir; - const char *p; - - indir = 0; - if (tok == '*') { - next(); - indir = OP_INDIR; - } - - if (tok == '%') { - next(); - if (tok >= TOK_ASM_al && tok <= TOK_ASM_db7) { - reg = tok - TOK_ASM_al; - op->type = 1 << (reg >> 3); /* WARNING: do not change constant order */ - op->reg = reg & 7; - if ((op->type & OP_REG) && op->reg == TREG_XAX) - op->type |= OP_EAX; - else if (op->type == OP_REG8 && op->reg == TREG_XCX) - op->type |= OP_CL; - else if (op->type == OP_REG16 && op->reg == TREG_XDX) - op->type |= OP_DX; - } else if (tok >= TOK_ASM_dr0 && tok <= TOK_ASM_dr7) { - op->type = OP_DB; - op->reg = tok - TOK_ASM_dr0; - } else if (tok >= TOK_ASM_es && tok <= TOK_ASM_gs) { - op->type = OP_SEG; - op->reg = tok - TOK_ASM_es; - } else if (tok == TOK_ASM_st) { - op->type = OP_ST; - op->reg = 0; - next(); - if (tok == '(') { - next(); - if (tok != TOK_PPNUM) - goto reg_error; - p = tokc.str.data; - reg = p[0] - '0'; - if ((unsigned)reg >= 8 || p[1] != '\0') - goto reg_error; - op->reg = reg; - next(); - skip(')'); - } - if (op->reg == 0) - op->type |= OP_ST0; - goto no_skip; -#ifdef TCC_TARGET_X86_64 - } else if (tok >= TOK_ASM_spl && tok <= TOK_ASM_dil) { - op->type = OP_REG8 | OP_REG8_LOW; - op->reg = 4 + tok - TOK_ASM_spl; - } else if ((op->reg = asm_parse_numeric_reg(tok, &op->type)) >= 0) { - ; -#endif - } else { - reg_error: - tcc_error("unknown register %%%s", get_tok_str(tok, &tokc)); - } - next(); - no_skip: ; - } else if (tok == '$') { - /* constant value */ - next(); - asm_expr(s1, &e); - op->type = OP_IM32; - op->e = e; - if (!op->e.sym) { - if (op->e.v == (uint8_t)op->e.v) - op->type |= OP_IM8; - if (op->e.v == (int8_t)op->e.v) - op->type |= OP_IM8S; - if (op->e.v == (uint16_t)op->e.v) - op->type |= OP_IM16; -#ifdef TCC_TARGET_X86_64 - if (op->e.v != (int32_t)op->e.v && op->e.v != (uint32_t)op->e.v) - op->type = OP_IM64; -#endif - } - } else { - /* address(reg,reg2,shift) with all variants */ - op->type = OP_EA; - op->reg = -1; - op->reg2 = -1; - op->shift = 0; - if (tok != '(') { - asm_expr(s1, &e); - op->e = e; - } else { - next(); - if (tok == '%') { - unget_tok('('); - op->e.v = 0; - op->e.sym = NULL; - } else { - /* bracketed offset expression */ - asm_expr(s1, &e); - if (tok != ')') - expect(")"); - next(); - op->e.v = e.v; - op->e.sym = e.sym; - } - op->e.pcrel = 0; - } - if (tok == '(') { - unsigned int type = 0; - next(); - if (tok != ',') { - op->reg = asm_parse_reg(&type); - } - if (tok == ',') { - next(); - if (tok != ',') { - op->reg2 = asm_parse_reg(&type); - } - if (tok == ',') { - next(); - op->shift = get_reg_shift(s1); - } - } - if (type & OP_REG32) - op->type |= OP_EA32; - skip(')'); - } - if (op->reg == -1 && op->reg2 == -1) - op->type |= OP_ADDR; - } - op->type |= indir; -} - -/* XXX: unify with C code output ? */ -ST_FUNC void gen_expr32(ExprValue *pe) -{ - if (pe->pcrel) - /* If PC-relative, always set VT_SYM, even without symbol, - so as to force a relocation to be emitted. */ - gen_addrpc32(VT_SYM, pe->sym, pe->v); - else - gen_addr32(pe->sym ? VT_SYM : 0, pe->sym, pe->v); -} - -#ifdef TCC_TARGET_X86_64 -ST_FUNC void gen_expr64(ExprValue *pe) -{ - gen_addr64(pe->sym ? VT_SYM : 0, pe->sym, pe->v); -} -#endif - -/* XXX: unify with C code output ? */ -static void gen_disp32(ExprValue *pe) -{ - Sym *sym = pe->sym; - ElfSym *esym = elfsym(sym); - if (esym && esym->st_shndx == cur_text_section->sh_num) { - /* same section: we can output an absolute value. Note - that the TCC compiler behaves differently here because - it always outputs a relocation to ease (future) code - elimination in the linker */ - gen_le32(pe->v + esym->st_value - ind - 4); - } else { - if (sym && sym->type.t == VT_VOID) { - sym->type.t = VT_FUNC; - sym->type.ref = NULL; - } - gen_addrpc32(VT_SYM, sym, pe->v); - } -} - -/* generate the modrm operand */ -static inline int asm_modrm(int reg, Operand *op) -{ - int mod, reg1, reg2, sib_reg1; - - if (op->type & (OP_REG | OP_MMX | OP_SSE)) { - g(0xc0 + (reg << 3) + op->reg); - } else if (op->reg == -1 && op->reg2 == -1) { - /* displacement only */ -#ifdef TCC_TARGET_X86_64 - g(0x04 + (reg << 3)); - g(0x25); -#else - g(0x05 + (reg << 3)); -#endif - gen_expr32(&op->e); -#ifdef TCC_TARGET_X86_64 - } else if (op->reg == -2) { - ExprValue *pe = &op->e; - g(0x05 + (reg << 3)); - gen_addrpc32(pe->sym ? VT_SYM : 0, pe->sym, pe->v); - return ind; -#endif - } else { - sib_reg1 = op->reg; - /* fist compute displacement encoding */ - if (sib_reg1 == -1) { - sib_reg1 = 5; - mod = 0x00; - } else if (op->e.v == 0 && !op->e.sym && op->reg != 5) { - mod = 0x00; - } else if (op->e.v == (int8_t)op->e.v && !op->e.sym) { - mod = 0x40; - } else { - mod = 0x80; - } - /* compute if sib byte needed */ - reg1 = op->reg; - if (op->reg2 != -1) - reg1 = 4; - g(mod + (reg << 3) + reg1); - if (reg1 == 4) { - /* add sib byte */ - reg2 = op->reg2; - if (reg2 == -1) - reg2 = 4; /* indicate no index */ - g((op->shift << 6) + (reg2 << 3) + sib_reg1); - } - /* add offset */ - if (mod == 0x40) { - g(op->e.v); - } else if (mod == 0x80 || op->reg == -1) { - gen_expr32(&op->e); - } - } - return 0; -} - -#ifdef TCC_TARGET_X86_64 -#define REX_W 0x48 -#define REX_R 0x44 -#define REX_X 0x42 -#define REX_B 0x41 - -static void asm_rex(int width64, Operand *ops, int nb_ops, int *op_type, - int regi, int rmi) -{ - unsigned char rex = width64 ? 0x48 : 0; - int saw_high_8bit = 0; - int i; - if (rmi == -1) { - /* No mod/rm byte, but we might have a register op nevertheless - (we will add it to the opcode later). */ - for(i = 0; i < nb_ops; i++) { - if (op_type[i] & (OP_REG | OP_ST)) { - if (ops[i].reg >= 8) { - rex |= REX_B; - ops[i].reg -= 8; - } else if (ops[i].type & OP_REG8_LOW) - rex |= 0x40; - else if (ops[i].type & OP_REG8 && ops[i].reg >= 4) - /* An 8 bit reg >= 4 without REG8 is ah/ch/dh/bh */ - saw_high_8bit = ops[i].reg; - break; - } - } - } else { - if (regi != -1) { - if (ops[regi].reg >= 8) { - rex |= REX_R; - ops[regi].reg -= 8; - } else if (ops[regi].type & OP_REG8_LOW) - rex |= 0x40; - else if (ops[regi].type & OP_REG8 && ops[regi].reg >= 4) - /* An 8 bit reg >= 4 without REG8 is ah/ch/dh/bh */ - saw_high_8bit = ops[regi].reg; - } - if (ops[rmi].type & (OP_REG | OP_MMX | OP_SSE | OP_CR | OP_EA)) { - if (ops[rmi].reg >= 8) { - rex |= REX_B; - ops[rmi].reg -= 8; - } else if (ops[rmi].type & OP_REG8_LOW) - rex |= 0x40; - else if (ops[rmi].type & OP_REG8 && ops[rmi].reg >= 4) - /* An 8 bit reg >= 4 without REG8 is ah/ch/dh/bh */ - saw_high_8bit = ops[rmi].reg; - } - if (ops[rmi].type & OP_EA && ops[rmi].reg2 >= 8) { - rex |= REX_X; - ops[rmi].reg2 -= 8; - } - } - if (rex) { - if (saw_high_8bit) - tcc_error("can't encode register %%%ch when REX prefix is required", - "acdb"[saw_high_8bit-4]); - g(rex); - } -} -#endif - - -static void maybe_print_stats (void) -{ - static int already; - - if (0 && !already) - /* print stats about opcodes */ - { - const struct ASMInstr *pa; - int freq[4]; - int op_vals[500]; - int nb_op_vals, i, j; - - already = 1; - nb_op_vals = 0; - memset(freq, 0, sizeof(freq)); - for(pa = asm_instrs; pa->sym != 0; pa++) { - freq[pa->nb_ops]++; - //for(i=0;inb_ops;i++) { - for(j=0;jop_type[i] == op_vals[j]) - if (pa->instr_type == op_vals[j]) - goto found; - } - //op_vals[nb_op_vals++] = pa->op_type[i]; - op_vals[nb_op_vals++] = pa->instr_type; - found: ; - //} - } - for(i=0;i= TOK_ASM_wait && opcode <= TOK_ASM_repnz) - unget_tok(';'); - - /* get operands */ - pop = ops; - nb_ops = 0; - seg_prefix = 0; - alltypes = 0; - for(;;) { - if (tok == ';' || tok == TOK_LINEFEED) - break; - if (nb_ops >= MAX_OPERANDS) { - tcc_error("incorrect number of operands"); - } - parse_operand(s1, pop); - if (tok == ':') { - if (pop->type != OP_SEG || seg_prefix) - tcc_error("incorrect prefix"); - seg_prefix = segment_prefixes[pop->reg]; - next(); - parse_operand(s1, pop); - if (!(pop->type & OP_EA)) { - tcc_error("segment prefix must be followed by memory reference"); - } - } - pop++; - nb_ops++; - if (tok != ',') - break; - next(); - } - - s = 0; /* avoid warning */ - -again: - /* optimize matching by using a lookup table (no hashing is needed - !) */ - for(pa = asm_instrs; pa->sym != 0; pa++) { - int it = pa->instr_type & OPCT_MASK; - s = 0; - if (it == OPC_FARITH) { - v = opcode - pa->sym; - if (!((unsigned)v < 8 * 6 && (v % 6) == 0)) - continue; - } else if (it == OPC_ARITH) { - if (!(opcode >= pa->sym && opcode < pa->sym + 8*NBWLX)) - continue; - s = (opcode - pa->sym) % NBWLX; - if ((pa->instr_type & OPC_BWLX) == OPC_WLX) - { - /* We need to reject the xxxb opcodes that we accepted above. - Note that pa->sym for WLX opcodes is the 'w' token, - to get the 'b' token subtract one. */ - if (((opcode - pa->sym + 1) % NBWLX) == 0) - continue; - s++; - } - } else if (it == OPC_SHIFT) { - if (!(opcode >= pa->sym && opcode < pa->sym + 7*NBWLX)) - continue; - s = (opcode - pa->sym) % NBWLX; - } else if (it == OPC_TEST) { - if (!(opcode >= pa->sym && opcode < pa->sym + NB_TEST_OPCODES)) - continue; - /* cmovxx is a test opcode but accepts multiple sizes. - The suffixes aren't encoded in the table, instead we - simply force size autodetection always and deal with suffixed - variants below when we don't find e.g. "cmovzl". */ - if (pa->instr_type & OPC_WLX) - s = NBWLX - 1; - } else if (pa->instr_type & OPC_B) { -#ifdef TCC_TARGET_X86_64 - /* Some instructions don't have the full size but only - bwl form. insb e.g. */ - if ((pa->instr_type & OPC_WLQ) != OPC_WLQ - && !(opcode >= pa->sym && opcode < pa->sym + NBWLX-1)) - continue; -#endif - if (!(opcode >= pa->sym && opcode < pa->sym + NBWLX)) - continue; - s = opcode - pa->sym; - } else if (pa->instr_type & OPC_WLX) { - if (!(opcode >= pa->sym && opcode < pa->sym + NBWLX-1)) - continue; - s = opcode - pa->sym + 1; - } else { - if (pa->sym != opcode) - continue; - } - if (pa->nb_ops != nb_ops) - continue; -#ifdef TCC_TARGET_X86_64 - /* Special case for moves. Selecting the IM64->REG64 form - should only be done if we really have an >32bit imm64, and that - is hardcoded. Ignore it here. */ - if (pa->opcode == 0xb0 && ops[0].type != OP_IM64 - && (ops[1].type & OP_REG) == OP_REG64 - && !(pa->instr_type & OPC_0F)) - continue; -#endif - /* now decode and check each operand */ - alltypes = 0; - for(i = 0; i < nb_ops; i++) { - int op1, op2; - op1 = pa->op_type[i]; - op2 = op1 & 0x1f; - switch(op2) { - case OPT_IM: - v = OP_IM8 | OP_IM16 | OP_IM32; - break; - case OPT_REG: - v = OP_REG8 | OP_REG16 | OP_REG32 | OP_REG64; - break; - case OPT_REGW: - v = OP_REG16 | OP_REG32 | OP_REG64; - break; - case OPT_IMW: - v = OP_IM16 | OP_IM32; - break; - case OPT_MMXSSE: - v = OP_MMX | OP_SSE; - break; - case OPT_DISP: - case OPT_DISP8: - v = OP_ADDR; - break; - default: - v = 1 << op2; - break; - } - if (op1 & OPT_EA) - v |= OP_EA; - op_type[i] = v; - if ((ops[i].type & v) == 0) - goto next; - alltypes |= ops[i].type; - } - (void)alltypes; /* maybe unused */ - /* all is matching ! */ - break; - next: ; - } - if (pa->sym == 0) { - if (opcode >= TOK_ASM_first && opcode <= TOK_ASM_last) { - int b; - b = op0_codes[opcode - TOK_ASM_first]; - if (b & 0xff00) - g(b >> 8); - g(b); - return; - } else if (opcode <= TOK_ASM_alllast) { - tcc_error("bad operand with opcode '%s'", - get_tok_str(opcode, NULL)); - } else { - /* Special case for cmovcc, we accept size suffixes but ignore - them, but we don't want them to blow up our tables. */ - TokenSym *ts = table_ident[opcode - TOK_IDENT]; - if (ts->len >= 6 - && strchr("wlq", ts->str[ts->len-1]) - && !memcmp(ts->str, "cmov", 4)) { - opcode = tok_alloc(ts->str, ts->len-1)->tok; - goto again; - } - tcc_error("unknown opcode '%s'", ts->str); - } - } - /* if the size is unknown, then evaluate it (OPC_B or OPC_WL case) */ - autosize = NBWLX-1; -#ifdef TCC_TARGET_X86_64 - /* XXX the autosize should rather be zero, to not have to adjust this - all the time. */ - if ((pa->instr_type & OPC_BWLQ) == OPC_B) - autosize = NBWLX-2; -#endif - if (s == autosize) { - /* Check for register operands providing hints about the size. - Start from the end, i.e. destination operands. This matters - only for opcodes accepting different sized registers, lar and lsl - are such opcodes. */ - for(i = nb_ops - 1; s == autosize && i >= 0; i--) { - if ((ops[i].type & OP_REG) && !(op_type[i] & (OP_CL | OP_DX))) - s = reg_to_size[ops[i].type & OP_REG]; - } - if (s == autosize) { - if ((opcode == TOK_ASM_push || opcode == TOK_ASM_pop) && - (ops[0].type & (OP_SEG | OP_IM8S | OP_IM32))) - s = 2; - else if ((opcode == TOK_ASM_push || opcode == TOK_ASM_pop) && - (ops[0].type & OP_EA)) - s = NBWLX - 2; - else - tcc_error("cannot infer opcode suffix"); - } - } - -#ifdef TCC_TARGET_X86_64 - rex64 = 0; - if (pa->instr_type & OPC_48) - rex64 = 1; - else if (s == 3 || (alltypes & OP_REG64)) { - /* generate REX prefix */ - int default64 = 0; - for(i = 0; i < nb_ops; i++) { - if (op_type[i] == OP_REG64 && pa->opcode != 0xb8) { - /* If only 64bit regs are accepted in one operand - this is a default64 instruction without need for - REX prefixes, except for movabs(0xb8). */ - default64 = 1; - break; - } - } - /* XXX find better encoding for the default64 instructions. */ - if (((opcode != TOK_ASM_push && opcode != TOK_ASM_pop - && opcode != TOK_ASM_pushw && opcode != TOK_ASM_pushl - && opcode != TOK_ASM_pushq && opcode != TOK_ASM_popw - && opcode != TOK_ASM_popl && opcode != TOK_ASM_popq - && opcode != TOK_ASM_call && opcode != TOK_ASM_jmp)) - && !default64) - rex64 = 1; - } -#endif - - /* now generates the operation */ - if (OPCT_IS(pa->instr_type, OPC_FWAIT)) - g(0x9b); - if (seg_prefix) - g(seg_prefix); -#ifdef TCC_TARGET_X86_64 - /* Generate addr32 prefix if needed */ - for(i = 0; i < nb_ops; i++) { - if (ops[i].type & OP_EA32) { - g(0x67); - break; - } - } -#endif - /* generate data16 prefix if needed */ - p66 = 0; - if (s == 1) - p66 = 1; - else { - /* accepting mmx+sse in all operands --> needs 0x66 to - switch to sse mode. Accepting only sse in an operand --> is - already SSE insn and needs 0x66/f2/f3 handling. */ - for (i = 0; i < nb_ops; i++) - if ((op_type[i] & (OP_MMX | OP_SSE)) == (OP_MMX | OP_SSE) - && ops[i].type & OP_SSE) - p66 = 1; - } - if (p66) - g(0x66); - - v = pa->opcode; - p = v >> 8; /* possibly prefix byte(s) */ - switch (p) { - case 0: break; /* no prefix */ - case 0x48: break; /* REX, handled elsewhere */ - case 0x66: - case 0x67: - case 0xf2: - case 0xf3: v = v & 0xff; g(p); break; - case 0xd4: case 0xd5: break; /* aam and aad, not prefix, but hardcoded immediate argument "10" */ - case 0xd8: case 0xd9: case 0xda: case 0xdb: /* x87, no normal prefix */ - case 0xdc: case 0xdd: case 0xde: case 0xdf: break; - default: tcc_error("bad prefix 0x%2x in opcode table", p); break; - } - if (pa->instr_type & OPC_0F) - v = ((v & ~0xff) << 8) | 0x0f00 | (v & 0xff); - if ((v == 0x69 || v == 0x6b) && nb_ops == 2) { - /* kludge for imul $im, %reg */ - nb_ops = 3; - ops[2] = ops[1]; - op_type[2] = op_type[1]; - } else if (v == 0xcd && ops[0].e.v == 3 && !ops[0].e.sym) { - v--; /* int $3 case */ - nb_ops = 0; - } else if ((v == 0x06 || v == 0x07)) { - if (ops[0].reg >= 4) { - /* push/pop %fs or %gs */ - v = 0x0fa0 + (v - 0x06) + ((ops[0].reg - 4) << 3); - } else { - v += ops[0].reg << 3; - } - nb_ops = 0; - } else if (v <= 0x05) { - /* arith case */ - v += ((opcode - TOK_ASM_addb) / NBWLX) << 3; - } else if ((pa->instr_type & (OPCT_MASK | OPC_MODRM)) == OPC_FARITH) { - /* fpu arith case */ - v += ((opcode - pa->sym) / 6) << 3; - } - - /* search which operand will be used for modrm */ - modrm_index = -1; - modreg_index = -1; - if (pa->instr_type & OPC_MODRM) { - if (!nb_ops) { - /* A modrm opcode without operands is a special case (e.g. mfence). - It has a group and acts as if there's an register operand 0 - (ax). */ - i = 0; - ops[i].type = OP_REG; - ops[i].reg = 0; - goto modrm_found; - } - /* first look for an ea operand */ - for(i = 0;i < nb_ops; i++) { - if (op_type[i] & OP_EA) - goto modrm_found; - } - /* then if not found, a register or indirection (shift instructions) */ - for(i = 0;i < nb_ops; i++) { - if (op_type[i] & (OP_REG | OP_MMX | OP_SSE | OP_INDIR)) - goto modrm_found; - } -#ifdef ASM_DEBUG - tcc_error("bad op table"); -#endif - modrm_found: - modrm_index = i; - /* if a register is used in another operand then it is - used instead of group */ - for(i = 0;i < nb_ops; i++) { - int t = op_type[i]; - if (i != modrm_index && - (t & (OP_REG | OP_MMX | OP_SSE | OP_CR | OP_TR | OP_DB | OP_SEG))) { - modreg_index = i; - break; - } - } - } -#ifdef TCC_TARGET_X86_64 - asm_rex (rex64, ops, nb_ops, op_type, modreg_index, modrm_index); -#endif - - if (pa->instr_type & OPC_REG) { - /* mov $im, %reg case */ - if (v == 0xb0 && s >= 1) - v += 7; - for(i = 0; i < nb_ops; i++) { - if (op_type[i] & (OP_REG | OP_ST)) { - v += ops[i].reg; - break; - } - } - } - if (pa->instr_type & OPC_B) - v += s >= 1; - if (nb_ops == 1 && pa->op_type[0] == OPT_DISP8) { - ElfSym *esym; - int jmp_disp; - - /* see if we can really generate the jump with a byte offset */ - esym = elfsym(ops[0].e.sym); - if (!esym || esym->st_shndx != cur_text_section->sh_num) - goto no_short_jump; - jmp_disp = ops[0].e.v + esym->st_value - ind - 2 - (v >= 0xff); - if (jmp_disp == (int8_t)jmp_disp) { - /* OK to generate jump */ - ops[0].e.sym = 0; - ops[0].e.v = jmp_disp; - op_type[0] = OP_IM8S; - } else { - no_short_jump: - /* long jump will be allowed. need to modify the - opcode slightly */ - if (v == 0xeb) /* jmp */ - v = 0xe9; - else if (v == 0x70) /* jcc */ - v += 0x0f10; - else - tcc_error("invalid displacement"); - } - } - if (OPCT_IS(pa->instr_type, OPC_TEST)) - v += test_bits[opcode - pa->sym]; - else if (OPCT_IS(pa->instr_type, OPC_0F01)) - v |= 0x0f0100; - op1 = v >> 16; - if (op1) - g(op1); - op1 = (v >> 8) & 0xff; - if (op1) - g(op1); - g(v); - - if (OPCT_IS(pa->instr_type, OPC_SHIFT)) { - reg = (opcode - pa->sym) / NBWLX; - if (reg == 6) - reg = 7; - } else if (OPCT_IS(pa->instr_type, OPC_ARITH)) { - reg = (opcode - pa->sym) / NBWLX; - } else if (OPCT_IS(pa->instr_type, OPC_FARITH)) { - reg = (opcode - pa->sym) / 6; - } else { - reg = (pa->instr_type >> OPC_GROUP_SHIFT) & 7; - } - - pc = 0; - if (pa->instr_type & OPC_MODRM) { - /* if a register is used in another operand then it is - used instead of group */ - if (modreg_index >= 0) - reg = ops[modreg_index].reg; - pc = asm_modrm(reg, &ops[modrm_index]); - } - - /* emit constants */ -#ifndef TCC_TARGET_X86_64 - if (!(pa->instr_type & OPC_0F) - && (pa->opcode == 0x9a || pa->opcode == 0xea)) { - /* ljmp or lcall kludge */ - gen_expr32(&ops[1].e); - if (ops[0].e.sym) - tcc_error("cannot relocate"); - gen_le16(ops[0].e.v); - return; - } -#endif - for(i = 0;i < nb_ops; i++) { - v = op_type[i]; - if (v & (OP_IM8 | OP_IM16 | OP_IM32 | OP_IM64 | OP_IM8S | OP_ADDR)) { - /* if multiple sizes are given it means we must look - at the op size */ - if ((v | OP_IM8 | OP_IM64) == (OP_IM8 | OP_IM16 | OP_IM32 | OP_IM64)) { - if (s == 0) - v = OP_IM8; - else if (s == 1) - v = OP_IM16; - else if (s == 2 || (v & OP_IM64) == 0) - v = OP_IM32; - else - v = OP_IM64; - } - - if ((v & (OP_IM8 | OP_IM8S | OP_IM16)) && ops[i].e.sym) - tcc_error("cannot relocate"); - - if (v & (OP_IM8 | OP_IM8S)) { - g(ops[i].e.v); - } else if (v & OP_IM16) { - gen_le16(ops[i].e.v); -#ifdef TCC_TARGET_X86_64 - } else if (v & OP_IM64) { - gen_expr64(&ops[i].e); -#endif - } else if (pa->op_type[i] == OPT_DISP || pa->op_type[i] == OPT_DISP8) { - gen_disp32(&ops[i].e); - } else { - gen_expr32(&ops[i].e); - } - } - } - - /* after immediate operands, adjust pc-relative address */ - if (pc) - add32le(cur_text_section->data + pc - 4, pc - ind); -} - -/* return the constraint priority (we allocate first the lowest - numbered constraints) */ -static inline int constraint_priority(const char *str) -{ - int priority, c, pr; - - /* we take the lowest priority */ - priority = 0; - for(;;) { - c = *str; - if (c == '\0') - break; - str++; - switch(c) { - case 'A': - pr = 0; - break; - case 'a': - case 'b': - case 'c': - case 'd': - case 'S': - case 'D': - pr = 1; - break; - case 'q': - pr = 2; - break; - case 'r': - case 'R': - case 'p': - pr = 3; - break; - case 'N': - case 'M': - case 'I': - case 'e': - case 'i': - case 'm': - case 'g': - pr = 4; - break; - default: - tcc_error("unknown constraint '%c'", c); - pr = 0; - } - if (pr > priority) - priority = pr; - } - return priority; -} - -static const char *skip_constraint_modifiers(const char *p) -{ - while (*p == '=' || *p == '&' || *p == '+' || *p == '%') - p++; - return p; -} - -/* If T (a token) is of the form "%reg" returns the register - number and type, otherwise return -1. */ -ST_FUNC int asm_parse_regvar (int t) -{ - const char *s; - Operand op; - if (t < TOK_IDENT || (t & SYM_FIELD)) - return -1; - s = table_ident[t - TOK_IDENT]->str; - if (s[0] != '%') - return -1; - t = tok_alloc_const(s + 1); - unget_tok(t); - unget_tok('%'); - parse_operand(tcc_state, &op); - /* Accept only integer regs for now. */ - if (op.type & OP_REG) - return op.reg; - else - return -1; -} - -#define REG_OUT_MASK 0x01 -#define REG_IN_MASK 0x02 - -#define is_reg_allocated(reg) (regs_allocated[reg] & reg_mask) - -ST_FUNC void asm_compute_constraints(ASMOperand *operands, - int nb_operands, int nb_outputs, - const uint8_t *clobber_regs, - int *pout_reg) -{ - ASMOperand *op; - int sorted_op[MAX_ASM_OPERANDS]; - int i, j, k, p1, p2, tmp, reg, c, reg_mask; - const char *str; - uint8_t regs_allocated[NB_ASM_REGS]; - - /* init fields */ - for(i=0;iinput_index = -1; - op->ref_index = -1; - op->reg = -1; - op->is_memory = 0; - op->is_rw = 0; - } - /* compute constraint priority and evaluate references to output - constraints if input constraints */ - for(i=0;iconstraint; - str = skip_constraint_modifiers(str); - if (isnum(*str) || *str == '[') { - /* this is a reference to another constraint */ - k = find_constraint(operands, nb_operands, str, NULL); - if ((unsigned)k >= i || i < nb_outputs) - tcc_error("invalid reference in constraint %d ('%s')", - i, str); - op->ref_index = k; - if (operands[k].input_index >= 0) - tcc_error("cannot reference twice the same operand"); - operands[k].input_index = i; - op->priority = 5; - } else if ((op->vt->r & VT_VALMASK) == VT_LOCAL - && op->vt->sym - && (reg = op->vt->sym->r & VT_VALMASK) < VT_CONST) { - op->priority = 1; - op->reg = reg; - } else { - op->priority = constraint_priority(str); - } - } - - /* sort operands according to their priority */ - for(i=0;iconstraint; - /* no need to allocate references */ - if (op->ref_index >= 0) - continue; - /* select if register is used for output, input or both */ - if (op->input_index >= 0) { - reg_mask = REG_IN_MASK | REG_OUT_MASK; - } else if (j < nb_outputs) { - reg_mask = REG_OUT_MASK; - } else { - reg_mask = REG_IN_MASK; - } - if (op->reg >= 0) { - if (is_reg_allocated(op->reg)) - tcc_error("asm regvar requests register that's taken already"); - reg = op->reg; - goto reg_found; - } - try_next: - c = *str++; - switch(c) { - case '=': - goto try_next; - case '+': - op->is_rw = 1; - /* FALL THRU */ - case '&': - if (j >= nb_outputs) - tcc_error("'%c' modifier can only be applied to outputs", c); - reg_mask = REG_IN_MASK | REG_OUT_MASK; - goto try_next; - case 'A': - /* allocate both eax and edx */ - if (is_reg_allocated(TREG_XAX) || - is_reg_allocated(TREG_XDX)) - goto try_next; - op->is_llong = 1; - op->reg = TREG_XAX; - regs_allocated[TREG_XAX] |= reg_mask; - regs_allocated[TREG_XDX] |= reg_mask; - break; - case 'a': - reg = TREG_XAX; - goto alloc_reg; - case 'b': - reg = 3; - goto alloc_reg; - case 'c': - reg = TREG_XCX; - goto alloc_reg; - case 'd': - reg = TREG_XDX; - goto alloc_reg; - case 'S': - reg = 6; - goto alloc_reg; - case 'D': - reg = 7; - alloc_reg: - if (is_reg_allocated(reg)) - goto try_next; - goto reg_found; - case 'q': - /* eax, ebx, ecx or edx */ - for(reg = 0; reg < 4; reg++) { - if (!is_reg_allocated(reg)) - goto reg_found; - } - goto try_next; - case 'r': - case 'R': - case 'p': /* A general address, for x86(64) any register is acceptable*/ - /* any general register */ - for(reg = 0; reg < 8; reg++) { - if (!is_reg_allocated(reg)) - goto reg_found; - } - goto try_next; - reg_found: - /* now we can reload in the register */ - op->is_llong = 0; - op->reg = reg; - regs_allocated[reg] |= reg_mask; - break; - case 'e': - case 'i': - if (!((op->vt->r & (VT_VALMASK | VT_LVAL)) == VT_CONST)) - goto try_next; - break; - case 'I': - case 'N': - case 'M': - if (!((op->vt->r & (VT_VALMASK | VT_LVAL | VT_SYM)) == VT_CONST)) - goto try_next; - break; - case 'm': - case 'g': - /* nothing special to do because the operand is already in - memory, except if the pointer itself is stored in a - memory variable (VT_LLOCAL case) */ - /* XXX: fix constant case */ - /* if it is a reference to a memory zone, it must lie - in a register, so we reserve the register in the - input registers and a load will be generated - later */ - if (j < nb_outputs || c == 'm') { - if ((op->vt->r & VT_VALMASK) == VT_LLOCAL) { - /* any general register */ - for(reg = 0; reg < 8; reg++) { - if (!(regs_allocated[reg] & REG_IN_MASK)) - goto reg_found1; - } - goto try_next; - reg_found1: - /* now we can reload in the register */ - regs_allocated[reg] |= REG_IN_MASK; - op->reg = reg; - op->is_memory = 1; - } - } - break; - default: - tcc_error("asm constraint %d ('%s') could not be satisfied", - j, op->constraint); - break; - } - /* if a reference is present for that operand, we assign it too */ - if (op->input_index >= 0) { - operands[op->input_index].reg = op->reg; - operands[op->input_index].is_llong = op->is_llong; - } - } - - /* compute out_reg. It is used to store outputs registers to memory - locations references by pointers (VT_LLOCAL case) */ - *pout_reg = -1; - for(i=0;ireg >= 0 && - (op->vt->r & VT_VALMASK) == VT_LLOCAL && - !op->is_memory) { - for(reg = 0; reg < 8; reg++) { - if (!(regs_allocated[reg] & REG_OUT_MASK)) - goto reg_found2; - } - tcc_error("could not find free output register for reloading"); - reg_found2: - *pout_reg = reg; - break; - } - } - - /* print sorted constraints */ -#ifdef ASM_DEBUG - for(i=0;iid ? get_tok_str(op->id, NULL) : "", - op->constraint, - op->vt->r, - op->reg); - } - if (*pout_reg >= 0) - printf("out_reg=%d\n", *pout_reg); -#endif -} - -ST_FUNC void subst_asm_operand(CString *add_str, - SValue *sv, int modifier) -{ - int r, reg, size, val; - char buf[64]; - - r = sv->r; - if ((r & VT_VALMASK) == VT_CONST) { - if (!(r & VT_LVAL) && modifier != 'c' && modifier != 'n' && - modifier != 'P') - cstr_ccat(add_str, '$'); - if (r & VT_SYM) { - const char *name = get_tok_str(sv->sym->v, NULL); - if (sv->sym->v >= SYM_FIRST_ANOM) { - /* In case of anonymous symbols ("L.42", used - for static data labels) we can't find them - in the C symbol table when later looking up - this name. So enter them now into the asm label - list when we still know the symbol. */ - get_asm_sym(tok_alloc_const(name), sv->sym); - } - if (tcc_state->leading_underscore) - cstr_ccat(add_str, '_'); - cstr_cat(add_str, name, -1); - if ((uint32_t)sv->c.i == 0) - goto no_offset; - cstr_ccat(add_str, '+'); - } - val = sv->c.i; - if (modifier == 'n') - val = -val; - snprintf(buf, sizeof(buf), "%d", (int)sv->c.i); - cstr_cat(add_str, buf, -1); - no_offset:; -#ifdef TCC_TARGET_X86_64 - if (r & VT_LVAL) - cstr_cat(add_str, "(%rip)", -1); -#endif - } else if ((r & VT_VALMASK) == VT_LOCAL) { -#ifdef TCC_TARGET_X86_64 - snprintf(buf, sizeof(buf), "%d(%%rbp)", (int)sv->c.i); -#else - snprintf(buf, sizeof(buf), "%d(%%ebp)", (int)sv->c.i); -#endif - cstr_cat(add_str, buf, -1); - } else if (r & VT_LVAL) { - reg = r & VT_VALMASK; - if (reg >= VT_CONST) - tcc_internal_error(""); - snprintf(buf, sizeof(buf), "(%%%s)", -#ifdef TCC_TARGET_X86_64 - get_tok_str(TOK_ASM_rax + reg, NULL) -#else - get_tok_str(TOK_ASM_eax + reg, NULL) -#endif - ); - cstr_cat(add_str, buf, -1); - } else { - /* register case */ - reg = r & VT_VALMASK; - if (reg >= VT_CONST) - tcc_internal_error(""); - - /* choose register operand size */ - if ((sv->type.t & VT_BTYPE) == VT_BYTE || - (sv->type.t & VT_BTYPE) == VT_BOOL) - size = 1; - else if ((sv->type.t & VT_BTYPE) == VT_SHORT) - size = 2; -#ifdef TCC_TARGET_X86_64 - else if ((sv->type.t & VT_BTYPE) == VT_LLONG || - (sv->type.t & VT_BTYPE) == VT_PTR) - size = 8; -#endif - else - size = 4; - if (size == 1 && reg >= 4) - size = 4; - - if (modifier == 'b') { - if (reg >= 4) - tcc_error("cannot use byte register"); - size = 1; - } else if (modifier == 'h') { - if (reg >= 4) - tcc_error("cannot use byte register"); - size = -1; - } else if (modifier == 'w') { - size = 2; - } else if (modifier == 'k') { - size = 4; -#ifdef TCC_TARGET_X86_64 - } else if (modifier == 'q') { - size = 8; -#endif - } - - switch(size) { - case -1: - reg = TOK_ASM_ah + reg; - break; - case 1: - reg = TOK_ASM_al + reg; - break; - case 2: - reg = TOK_ASM_ax + reg; - break; - default: - reg = TOK_ASM_eax + reg; - break; -#ifdef TCC_TARGET_X86_64 - case 8: - reg = TOK_ASM_rax + reg; - break; -#endif - } - snprintf(buf, sizeof(buf), "%%%s", get_tok_str(reg, NULL)); - cstr_cat(add_str, buf, -1); - } -} - -/* generate prolog and epilog code for asm statement */ -ST_FUNC void asm_gen_code(ASMOperand *operands, int nb_operands, - int nb_outputs, int is_output, - uint8_t *clobber_regs, - int out_reg) -{ - uint8_t regs_allocated[NB_ASM_REGS]; - ASMOperand *op; - int i, reg; - - /* Strictly speaking %Xbp and %Xsp should be included in the - call-preserved registers, but currently it doesn't matter. */ -#ifdef TCC_TARGET_X86_64 -#ifdef TCC_TARGET_PE - static const uint8_t reg_saved[] = { 3, 6, 7, 12, 13, 14, 15 }; -#else - static const uint8_t reg_saved[] = { 3, 12, 13, 14, 15 }; -#endif -#else - static const uint8_t reg_saved[] = { 3, 6, 7 }; -#endif - - /* mark all used registers */ - memcpy(regs_allocated, clobber_regs, sizeof(regs_allocated)); - for(i = 0; i < nb_operands;i++) { - op = &operands[i]; - if (op->reg >= 0) - regs_allocated[op->reg] = 1; - } - if (!is_output) { - /* generate reg save code */ - for(i = 0; i < sizeof(reg_saved)/sizeof(reg_saved[0]); i++) { - reg = reg_saved[i]; - if (regs_allocated[reg]) { - if (reg >= 8) - g(0x41), reg-=8; - g(0x50 + reg); - } - } - - /* generate load code */ - for(i = 0; i < nb_operands; i++) { - op = &operands[i]; - if (op->reg >= 0) { - if ((op->vt->r & VT_VALMASK) == VT_LLOCAL && - op->is_memory) { - /* memory reference case (for both input and - output cases) */ - SValue sv; - sv = *op->vt; - sv.r = (sv.r & ~VT_VALMASK) | VT_LOCAL | VT_LVAL; - sv.type.t = VT_PTR; - load(op->reg, &sv); - } else if (i >= nb_outputs || op->is_rw) { - /* load value in register */ - load(op->reg, op->vt); - if (op->is_llong) { - SValue sv; - sv = *op->vt; - sv.c.i += 4; - load(TREG_XDX, &sv); - } - } - } - } - } else { - /* generate save code */ - for(i = 0 ; i < nb_outputs; i++) { - op = &operands[i]; - if (op->reg >= 0) { - if ((op->vt->r & VT_VALMASK) == VT_LLOCAL) { - if (!op->is_memory) { - SValue sv; - sv = *op->vt; - sv.r = (sv.r & ~VT_VALMASK) | VT_LOCAL; - sv.type.t = VT_PTR; - load(out_reg, &sv); - - sv = *op->vt; - sv.r = (sv.r & ~VT_VALMASK) | out_reg; - store(op->reg, &sv); - } - } else { - store(op->reg, op->vt); - if (op->is_llong) { - SValue sv; - sv = *op->vt; - sv.c.i += 4; - store(TREG_XDX, &sv); - } - } - } - } - /* generate reg restore code */ - for(i = sizeof(reg_saved)/sizeof(reg_saved[0]) - 1; i >= 0; i--) { - reg = reg_saved[i]; - if (regs_allocated[reg]) { - if (reg >= 8) - g(0x41), reg-=8; - g(0x58 + reg); - } - } - } -} - -ST_FUNC void asm_clobber(uint8_t *clobber_regs, const char *str) -{ - int reg; -#ifdef TCC_TARGET_X86_64 - unsigned int type; -#endif - - if (!strcmp(str, "memory") || - !strcmp(str, "cc") || - !strcmp(str, "flags")) - return; - reg = tok_alloc_const(str); - if (reg >= TOK_ASM_eax && reg <= TOK_ASM_edi) { - reg -= TOK_ASM_eax; - } else if (reg >= TOK_ASM_ax && reg <= TOK_ASM_di) { - reg -= TOK_ASM_ax; -#ifdef TCC_TARGET_X86_64 - } else if (reg >= TOK_ASM_rax && reg <= TOK_ASM_rdi) { - reg -= TOK_ASM_rax; - } else if ((reg = asm_parse_numeric_reg(reg, &type)) >= 0) { - ; -#endif - } else { - tcc_error("invalid clobber register '%s'", str); - } - clobber_regs[reg] = 1; -} diff --git a/vendor/tcc/i386-asm.h b/vendor/tcc/i386-asm.h deleted file mode 100644 index 0f99b286..00000000 --- a/vendor/tcc/i386-asm.h +++ /dev/null @@ -1,487 +0,0 @@ - DEF_ASM_OP0(clc, 0xf8) /* must be first OP0 */ - DEF_ASM_OP0(cld, 0xfc) - DEF_ASM_OP0(cli, 0xfa) - DEF_ASM_OP0(clts, 0x0f06) - DEF_ASM_OP0(cmc, 0xf5) - DEF_ASM_OP0(lahf, 0x9f) - DEF_ASM_OP0(sahf, 0x9e) - DEF_ASM_OP0(pusha, 0x60) - DEF_ASM_OP0(popa, 0x61) - DEF_ASM_OP0(pushfl, 0x9c) - DEF_ASM_OP0(popfl, 0x9d) - DEF_ASM_OP0(pushf, 0x9c) - DEF_ASM_OP0(popf, 0x9d) - DEF_ASM_OP0(stc, 0xf9) - DEF_ASM_OP0(std, 0xfd) - DEF_ASM_OP0(sti, 0xfb) - DEF_ASM_OP0(aaa, 0x37) - DEF_ASM_OP0(aas, 0x3f) - DEF_ASM_OP0(daa, 0x27) - DEF_ASM_OP0(das, 0x2f) - DEF_ASM_OP0(aad, 0xd50a) - DEF_ASM_OP0(aam, 0xd40a) - DEF_ASM_OP0(cbw, 0x6698) - DEF_ASM_OP0(cwd, 0x6699) - DEF_ASM_OP0(cwde, 0x98) - DEF_ASM_OP0(cdq, 0x99) - DEF_ASM_OP0(cbtw, 0x6698) - DEF_ASM_OP0(cwtl, 0x98) - DEF_ASM_OP0(cwtd, 0x6699) - DEF_ASM_OP0(cltd, 0x99) - DEF_ASM_OP0(int3, 0xcc) - DEF_ASM_OP0(into, 0xce) - DEF_ASM_OP0(iret, 0xcf) - DEF_ASM_OP0(rsm, 0x0faa) - DEF_ASM_OP0(hlt, 0xf4) - DEF_ASM_OP0(nop, 0x90) - DEF_ASM_OP0(pause, 0xf390) - DEF_ASM_OP0(xlat, 0xd7) - - /* strings */ -ALT(DEF_ASM_OP0L(cmpsb, 0xa6, 0, OPC_BWLX)) -ALT(DEF_ASM_OP0L(scmpb, 0xa6, 0, OPC_BWLX)) - -ALT(DEF_ASM_OP0L(insb, 0x6c, 0, OPC_BWL)) -ALT(DEF_ASM_OP0L(outsb, 0x6e, 0, OPC_BWL)) - -ALT(DEF_ASM_OP0L(lodsb, 0xac, 0, OPC_BWLX)) -ALT(DEF_ASM_OP0L(slodb, 0xac, 0, OPC_BWLX)) - -ALT(DEF_ASM_OP0L(movsb, 0xa4, 0, OPC_BWLX)) -ALT(DEF_ASM_OP0L(smovb, 0xa4, 0, OPC_BWLX)) - -ALT(DEF_ASM_OP0L(scasb, 0xae, 0, OPC_BWLX)) -ALT(DEF_ASM_OP0L(sscab, 0xae, 0, OPC_BWLX)) - -ALT(DEF_ASM_OP0L(stosb, 0xaa, 0, OPC_BWLX)) -ALT(DEF_ASM_OP0L(sstob, 0xaa, 0, OPC_BWLX)) - - /* bits */ - -ALT(DEF_ASM_OP2(bsfw, 0x0fbc, 0, OPC_MODRM | OPC_WLX, OPT_REGW | OPT_EA, OPT_REGW)) -ALT(DEF_ASM_OP2(bsrw, 0x0fbd, 0, OPC_MODRM | OPC_WLX, OPT_REGW | OPT_EA, OPT_REGW)) - -ALT(DEF_ASM_OP2(btw, 0x0fa3, 0, OPC_MODRM | OPC_WLX, OPT_REGW, OPT_REGW | OPT_EA)) -ALT(DEF_ASM_OP2(btw, 0x0fba, 4, OPC_MODRM | OPC_WLX, OPT_IM8, OPT_REGW | OPT_EA)) - -ALT(DEF_ASM_OP2(btsw, 0x0fab, 0, OPC_MODRM | OPC_WLX, OPT_REGW, OPT_REGW | OPT_EA)) -ALT(DEF_ASM_OP2(btsw, 0x0fba, 5, OPC_MODRM | OPC_WLX, OPT_IM8, OPT_REGW | OPT_EA)) - -ALT(DEF_ASM_OP2(btrw, 0x0fb3, 0, OPC_MODRM | OPC_WLX, OPT_REGW, OPT_REGW | OPT_EA)) -ALT(DEF_ASM_OP2(btrw, 0x0fba, 6, OPC_MODRM | OPC_WLX, OPT_IM8, OPT_REGW | OPT_EA)) - -ALT(DEF_ASM_OP2(btcw, 0x0fbb, 0, OPC_MODRM | OPC_WLX, OPT_REGW, OPT_REGW | OPT_EA)) -ALT(DEF_ASM_OP2(btcw, 0x0fba, 7, OPC_MODRM | OPC_WLX, OPT_IM8, OPT_REGW | OPT_EA)) - -ALT(DEF_ASM_OP2(popcntw, 0xf30fb8, 0, OPC_MODRM | OPC_WLX, OPT_REGW | OPT_EA, OPT_REGW)) - -ALT(DEF_ASM_OP2(tzcntw, 0xf30fbc, 0, OPC_MODRM | OPC_WLX, OPT_REGW | OPT_EA, OPT_REGW)) -ALT(DEF_ASM_OP2(lzcntw, 0xf30fbd, 0, OPC_MODRM | OPC_WLX, OPT_REGW | OPT_EA, OPT_REGW)) - - /* prefixes */ - DEF_ASM_OP0(wait, 0x9b) - DEF_ASM_OP0(fwait, 0x9b) - DEF_ASM_OP0(aword, 0x67) - DEF_ASM_OP0(addr16, 0x67) - ALT(DEF_ASM_OP0(word, 0x66)) - DEF_ASM_OP0(data16, 0x66) - DEF_ASM_OP0(lock, 0xf0) - DEF_ASM_OP0(rep, 0xf3) - DEF_ASM_OP0(repe, 0xf3) - DEF_ASM_OP0(repz, 0xf3) - DEF_ASM_OP0(repne, 0xf2) - DEF_ASM_OP0(repnz, 0xf2) - - DEF_ASM_OP0(invd, 0x0f08) - DEF_ASM_OP0(wbinvd, 0x0f09) - DEF_ASM_OP0(cpuid, 0x0fa2) - DEF_ASM_OP0(wrmsr, 0x0f30) - DEF_ASM_OP0(rdtsc, 0x0f31) - DEF_ASM_OP0(rdmsr, 0x0f32) - DEF_ASM_OP0(rdpmc, 0x0f33) - DEF_ASM_OP0(ud2, 0x0f0b) - - /* NOTE: we took the same order as gas opcode definition order */ -ALT(DEF_ASM_OP2(movb, 0xa0, 0, OPC_BWLX, OPT_ADDR, OPT_EAX)) -ALT(DEF_ASM_OP2(movb, 0xa2, 0, OPC_BWLX, OPT_EAX, OPT_ADDR)) -ALT(DEF_ASM_OP2(movb, 0x88, 0, OPC_MODRM | OPC_BWLX, OPT_REG, OPT_EA | OPT_REG)) -ALT(DEF_ASM_OP2(movb, 0x8a, 0, OPC_MODRM | OPC_BWLX, OPT_EA | OPT_REG, OPT_REG)) -ALT(DEF_ASM_OP2(movb, 0xb0, 0, OPC_REG | OPC_BWLX, OPT_IM, OPT_REG)) -ALT(DEF_ASM_OP2(movb, 0xc6, 0, OPC_MODRM | OPC_BWLX, OPT_IM, OPT_REG | OPT_EA)) - -ALT(DEF_ASM_OP2(movw, 0x8c, 0, OPC_MODRM | OPC_WLX, OPT_SEG, OPT_EA | OPT_REG)) -ALT(DEF_ASM_OP2(movw, 0x8e, 0, OPC_MODRM | OPC_WLX, OPT_EA | OPT_REG, OPT_SEG)) - -ALT(DEF_ASM_OP2(movw, 0x0f20, 0, OPC_MODRM | OPC_WLX, OPT_CR, OPT_REG32)) -ALT(DEF_ASM_OP2(movw, 0x0f21, 0, OPC_MODRM | OPC_WLX, OPT_DB, OPT_REG32)) -ALT(DEF_ASM_OP2(movw, 0x0f24, 0, OPC_MODRM | OPC_WLX, OPT_TR, OPT_REG32)) -ALT(DEF_ASM_OP2(movw, 0x0f22, 0, OPC_MODRM | OPC_WLX, OPT_REG32, OPT_CR)) -ALT(DEF_ASM_OP2(movw, 0x0f23, 0, OPC_MODRM | OPC_WLX, OPT_REG32, OPT_DB)) -ALT(DEF_ASM_OP2(movw, 0x0f26, 0, OPC_MODRM | OPC_WLX, OPT_REG32, OPT_TR)) - -ALT(DEF_ASM_OP2(movsbl, 0x0fbe, 0, OPC_MODRM, OPT_REG8 | OPT_EA, OPT_REG32)) -ALT(DEF_ASM_OP2(movsbw, 0x660fbe, 0, OPC_MODRM, OPT_REG8 | OPT_EA, OPT_REG16)) -ALT(DEF_ASM_OP2(movswl, 0x0fbf, 0, OPC_MODRM, OPT_REG16 | OPT_EA, OPT_REG32)) -ALT(DEF_ASM_OP2(movzbw, 0x0fb6, 0, OPC_MODRM | OPC_WLX, OPT_REG8 | OPT_EA, OPT_REGW)) -ALT(DEF_ASM_OP2(movzwl, 0x0fb7, 0, OPC_MODRM, OPT_REG16 | OPT_EA, OPT_REG32)) - -ALT(DEF_ASM_OP1(pushw, 0x50, 0, OPC_REG | OPC_WLX, OPT_REGW)) -ALT(DEF_ASM_OP1(pushw, 0xff, 6, OPC_MODRM | OPC_WLX, OPT_REGW | OPT_EA)) -ALT(DEF_ASM_OP1(pushw, 0x6a, 0, OPC_WLX, OPT_IM8S)) -ALT(DEF_ASM_OP1(pushw, 0x68, 0, OPC_WLX, OPT_IM32)) -ALT(DEF_ASM_OP1(pushw, 0x06, 0, OPC_WLX, OPT_SEG)) - -ALT(DEF_ASM_OP1(popw, 0x58, 0, OPC_REG | OPC_WLX, OPT_REGW)) -ALT(DEF_ASM_OP1(popw, 0x8f, 0, OPC_MODRM | OPC_WLX, OPT_REGW | OPT_EA)) -ALT(DEF_ASM_OP1(popw, 0x07, 0, OPC_WLX, OPT_SEG)) - -ALT(DEF_ASM_OP2(xchgw, 0x90, 0, OPC_REG | OPC_WLX, OPT_REGW, OPT_EAX)) -ALT(DEF_ASM_OP2(xchgw, 0x90, 0, OPC_REG | OPC_WLX, OPT_EAX, OPT_REGW)) -ALT(DEF_ASM_OP2(xchgb, 0x86, 0, OPC_MODRM | OPC_BWLX, OPT_REG, OPT_EA | OPT_REG)) -ALT(DEF_ASM_OP2(xchgb, 0x86, 0, OPC_MODRM | OPC_BWLX, OPT_EA | OPT_REG, OPT_REG)) - -ALT(DEF_ASM_OP2(inb, 0xe4, 0, OPC_BWL, OPT_IM8, OPT_EAX)) -ALT(DEF_ASM_OP1(inb, 0xe4, 0, OPC_BWL, OPT_IM8)) -ALT(DEF_ASM_OP2(inb, 0xec, 0, OPC_BWL, OPT_DX, OPT_EAX)) -ALT(DEF_ASM_OP1(inb, 0xec, 0, OPC_BWL, OPT_DX)) - -ALT(DEF_ASM_OP2(outb, 0xe6, 0, OPC_BWL, OPT_EAX, OPT_IM8)) -ALT(DEF_ASM_OP1(outb, 0xe6, 0, OPC_BWL, OPT_IM8)) -ALT(DEF_ASM_OP2(outb, 0xee, 0, OPC_BWL, OPT_EAX, OPT_DX)) -ALT(DEF_ASM_OP1(outb, 0xee, 0, OPC_BWL, OPT_DX)) - -ALT(DEF_ASM_OP2(leaw, 0x8d, 0, OPC_MODRM | OPC_WLX, OPT_EA, OPT_REG)) - -ALT(DEF_ASM_OP2(les, 0xc4, 0, OPC_MODRM, OPT_EA, OPT_REG32)) -ALT(DEF_ASM_OP2(lds, 0xc5, 0, OPC_MODRM, OPT_EA, OPT_REG32)) -ALT(DEF_ASM_OP2(lss, 0x0fb2, 0, OPC_MODRM, OPT_EA, OPT_REG32)) -ALT(DEF_ASM_OP2(lfs, 0x0fb4, 0, OPC_MODRM, OPT_EA, OPT_REG32)) -ALT(DEF_ASM_OP2(lgs, 0x0fb5, 0, OPC_MODRM, OPT_EA, OPT_REG32)) - - /* arith */ -ALT(DEF_ASM_OP2(addb, 0x00, 0, OPC_ARITH | OPC_MODRM | OPC_BWLX, OPT_REG, OPT_EA | OPT_REG)) /* XXX: use D bit ? */ -ALT(DEF_ASM_OP2(addb, 0x02, 0, OPC_ARITH | OPC_MODRM | OPC_BWLX, OPT_EA | OPT_REG, OPT_REG)) -ALT(DEF_ASM_OP2(addb, 0x04, 0, OPC_ARITH | OPC_BWLX, OPT_IM, OPT_EAX)) -ALT(DEF_ASM_OP2(addw, 0x83, 0, OPC_ARITH | OPC_MODRM | OPC_WLX, OPT_IM8S, OPT_EA | OPT_REGW)) -ALT(DEF_ASM_OP2(addb, 0x80, 0, OPC_ARITH | OPC_MODRM | OPC_BWLX, OPT_IM, OPT_EA | OPT_REG)) - -ALT(DEF_ASM_OP2(testb, 0x84, 0, OPC_MODRM | OPC_BWLX, OPT_REG, OPT_EA | OPT_REG)) -ALT(DEF_ASM_OP2(testb, 0x84, 0, OPC_MODRM | OPC_BWLX, OPT_EA | OPT_REG, OPT_REG)) -ALT(DEF_ASM_OP2(testb, 0xa8, 0, OPC_BWLX, OPT_IM, OPT_EAX)) -ALT(DEF_ASM_OP2(testb, 0xf6, 0, OPC_MODRM | OPC_BWLX, OPT_IM, OPT_EA | OPT_REG)) - -ALT(DEF_ASM_OP1(incw, 0x40, 0, OPC_REG | OPC_WLX, OPT_REGW)) -ALT(DEF_ASM_OP1(incb, 0xfe, 0, OPC_MODRM | OPC_BWLX, OPT_REG | OPT_EA)) -ALT(DEF_ASM_OP1(decw, 0x48, 0, OPC_REG | OPC_WLX, OPT_REGW)) -ALT(DEF_ASM_OP1(decb, 0xfe, 1, OPC_MODRM | OPC_BWLX, OPT_REG | OPT_EA)) - -ALT(DEF_ASM_OP1(notb, 0xf6, 2, OPC_MODRM | OPC_BWLX, OPT_REG | OPT_EA)) -ALT(DEF_ASM_OP1(negb, 0xf6, 3, OPC_MODRM | OPC_BWLX, OPT_REG | OPT_EA)) - -ALT(DEF_ASM_OP1(mulb, 0xf6, 4, OPC_MODRM | OPC_BWLX, OPT_REG | OPT_EA)) -ALT(DEF_ASM_OP1(imulb, 0xf6, 5, OPC_MODRM | OPC_BWLX, OPT_REG | OPT_EA)) - -ALT(DEF_ASM_OP2(imulw, 0x0faf, 0, OPC_MODRM | OPC_WLX, OPT_REG | OPT_EA, OPT_REG)) -ALT(DEF_ASM_OP3(imulw, 0x6b, 0, OPC_MODRM | OPC_WLX, OPT_IM8S, OPT_REGW | OPT_EA, OPT_REGW)) -ALT(DEF_ASM_OP2(imulw, 0x6b, 0, OPC_MODRM | OPC_WLX, OPT_IM8S, OPT_REGW)) -ALT(DEF_ASM_OP3(imulw, 0x69, 0, OPC_MODRM | OPC_WLX, OPT_IMW, OPT_REGW | OPT_EA, OPT_REGW)) -ALT(DEF_ASM_OP2(imulw, 0x69, 0, OPC_MODRM | OPC_WLX, OPT_IMW, OPT_REGW)) - -ALT(DEF_ASM_OP1(divb, 0xf6, 6, OPC_MODRM | OPC_BWLX, OPT_REG | OPT_EA)) -ALT(DEF_ASM_OP2(divb, 0xf6, 6, OPC_MODRM | OPC_BWLX, OPT_REG | OPT_EA, OPT_EAX)) -ALT(DEF_ASM_OP1(idivb, 0xf6, 7, OPC_MODRM | OPC_BWLX, OPT_REG | OPT_EA)) -ALT(DEF_ASM_OP2(idivb, 0xf6, 7, OPC_MODRM | OPC_BWLX, OPT_REG | OPT_EA, OPT_EAX)) - - /* shifts */ -ALT(DEF_ASM_OP2(rolb, 0xc0, 0, OPC_MODRM | OPC_BWLX | OPC_SHIFT, OPT_IM8, OPT_EA | OPT_REG)) -ALT(DEF_ASM_OP2(rolb, 0xd2, 0, OPC_MODRM | OPC_BWLX | OPC_SHIFT, OPT_CL, OPT_EA | OPT_REG)) -ALT(DEF_ASM_OP1(rolb, 0xd0, 0, OPC_MODRM | OPC_BWLX | OPC_SHIFT, OPT_EA | OPT_REG)) - -ALT(DEF_ASM_OP3(shldw, 0x0fa4, 0, OPC_MODRM | OPC_WLX, OPT_IM8, OPT_REGW, OPT_EA | OPT_REGW)) -ALT(DEF_ASM_OP3(shldw, 0x0fa5, 0, OPC_MODRM | OPC_WLX, OPT_CL, OPT_REGW, OPT_EA | OPT_REGW)) -ALT(DEF_ASM_OP2(shldw, 0x0fa5, 0, OPC_MODRM | OPC_WLX, OPT_REGW, OPT_EA | OPT_REGW)) -ALT(DEF_ASM_OP3(shrdw, 0x0fac, 0, OPC_MODRM | OPC_WLX, OPT_IM8, OPT_REGW, OPT_EA | OPT_REGW)) -ALT(DEF_ASM_OP3(shrdw, 0x0fad, 0, OPC_MODRM | OPC_WLX, OPT_CL, OPT_REGW, OPT_EA | OPT_REGW)) -ALT(DEF_ASM_OP2(shrdw, 0x0fad, 0, OPC_MODRM | OPC_WLX, OPT_REGW, OPT_EA | OPT_REGW)) - -ALT(DEF_ASM_OP1(call, 0xff, 2, OPC_MODRM, OPT_INDIR)) -ALT(DEF_ASM_OP1(call, 0xe8, 0, 0, OPT_DISP)) -ALT(DEF_ASM_OP1(jmp, 0xff, 4, OPC_MODRM, OPT_INDIR)) -ALT(DEF_ASM_OP1(jmp, 0xeb, 0, 0, OPT_DISP8)) - -ALT(DEF_ASM_OP2(lcall, 0x9a, 0, 0, OPT_IM16, OPT_IM32)) -ALT(DEF_ASM_OP1(lcall, 0xff, 3, OPC_MODRM, OPT_EA)) -ALT(DEF_ASM_OP2(ljmp, 0xea, 0, 0, OPT_IM16, OPT_IM32)) -ALT(DEF_ASM_OP1(ljmp, 0xff, 5, OPC_MODRM, OPT_EA)) - -ALT(DEF_ASM_OP1(int, 0xcd, 0, 0, OPT_IM8)) -ALT(DEF_ASM_OP1(seto, 0x0f90, 0, OPC_MODRM | OPC_TEST, OPT_REG8 | OPT_EA)) -ALT(DEF_ASM_OP1(setob, 0x0f90, 0, OPC_MODRM | OPC_TEST, OPT_REG8 | OPT_EA)) - DEF_ASM_OP2(enter, 0xc8, 0, 0, OPT_IM16, OPT_IM8) - DEF_ASM_OP0(leave, 0xc9) - DEF_ASM_OP0(ret, 0xc3) - DEF_ASM_OP0(retl,0xc3) -ALT(DEF_ASM_OP1(retl,0xc2, 0, 0, OPT_IM16)) -ALT(DEF_ASM_OP1(ret, 0xc2, 0, 0, OPT_IM16)) - DEF_ASM_OP0(lret, 0xcb) -ALT(DEF_ASM_OP1(lret, 0xca, 0, 0, OPT_IM16)) - -ALT(DEF_ASM_OP1(jo, 0x70, 0, OPC_TEST, OPT_DISP8)) - DEF_ASM_OP1(loopne, 0xe0, 0, 0, OPT_DISP8) - DEF_ASM_OP1(loopnz, 0xe0, 0, 0, OPT_DISP8) - DEF_ASM_OP1(loope, 0xe1, 0, 0, OPT_DISP8) - DEF_ASM_OP1(loopz, 0xe1, 0, 0, OPT_DISP8) - DEF_ASM_OP1(loop, 0xe2, 0, 0, OPT_DISP8) - DEF_ASM_OP1(jecxz, 0xe3, 0, 0, OPT_DISP8) - - /* float */ - /* specific fcomp handling */ -ALT(DEF_ASM_OP0L(fcomp, 0xd8d9, 0, 0)) - -ALT(DEF_ASM_OP1(fadd, 0xd8c0, 0, OPC_FARITH | OPC_REG, OPT_ST)) -ALT(DEF_ASM_OP2(fadd, 0xd8c0, 0, OPC_FARITH | OPC_REG, OPT_ST, OPT_ST0)) -ALT(DEF_ASM_OP2(fadd, 0xdcc0, 0, OPC_FARITH | OPC_REG, OPT_ST0, OPT_ST)) -ALT(DEF_ASM_OP2(fmul, 0xdcc8, 0, OPC_FARITH | OPC_REG, OPT_ST0, OPT_ST)) -ALT(DEF_ASM_OP0L(fadd, 0xdec1, 0, OPC_FARITH)) -ALT(DEF_ASM_OP1(faddp, 0xdec0, 0, OPC_FARITH | OPC_REG, OPT_ST)) -ALT(DEF_ASM_OP2(faddp, 0xdec0, 0, OPC_FARITH | OPC_REG, OPT_ST, OPT_ST0)) -ALT(DEF_ASM_OP2(faddp, 0xdec0, 0, OPC_FARITH | OPC_REG, OPT_ST0, OPT_ST)) -ALT(DEF_ASM_OP0L(faddp, 0xdec1, 0, OPC_FARITH)) -ALT(DEF_ASM_OP1(fadds, 0xd8, 0, OPC_FARITH | OPC_MODRM, OPT_EA)) -ALT(DEF_ASM_OP1(fiaddl, 0xda, 0, OPC_FARITH | OPC_MODRM, OPT_EA)) -ALT(DEF_ASM_OP1(faddl, 0xdc, 0, OPC_FARITH | OPC_MODRM, OPT_EA)) -ALT(DEF_ASM_OP1(fiadds, 0xde, 0, OPC_FARITH | OPC_MODRM, OPT_EA)) - - DEF_ASM_OP0(fucompp, 0xdae9) - DEF_ASM_OP0(ftst, 0xd9e4) - DEF_ASM_OP0(fxam, 0xd9e5) - DEF_ASM_OP0(fld1, 0xd9e8) - DEF_ASM_OP0(fldl2t, 0xd9e9) - DEF_ASM_OP0(fldl2e, 0xd9ea) - DEF_ASM_OP0(fldpi, 0xd9eb) - DEF_ASM_OP0(fldlg2, 0xd9ec) - DEF_ASM_OP0(fldln2, 0xd9ed) - DEF_ASM_OP0(fldz, 0xd9ee) - - DEF_ASM_OP0(f2xm1, 0xd9f0) - DEF_ASM_OP0(fyl2x, 0xd9f1) - DEF_ASM_OP0(fptan, 0xd9f2) - DEF_ASM_OP0(fpatan, 0xd9f3) - DEF_ASM_OP0(fxtract, 0xd9f4) - DEF_ASM_OP0(fprem1, 0xd9f5) - DEF_ASM_OP0(fdecstp, 0xd9f6) - DEF_ASM_OP0(fincstp, 0xd9f7) - DEF_ASM_OP0(fprem, 0xd9f8) - DEF_ASM_OP0(fyl2xp1, 0xd9f9) - DEF_ASM_OP0(fsqrt, 0xd9fa) - DEF_ASM_OP0(fsincos, 0xd9fb) - DEF_ASM_OP0(frndint, 0xd9fc) - DEF_ASM_OP0(fscale, 0xd9fd) - DEF_ASM_OP0(fsin, 0xd9fe) - DEF_ASM_OP0(fcos, 0xd9ff) - DEF_ASM_OP0(fchs, 0xd9e0) - DEF_ASM_OP0(fabs, 0xd9e1) - DEF_ASM_OP0(fninit, 0xdbe3) - DEF_ASM_OP0(fnclex, 0xdbe2) - DEF_ASM_OP0(fnop, 0xd9d0) - - /* fp load */ - DEF_ASM_OP1(fld, 0xd9c0, 0, OPC_REG, OPT_ST) - DEF_ASM_OP1(fldl, 0xd9c0, 0, OPC_REG, OPT_ST) - DEF_ASM_OP1(flds, 0xd9, 0, OPC_MODRM, OPT_EA) -ALT(DEF_ASM_OP1(fldl, 0xdd, 0, OPC_MODRM, OPT_EA)) - DEF_ASM_OP1(fildl, 0xdb, 0, OPC_MODRM, OPT_EA) - DEF_ASM_OP1(fildq, 0xdf, 5, OPC_MODRM, OPT_EA) - DEF_ASM_OP1(fildll, 0xdf, 5, OPC_MODRM,OPT_EA) - DEF_ASM_OP1(fldt, 0xdb, 5, OPC_MODRM, OPT_EA) - DEF_ASM_OP1(fbld, 0xdf, 4, OPC_MODRM, OPT_EA) - - /* fp store */ - DEF_ASM_OP1(fst, 0xddd0, 0, OPC_REG, OPT_ST) - DEF_ASM_OP1(fstl, 0xddd0, 0, OPC_REG, OPT_ST) - DEF_ASM_OP1(fsts, 0xd9, 2, OPC_MODRM, OPT_EA) - DEF_ASM_OP1(fstps, 0xd9, 3, OPC_MODRM, OPT_EA) -ALT(DEF_ASM_OP1(fstl, 0xdd, 2, OPC_MODRM, OPT_EA)) - DEF_ASM_OP1(fstpl, 0xdd, 3, OPC_MODRM, OPT_EA) - DEF_ASM_OP1(fist, 0xdf, 2, OPC_MODRM, OPT_EA) - DEF_ASM_OP1(fistp, 0xdf, 3, OPC_MODRM, OPT_EA) - DEF_ASM_OP1(fistl, 0xdb, 2, OPC_MODRM, OPT_EA) - DEF_ASM_OP1(fistpl, 0xdb, 3, OPC_MODRM, OPT_EA) - - DEF_ASM_OP1(fstp, 0xddd8, 0, OPC_REG, OPT_ST) - DEF_ASM_OP1(fistpq, 0xdf, 7, OPC_MODRM, OPT_EA) - DEF_ASM_OP1(fistpll, 0xdf, 7, OPC_MODRM, OPT_EA) - DEF_ASM_OP1(fstpt, 0xdb, 7, OPC_MODRM, OPT_EA) - DEF_ASM_OP1(fbstp, 0xdf, 6, OPC_MODRM, OPT_EA) - - /* exchange */ - DEF_ASM_OP0(fxch, 0xd9c9) -ALT(DEF_ASM_OP1(fxch, 0xd9c8, 0, OPC_REG, OPT_ST)) - - /* misc FPU */ - DEF_ASM_OP1(fucom, 0xdde0, 0, OPC_REG, OPT_ST ) - DEF_ASM_OP1(fucomp, 0xdde8, 0, OPC_REG, OPT_ST ) - - DEF_ASM_OP0L(finit, 0xdbe3, 0, OPC_FWAIT) - DEF_ASM_OP1(fldcw, 0xd9, 5, OPC_MODRM, OPT_EA ) - DEF_ASM_OP1(fnstcw, 0xd9, 7, OPC_MODRM, OPT_EA ) - DEF_ASM_OP1(fstcw, 0xd9, 7, OPC_MODRM | OPC_FWAIT, OPT_EA ) - DEF_ASM_OP0(fnstsw, 0xdfe0) -ALT(DEF_ASM_OP1(fnstsw, 0xdfe0, 0, 0, OPT_EAX )) -ALT(DEF_ASM_OP1(fnstsw, 0xdd, 7, OPC_MODRM, OPT_EA )) - DEF_ASM_OP1(fstsw, 0xdfe0, 0, OPC_FWAIT, OPT_EAX ) -ALT(DEF_ASM_OP0L(fstsw, 0xdfe0, 0, OPC_FWAIT)) -ALT(DEF_ASM_OP1(fstsw, 0xdd, 7, OPC_MODRM | OPC_FWAIT, OPT_EA )) - DEF_ASM_OP0L(fclex, 0xdbe2, 0, OPC_FWAIT) - DEF_ASM_OP1(fnstenv, 0xd9, 6, OPC_MODRM, OPT_EA ) - DEF_ASM_OP1(fstenv, 0xd9, 6, OPC_MODRM | OPC_FWAIT, OPT_EA ) - DEF_ASM_OP1(fldenv, 0xd9, 4, OPC_MODRM, OPT_EA ) - DEF_ASM_OP1(fnsave, 0xdd, 6, OPC_MODRM, OPT_EA ) - DEF_ASM_OP1(fsave, 0xdd, 6, OPC_MODRM | OPC_FWAIT, OPT_EA ) - DEF_ASM_OP1(frstor, 0xdd, 4, OPC_MODRM, OPT_EA ) - DEF_ASM_OP1(ffree, 0xddc0, 4, OPC_REG, OPT_ST ) - DEF_ASM_OP1(ffreep, 0xdfc0, 4, OPC_REG, OPT_ST ) - DEF_ASM_OP1(fxsave, 0x0fae, 0, OPC_MODRM, OPT_EA ) - DEF_ASM_OP1(fxrstor, 0x0fae, 1, OPC_MODRM, OPT_EA ) - - /* segments */ - DEF_ASM_OP2(arpl, 0x63, 0, OPC_MODRM, OPT_REG16, OPT_REG16 | OPT_EA) -ALT(DEF_ASM_OP2(larw, 0x0f02, 0, OPC_MODRM | OPC_WLX, OPT_REG | OPT_EA, OPT_REG)) - DEF_ASM_OP1(lgdt, 0x0f01, 2, OPC_MODRM, OPT_EA) - DEF_ASM_OP1(lidt, 0x0f01, 3, OPC_MODRM, OPT_EA) - DEF_ASM_OP1(lldt, 0x0f00, 2, OPC_MODRM, OPT_EA | OPT_REG) - DEF_ASM_OP1(lmsw, 0x0f01, 6, OPC_MODRM, OPT_EA | OPT_REG) -ALT(DEF_ASM_OP2(lslw, 0x0f03, 0, OPC_MODRM | OPC_WLX, OPT_EA | OPT_REG, OPT_REG)) - DEF_ASM_OP1(ltr, 0x0f00, 3, OPC_MODRM, OPT_EA | OPT_REG) - DEF_ASM_OP1(sgdt, 0x0f01, 0, OPC_MODRM, OPT_EA) - DEF_ASM_OP1(sidt, 0x0f01, 1, OPC_MODRM, OPT_EA) - DEF_ASM_OP1(sldt, 0x0f00, 0, OPC_MODRM, OPT_REG | OPT_EA) - DEF_ASM_OP1(smsw, 0x0f01, 4, OPC_MODRM, OPT_REG | OPT_EA) - DEF_ASM_OP1(str, 0x0f00, 1, OPC_MODRM, OPT_REG16| OPT_EA) - DEF_ASM_OP1(verr, 0x0f00, 4, OPC_MODRM, OPT_REG | OPT_EA) - DEF_ASM_OP1(verw, 0x0f00, 5, OPC_MODRM, OPT_REG | OPT_EA) - - /* 486 */ - DEF_ASM_OP1(bswap, 0x0fc8, 0, OPC_REG, OPT_REG32 ) -ALT(DEF_ASM_OP2(xaddb, 0x0fc0, 0, OPC_MODRM | OPC_BWLX, OPT_REG, OPT_REG | OPT_EA )) -ALT(DEF_ASM_OP2(cmpxchgb, 0x0fb0, 0, OPC_MODRM | OPC_BWLX, OPT_REG, OPT_REG | OPT_EA )) - DEF_ASM_OP1(invlpg, 0x0f01, 7, OPC_MODRM, OPT_EA ) - - DEF_ASM_OP2(boundl, 0x62, 0, OPC_MODRM, OPT_REG32, OPT_EA) - DEF_ASM_OP2(boundw, 0x6662, 0, OPC_MODRM, OPT_REG16, OPT_EA) - - /* pentium */ - DEF_ASM_OP1(cmpxchg8b, 0x0fc7, 1, OPC_MODRM, OPT_EA ) - - /* pentium pro */ -ALT(DEF_ASM_OP2(cmovo, 0x0f40, 0, OPC_MODRM | OPC_TEST | OPC_WLX, OPT_REGW | OPT_EA, OPT_REGW)) - DEF_ASM_OP2(fcmovb, 0xdac0, 0, OPC_REG, OPT_ST, OPT_ST0 ) - DEF_ASM_OP2(fcmove, 0xdac8, 0, OPC_REG, OPT_ST, OPT_ST0 ) - DEF_ASM_OP2(fcmovbe, 0xdad0, 0, OPC_REG, OPT_ST, OPT_ST0 ) - DEF_ASM_OP2(fcmovu, 0xdad8, 0, OPC_REG, OPT_ST, OPT_ST0 ) - DEF_ASM_OP2(fcmovnb, 0xdbc0, 0, OPC_REG, OPT_ST, OPT_ST0 ) - DEF_ASM_OP2(fcmovne, 0xdbc8, 0, OPC_REG, OPT_ST, OPT_ST0 ) - DEF_ASM_OP2(fcmovnbe, 0xdbd0, 0, OPC_REG, OPT_ST, OPT_ST0 ) - DEF_ASM_OP2(fcmovnu, 0xdbd8, 0, OPC_REG, OPT_ST, OPT_ST0 ) - - DEF_ASM_OP2(fucomi, 0xdbe8, 0, OPC_REG, OPT_ST, OPT_ST0 ) - DEF_ASM_OP2(fcomi, 0xdbf0, 0, OPC_REG, OPT_ST, OPT_ST0 ) - DEF_ASM_OP2(fucomip, 0xdfe8, 0, OPC_REG, OPT_ST, OPT_ST0 ) - DEF_ASM_OP2(fcomip, 0xdff0, 0, OPC_REG, OPT_ST, OPT_ST0 ) - - /* mmx */ - DEF_ASM_OP0(emms, 0x0f77) /* must be last OP0 */ - DEF_ASM_OP2(movd, 0x0f6e, 0, OPC_MODRM, OPT_EA | OPT_REG32, OPT_MMXSSE ) - DEF_ASM_OP2(movq, 0x0f6f, 0, OPC_MODRM, OPT_EA | OPT_MMX, OPT_MMX ) -ALT(DEF_ASM_OP2(movd, 0x0f7e, 0, OPC_MODRM, OPT_MMXSSE, OPT_EA | OPT_REG32 )) -ALT(DEF_ASM_OP2(movq, 0x0f7f, 0, OPC_MODRM, OPT_MMX, OPT_EA | OPT_MMX )) -ALT(DEF_ASM_OP2(movq, 0x660fd6, 0, OPC_MODRM, OPT_SSE, OPT_EA | OPT_SSE )) -ALT(DEF_ASM_OP2(movq, 0xf30f7e, 0, OPC_MODRM, OPT_EA | OPT_SSE, OPT_SSE )) - - DEF_ASM_OP2(packssdw, 0x0f6b, 0, OPC_MODRM, OPT_EA | OPT_MMXSSE, OPT_MMXSSE ) - DEF_ASM_OP2(packsswb, 0x0f63, 0, OPC_MODRM, OPT_EA | OPT_MMXSSE, OPT_MMXSSE ) - DEF_ASM_OP2(packuswb, 0x0f67, 0, OPC_MODRM, OPT_EA | OPT_MMXSSE, OPT_MMXSSE ) - DEF_ASM_OP2(paddb, 0x0ffc, 0, OPC_MODRM, OPT_EA | OPT_MMXSSE, OPT_MMXSSE ) - DEF_ASM_OP2(paddw, 0x0ffd, 0, OPC_MODRM, OPT_EA | OPT_MMXSSE, OPT_MMXSSE ) - DEF_ASM_OP2(paddd, 0x0ffe, 0, OPC_MODRM, OPT_EA | OPT_MMXSSE, OPT_MMXSSE ) - DEF_ASM_OP2(paddsb, 0x0fec, 0, OPC_MODRM, OPT_EA | OPT_MMXSSE, OPT_MMXSSE ) - DEF_ASM_OP2(paddsw, 0x0fed, 0, OPC_MODRM, OPT_EA | OPT_MMXSSE, OPT_MMXSSE ) - DEF_ASM_OP2(paddusb, 0x0fdc, 0, OPC_MODRM, OPT_EA | OPT_MMXSSE, OPT_MMXSSE ) - DEF_ASM_OP2(paddusw, 0x0fdd, 0, OPC_MODRM, OPT_EA | OPT_MMXSSE, OPT_MMXSSE ) - DEF_ASM_OP2(pand, 0x0fdb, 0, OPC_MODRM, OPT_EA | OPT_MMXSSE, OPT_MMXSSE ) - DEF_ASM_OP2(pandn, 0x0fdf, 0, OPC_MODRM, OPT_EA | OPT_MMXSSE, OPT_MMXSSE ) - DEF_ASM_OP2(pcmpeqb, 0x0f74, 0, OPC_MODRM, OPT_EA | OPT_MMXSSE, OPT_MMXSSE ) - DEF_ASM_OP2(pcmpeqw, 0x0f75, 0, OPC_MODRM, OPT_EA | OPT_MMXSSE, OPT_MMXSSE ) - DEF_ASM_OP2(pcmpeqd, 0x0f76, 0, OPC_MODRM, OPT_EA | OPT_MMXSSE, OPT_MMXSSE ) - DEF_ASM_OP2(pcmpgtb, 0x0f64, 0, OPC_MODRM, OPT_EA | OPT_MMXSSE, OPT_MMXSSE ) - DEF_ASM_OP2(pcmpgtw, 0x0f65, 0, OPC_MODRM, OPT_EA | OPT_MMXSSE, OPT_MMXSSE ) - DEF_ASM_OP2(pcmpgtd, 0x0f66, 0, OPC_MODRM, OPT_EA | OPT_MMXSSE, OPT_MMXSSE ) - DEF_ASM_OP2(pmaddwd, 0x0ff5, 0, OPC_MODRM, OPT_EA | OPT_MMXSSE, OPT_MMXSSE ) - DEF_ASM_OP2(pmulhw, 0x0fe5, 0, OPC_MODRM, OPT_EA | OPT_MMXSSE, OPT_MMXSSE ) - DEF_ASM_OP2(pmullw, 0x0fd5, 0, OPC_MODRM, OPT_EA | OPT_MMXSSE, OPT_MMXSSE ) - DEF_ASM_OP2(por, 0x0feb, 0, OPC_MODRM, OPT_EA | OPT_MMXSSE, OPT_MMXSSE ) - DEF_ASM_OP2(psllw, 0x0ff1, 0, OPC_MODRM, OPT_EA | OPT_MMXSSE, OPT_MMXSSE ) -ALT(DEF_ASM_OP2(psllw, 0x0f71, 6, OPC_MODRM, OPT_IM8, OPT_MMXSSE )) - DEF_ASM_OP2(pslld, 0x0ff2, 0, OPC_MODRM, OPT_EA | OPT_MMXSSE, OPT_MMXSSE ) -ALT(DEF_ASM_OP2(pslld, 0x0f72, 6, OPC_MODRM, OPT_IM8, OPT_MMXSSE )) - DEF_ASM_OP2(psllq, 0x0ff3, 0, OPC_MODRM, OPT_EA | OPT_MMXSSE, OPT_MMXSSE ) -ALT(DEF_ASM_OP2(psllq, 0x0f73, 6, OPC_MODRM, OPT_IM8, OPT_MMXSSE )) - DEF_ASM_OP2(psraw, 0x0fe1, 0, OPC_MODRM, OPT_EA | OPT_MMXSSE, OPT_MMXSSE ) -ALT(DEF_ASM_OP2(psraw, 0x0f71, 4, OPC_MODRM, OPT_IM8, OPT_MMXSSE )) - DEF_ASM_OP2(psrad, 0x0fe2, 0, OPC_MODRM, OPT_EA | OPT_MMXSSE, OPT_MMXSSE ) -ALT(DEF_ASM_OP2(psrad, 0x0f72, 4, OPC_MODRM, OPT_IM8, OPT_MMXSSE )) - DEF_ASM_OP2(psrlw, 0x0fd1, 0, OPC_MODRM, OPT_EA | OPT_MMXSSE, OPT_MMXSSE ) -ALT(DEF_ASM_OP2(psrlw, 0x0f71, 2, OPC_MODRM, OPT_IM8, OPT_MMXSSE )) - DEF_ASM_OP2(psrld, 0x0fd2, 0, OPC_MODRM, OPT_EA | OPT_MMXSSE, OPT_MMXSSE ) -ALT(DEF_ASM_OP2(psrld, 0x0f72, 2, OPC_MODRM, OPT_IM8, OPT_MMXSSE )) - DEF_ASM_OP2(psrlq, 0x0fd3, 0, OPC_MODRM, OPT_EA | OPT_MMXSSE, OPT_MMXSSE ) -ALT(DEF_ASM_OP2(psrlq, 0x0f73, 2, OPC_MODRM, OPT_IM8, OPT_MMXSSE )) - DEF_ASM_OP2(psubb, 0x0ff8, 0, OPC_MODRM, OPT_EA | OPT_MMXSSE, OPT_MMXSSE ) - DEF_ASM_OP2(psubw, 0x0ff9, 0, OPC_MODRM, OPT_EA | OPT_MMXSSE, OPT_MMXSSE ) - DEF_ASM_OP2(psubd, 0x0ffa, 0, OPC_MODRM, OPT_EA | OPT_MMXSSE, OPT_MMXSSE ) - DEF_ASM_OP2(psubsb, 0x0fe8, 0, OPC_MODRM, OPT_EA | OPT_MMXSSE, OPT_MMXSSE ) - DEF_ASM_OP2(psubsw, 0x0fe9, 0, OPC_MODRM, OPT_EA | OPT_MMXSSE, OPT_MMXSSE ) - DEF_ASM_OP2(psubusb, 0x0fd8, 0, OPC_MODRM, OPT_EA | OPT_MMXSSE, OPT_MMXSSE ) - DEF_ASM_OP2(psubusw, 0x0fd9, 0, OPC_MODRM, OPT_EA | OPT_MMXSSE, OPT_MMXSSE ) - DEF_ASM_OP2(punpckhbw, 0x0f68, 0, OPC_MODRM, OPT_EA | OPT_MMXSSE, OPT_MMXSSE ) - DEF_ASM_OP2(punpckhwd, 0x0f69, 0, OPC_MODRM, OPT_EA | OPT_MMXSSE, OPT_MMXSSE ) - DEF_ASM_OP2(punpckhdq, 0x0f6a, 0, OPC_MODRM, OPT_EA | OPT_MMXSSE, OPT_MMXSSE ) - DEF_ASM_OP2(punpcklbw, 0x0f60, 0, OPC_MODRM, OPT_EA | OPT_MMXSSE, OPT_MMXSSE ) - DEF_ASM_OP2(punpcklwd, 0x0f61, 0, OPC_MODRM, OPT_EA | OPT_MMXSSE, OPT_MMXSSE ) - DEF_ASM_OP2(punpckldq, 0x0f62, 0, OPC_MODRM, OPT_EA | OPT_MMXSSE, OPT_MMXSSE ) - DEF_ASM_OP2(pxor, 0x0fef, 0, OPC_MODRM, OPT_EA | OPT_MMXSSE, OPT_MMXSSE ) - - /* sse */ - DEF_ASM_OP1(ldmxcsr, 0x0fae, 2, OPC_MODRM, OPT_EA) - DEF_ASM_OP1(stmxcsr, 0x0fae, 3, OPC_MODRM, OPT_EA) - DEF_ASM_OP2(movups, 0x0f10, 0, OPC_MODRM, OPT_EA | OPT_REG32, OPT_SSE ) -ALT(DEF_ASM_OP2(movups, 0x0f11, 0, OPC_MODRM, OPT_SSE, OPT_EA | OPT_REG32 )) - DEF_ASM_OP2(movaps, 0x0f28, 0, OPC_MODRM, OPT_EA | OPT_REG32, OPT_SSE ) -ALT(DEF_ASM_OP2(movaps, 0x0f29, 0, OPC_MODRM, OPT_SSE, OPT_EA | OPT_REG32 )) - DEF_ASM_OP2(movhps, 0x0f16, 0, OPC_MODRM, OPT_EA | OPT_REG32, OPT_SSE ) -ALT(DEF_ASM_OP2(movhps, 0x0f17, 0, OPC_MODRM, OPT_SSE, OPT_EA | OPT_REG32 )) - DEF_ASM_OP2(addps, 0x0f58, 0, OPC_MODRM, OPT_EA | OPT_SSE, OPT_SSE ) - DEF_ASM_OP2(cvtpi2ps, 0x0f2a, 0, OPC_MODRM, OPT_EA | OPT_MMX, OPT_SSE ) - DEF_ASM_OP2(cvtps2pi, 0x0f2d, 0, OPC_MODRM, OPT_EA | OPT_SSE, OPT_MMX ) - DEF_ASM_OP2(cvttps2pi, 0x0f2c, 0, OPC_MODRM, OPT_EA | OPT_SSE, OPT_MMX ) - DEF_ASM_OP2(divps, 0x0f5e, 0, OPC_MODRM, OPT_EA | OPT_SSE, OPT_SSE ) - DEF_ASM_OP2(maxps, 0x0f5f, 0, OPC_MODRM, OPT_EA | OPT_SSE, OPT_SSE ) - DEF_ASM_OP2(minps, 0x0f5d, 0, OPC_MODRM, OPT_EA | OPT_SSE, OPT_SSE ) - DEF_ASM_OP2(mulps, 0x0f59, 0, OPC_MODRM, OPT_EA | OPT_SSE, OPT_SSE ) - DEF_ASM_OP2(pavgb, 0x0fe0, 0, OPC_MODRM, OPT_EA | OPT_SSE, OPT_SSE ) - DEF_ASM_OP2(pavgw, 0x0fe3, 0, OPC_MODRM, OPT_EA | OPT_SSE, OPT_SSE ) - DEF_ASM_OP2(pmaxsw, 0x0fee, 0, OPC_MODRM, OPT_EA | OPT_MMXSSE, OPT_MMXSSE ) - DEF_ASM_OP2(pmaxub, 0x0fde, 0, OPC_MODRM, OPT_EA | OPT_MMXSSE, OPT_MMXSSE ) - DEF_ASM_OP2(pminsw, 0x0fea, 0, OPC_MODRM, OPT_EA | OPT_MMXSSE, OPT_MMXSSE ) - DEF_ASM_OP2(pminub, 0x0fda, 0, OPC_MODRM, OPT_EA | OPT_MMXSSE, OPT_MMXSSE ) - DEF_ASM_OP2(rcpss, 0x0f53, 0, OPC_MODRM, OPT_EA | OPT_SSE, OPT_SSE ) - DEF_ASM_OP2(rsqrtps, 0x0f52, 0, OPC_MODRM, OPT_EA | OPT_SSE, OPT_SSE ) - DEF_ASM_OP2(sqrtps, 0x0f51, 0, OPC_MODRM, OPT_EA | OPT_SSE, OPT_SSE ) - DEF_ASM_OP2(subps, 0x0f5c, 0, OPC_MODRM, OPT_EA | OPT_SSE, OPT_SSE ) - -#undef ALT -#undef DEF_ASM_OP0 -#undef DEF_ASM_OP0L -#undef DEF_ASM_OP1 -#undef DEF_ASM_OP2 -#undef DEF_ASM_OP3 diff --git a/vendor/tcc/i386-tok.h b/vendor/tcc/i386-tok.h deleted file mode 100644 index 29071059..00000000 --- a/vendor/tcc/i386-tok.h +++ /dev/null @@ -1,332 +0,0 @@ -/* ------------------------------------------------------------------ */ -/* WARNING: relative order of tokens is important. */ - -#define DEF_BWL(x) \ - DEF(TOK_ASM_ ## x ## b, #x "b") \ - DEF(TOK_ASM_ ## x ## w, #x "w") \ - DEF(TOK_ASM_ ## x ## l, #x "l") \ - DEF(TOK_ASM_ ## x, #x) -#define DEF_WL(x) \ - DEF(TOK_ASM_ ## x ## w, #x "w") \ - DEF(TOK_ASM_ ## x ## l, #x "l") \ - DEF(TOK_ASM_ ## x, #x) -#ifdef TCC_TARGET_X86_64 -# define DEF_BWLQ(x) \ - DEF(TOK_ASM_ ## x ## b, #x "b") \ - DEF(TOK_ASM_ ## x ## w, #x "w") \ - DEF(TOK_ASM_ ## x ## l, #x "l") \ - DEF(TOK_ASM_ ## x ## q, #x "q") \ - DEF(TOK_ASM_ ## x, #x) -# define DEF_WLQ(x) \ - DEF(TOK_ASM_ ## x ## w, #x "w") \ - DEF(TOK_ASM_ ## x ## l, #x "l") \ - DEF(TOK_ASM_ ## x ## q, #x "q") \ - DEF(TOK_ASM_ ## x, #x) -# define DEF_BWLX DEF_BWLQ -# define DEF_WLX DEF_WLQ -/* number of sizes + 1 */ -# define NBWLX 5 -#else -# define DEF_BWLX DEF_BWL -# define DEF_WLX DEF_WL -/* number of sizes + 1 */ -# define NBWLX 4 -#endif - -#define DEF_FP1(x) \ - DEF(TOK_ASM_ ## f ## x ## s, "f" #x "s") \ - DEF(TOK_ASM_ ## fi ## x ## l, "fi" #x "l") \ - DEF(TOK_ASM_ ## f ## x ## l, "f" #x "l") \ - DEF(TOK_ASM_ ## fi ## x ## s, "fi" #x "s") - -#define DEF_FP(x) \ - DEF(TOK_ASM_ ## f ## x, "f" #x ) \ - DEF(TOK_ASM_ ## f ## x ## p, "f" #x "p") \ - DEF_FP1(x) - -#define DEF_ASMTEST(x,suffix) \ - DEF_ASM(x ## o ## suffix) \ - DEF_ASM(x ## no ## suffix) \ - DEF_ASM(x ## b ## suffix) \ - DEF_ASM(x ## c ## suffix) \ - DEF_ASM(x ## nae ## suffix) \ - DEF_ASM(x ## nb ## suffix) \ - DEF_ASM(x ## nc ## suffix) \ - DEF_ASM(x ## ae ## suffix) \ - DEF_ASM(x ## e ## suffix) \ - DEF_ASM(x ## z ## suffix) \ - DEF_ASM(x ## ne ## suffix) \ - DEF_ASM(x ## nz ## suffix) \ - DEF_ASM(x ## be ## suffix) \ - DEF_ASM(x ## na ## suffix) \ - DEF_ASM(x ## nbe ## suffix) \ - DEF_ASM(x ## a ## suffix) \ - DEF_ASM(x ## s ## suffix) \ - DEF_ASM(x ## ns ## suffix) \ - DEF_ASM(x ## p ## suffix) \ - DEF_ASM(x ## pe ## suffix) \ - DEF_ASM(x ## np ## suffix) \ - DEF_ASM(x ## po ## suffix) \ - DEF_ASM(x ## l ## suffix) \ - DEF_ASM(x ## nge ## suffix) \ - DEF_ASM(x ## nl ## suffix) \ - DEF_ASM(x ## ge ## suffix) \ - DEF_ASM(x ## le ## suffix) \ - DEF_ASM(x ## ng ## suffix) \ - DEF_ASM(x ## nle ## suffix) \ - DEF_ASM(x ## g ## suffix) - -/* ------------------------------------------------------------------ */ -/* register */ - DEF_ASM(al) - DEF_ASM(cl) - DEF_ASM(dl) - DEF_ASM(bl) - DEF_ASM(ah) - DEF_ASM(ch) - DEF_ASM(dh) - DEF_ASM(bh) - DEF_ASM(ax) - DEF_ASM(cx) - DEF_ASM(dx) - DEF_ASM(bx) - DEF_ASM(sp) - DEF_ASM(bp) - DEF_ASM(si) - DEF_ASM(di) - DEF_ASM(eax) - DEF_ASM(ecx) - DEF_ASM(edx) - DEF_ASM(ebx) - DEF_ASM(esp) - DEF_ASM(ebp) - DEF_ASM(esi) - DEF_ASM(edi) -#ifdef TCC_TARGET_X86_64 - DEF_ASM(rax) - DEF_ASM(rcx) - DEF_ASM(rdx) - DEF_ASM(rbx) - DEF_ASM(rsp) - DEF_ASM(rbp) - DEF_ASM(rsi) - DEF_ASM(rdi) -#endif - DEF_ASM(mm0) - DEF_ASM(mm1) - DEF_ASM(mm2) - DEF_ASM(mm3) - DEF_ASM(mm4) - DEF_ASM(mm5) - DEF_ASM(mm6) - DEF_ASM(mm7) - DEF_ASM(xmm0) - DEF_ASM(xmm1) - DEF_ASM(xmm2) - DEF_ASM(xmm3) - DEF_ASM(xmm4) - DEF_ASM(xmm5) - DEF_ASM(xmm6) - DEF_ASM(xmm7) - DEF_ASM(cr0) - DEF_ASM(cr1) - DEF_ASM(cr2) - DEF_ASM(cr3) - DEF_ASM(cr4) - DEF_ASM(cr5) - DEF_ASM(cr6) - DEF_ASM(cr7) - DEF_ASM(tr0) - DEF_ASM(tr1) - DEF_ASM(tr2) - DEF_ASM(tr3) - DEF_ASM(tr4) - DEF_ASM(tr5) - DEF_ASM(tr6) - DEF_ASM(tr7) - DEF_ASM(db0) - DEF_ASM(db1) - DEF_ASM(db2) - DEF_ASM(db3) - DEF_ASM(db4) - DEF_ASM(db5) - DEF_ASM(db6) - DEF_ASM(db7) - DEF_ASM(dr0) - DEF_ASM(dr1) - DEF_ASM(dr2) - DEF_ASM(dr3) - DEF_ASM(dr4) - DEF_ASM(dr5) - DEF_ASM(dr6) - DEF_ASM(dr7) - DEF_ASM(es) - DEF_ASM(cs) - DEF_ASM(ss) - DEF_ASM(ds) - DEF_ASM(fs) - DEF_ASM(gs) - DEF_ASM(st) - DEF_ASM(rip) - -#ifdef TCC_TARGET_X86_64 - /* The four low parts of sp/bp/si/di that exist only on - x86-64 (encoding aliased to ah,ch,dh,dh when not using REX). */ - DEF_ASM(spl) - DEF_ASM(bpl) - DEF_ASM(sil) - DEF_ASM(dil) -#endif - /* generic two operands */ - DEF_BWLX(mov) - - DEF_BWLX(add) - DEF_BWLX(or) - DEF_BWLX(adc) - DEF_BWLX(sbb) - DEF_BWLX(and) - DEF_BWLX(sub) - DEF_BWLX(xor) - DEF_BWLX(cmp) - - /* unary ops */ - DEF_BWLX(inc) - DEF_BWLX(dec) - DEF_BWLX(not) - DEF_BWLX(neg) - DEF_BWLX(mul) - DEF_BWLX(imul) - DEF_BWLX(div) - DEF_BWLX(idiv) - - DEF_BWLX(xchg) - DEF_BWLX(test) - - /* shifts */ - DEF_BWLX(rol) - DEF_BWLX(ror) - DEF_BWLX(rcl) - DEF_BWLX(rcr) - DEF_BWLX(shl) - DEF_BWLX(shr) - DEF_BWLX(sar) - - DEF_WLX(shld) - DEF_WLX(shrd) - - DEF_ASM(pushw) - DEF_ASM(pushl) -#ifdef TCC_TARGET_X86_64 - DEF_ASM(pushq) -#endif - DEF_ASM(push) - - DEF_ASM(popw) - DEF_ASM(popl) -#ifdef TCC_TARGET_X86_64 - DEF_ASM(popq) -#endif - DEF_ASM(pop) - - DEF_BWL(in) - DEF_BWL(out) - - DEF_WLX(movzb) - DEF_ASM(movzwl) - DEF_ASM(movsbw) - DEF_ASM(movsbl) - DEF_ASM(movswl) -#ifdef TCC_TARGET_X86_64 - DEF_ASM(movsbq) - DEF_ASM(movswq) - DEF_ASM(movzwq) - DEF_ASM(movslq) -#endif - - DEF_WLX(lea) - - DEF_ASM(les) - DEF_ASM(lds) - DEF_ASM(lss) - DEF_ASM(lfs) - DEF_ASM(lgs) - - DEF_ASM(call) - DEF_ASM(jmp) - DEF_ASM(lcall) - DEF_ASM(ljmp) - - DEF_ASMTEST(j,) - - DEF_ASMTEST(set,) - DEF_ASMTEST(set,b) - DEF_ASMTEST(cmov,) - - DEF_WLX(bsf) - DEF_WLX(bsr) - DEF_WLX(bt) - DEF_WLX(bts) - DEF_WLX(btr) - DEF_WLX(btc) - DEF_WLX(popcnt) - DEF_WLX(tzcnt) - DEF_WLX(lzcnt) - - DEF_WLX(lar) - DEF_WLX(lsl) - - /* generic FP ops */ - DEF_FP(add) - DEF_FP(mul) - - DEF_ASM(fcom) - DEF_ASM(fcom_1) /* non existent op, just to have a regular table */ - DEF_FP1(com) - - DEF_FP(comp) - DEF_FP(sub) - DEF_FP(subr) - DEF_FP(div) - DEF_FP(divr) - - DEF_BWLX(xadd) - DEF_BWLX(cmpxchg) - - /* string ops */ - DEF_BWLX(cmps) - DEF_BWLX(scmp) - DEF_BWL(ins) - DEF_BWL(outs) - DEF_BWLX(lods) - DEF_BWLX(slod) - DEF_BWLX(movs) - DEF_BWLX(smov) - DEF_BWLX(scas) - DEF_BWLX(ssca) - DEF_BWLX(stos) - DEF_BWLX(ssto) - - /* generic asm ops */ -#define ALT(x) -#define DEF_ASM_OP0(name, opcode) DEF_ASM(name) -#define DEF_ASM_OP0L(name, opcode, group, instr_type) -#define DEF_ASM_OP1(name, opcode, group, instr_type, op0) -#define DEF_ASM_OP2(name, opcode, group, instr_type, op0, op1) -#define DEF_ASM_OP3(name, opcode, group, instr_type, op0, op1, op2) -#ifdef TCC_TARGET_X86_64 -# include "x86_64-asm.h" -#else -# include "i386-asm.h" -#endif - -#define ALT(x) -#define DEF_ASM_OP0(name, opcode) -#define DEF_ASM_OP0L(name, opcode, group, instr_type) DEF_ASM(name) -#define DEF_ASM_OP1(name, opcode, group, instr_type, op0) DEF_ASM(name) -#define DEF_ASM_OP2(name, opcode, group, instr_type, op0, op1) DEF_ASM(name) -#define DEF_ASM_OP3(name, opcode, group, instr_type, op0, op1, op2) DEF_ASM(name) -#ifdef TCC_TARGET_X86_64 -# include "x86_64-asm.h" -#else -# include "i386-asm.h" -#endif diff --git a/vendor/tcc/lib/libtcc1.c b/vendor/tcc/lib/libtcc1.c deleted file mode 100644 index ae94af17..00000000 --- a/vendor/tcc/lib/libtcc1.c +++ /dev/null @@ -1,641 +0,0 @@ -/* TCC runtime library. - Parts of this code are (c) 2002 Fabrice Bellard - - Copyright (C) 1987, 1988, 1992, 1994, 1995 Free Software Foundation, Inc. - -This file is free software; you can redistribute it and/or modify it -under the terms of the GNU General Public License as published by the -Free Software Foundation; either version 2, or (at your option) any -later version. - -In addition to the permissions in the GNU General Public License, the -Free Software Foundation gives you unlimited permission to link the -compiled version of this file into combinations with other programs, -and to distribute those combinations without any restriction coming -from the use of this file. (The General Public License restrictions -do apply in other respects; for example, they cover modification of -the file, and distribution when not linked into a combine -executable.) - -This file is distributed in the hope that it will be useful, but -WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -General Public License for more details. - -You should have received a copy of the GNU General Public License -along with this program; see the file COPYING. If not, write to -the Free Software Foundation, 59 Temple Place - Suite 330, -Boston, MA 02111-1307, USA. -*/ - -#define W_TYPE_SIZE 32 -#define BITS_PER_UNIT 8 - -typedef int Wtype; -typedef unsigned int UWtype; -typedef unsigned int USItype; -typedef long long DWtype; -typedef unsigned long long UDWtype; - -struct DWstruct { - Wtype low, high; -}; - -typedef union -{ - struct DWstruct s; - DWtype ll; -} DWunion; - -typedef long double XFtype; -#define WORD_SIZE (sizeof (Wtype) * BITS_PER_UNIT) -#define HIGH_WORD_COEFF (((UDWtype) 1) << WORD_SIZE) - -/* the following deal with IEEE single-precision numbers */ -#define EXCESS 126 -#define SIGNBIT 0x80000000 -#define HIDDEN (1 << 23) -#define SIGN(fp) ((fp) & SIGNBIT) -#define EXP(fp) (((fp) >> 23) & 0xFF) -#define MANT(fp) (((fp) & 0x7FFFFF) | HIDDEN) -#define PACK(s,e,m) ((s) | ((e) << 23) | (m)) - -/* the following deal with IEEE double-precision numbers */ -#define EXCESSD 1022 -#define HIDDEND (1 << 20) -#define EXPD(fp) (((fp.l.upper) >> 20) & 0x7FF) -#define SIGND(fp) ((fp.l.upper) & SIGNBIT) -#define MANTD(fp) (((((fp.l.upper) & 0xFFFFF) | HIDDEND) << 10) | \ - (fp.l.lower >> 22)) -#define HIDDEND_LL ((long long)1 << 52) -#define MANTD_LL(fp) ((fp.ll & (HIDDEND_LL-1)) | HIDDEND_LL) -#define PACKD_LL(s,e,m) (((long long)((s)+((e)<<20))<<32)|(m)) - -/* the following deal with x86 long double-precision numbers */ -#define EXCESSLD 16382 -#define EXPLD(fp) (fp.l.upper & 0x7fff) -#define SIGNLD(fp) ((fp.l.upper) & 0x8000) - -/* only for x86 */ -union ldouble_long { - long double ld; - struct { - unsigned long long lower; - unsigned short upper; - } l; -}; - -union double_long { - double d; -#if 1 - struct { - unsigned int lower; - int upper; - } l; -#else - struct { - int upper; - unsigned int lower; - } l; -#endif - long long ll; -}; - -union float_long { - float f; - unsigned int l; -}; - -/* XXX: we don't support several builtin supports for now */ -#if !defined __x86_64__ && !defined __arm__ - -/* XXX: use gcc/tcc intrinsic ? */ -#if defined __i386__ -#define sub_ddmmss(sh, sl, ah, al, bh, bl) \ - __asm__ ("subl %5,%1\n\tsbbl %3,%0" \ - : "=r" ((USItype) (sh)), \ - "=&r" ((USItype) (sl)) \ - : "0" ((USItype) (ah)), \ - "g" ((USItype) (bh)), \ - "1" ((USItype) (al)), \ - "g" ((USItype) (bl))) -#define umul_ppmm(w1, w0, u, v) \ - __asm__ ("mull %3" \ - : "=a" ((USItype) (w0)), \ - "=d" ((USItype) (w1)) \ - : "%0" ((USItype) (u)), \ - "rm" ((USItype) (v))) -#define udiv_qrnnd(q, r, n1, n0, dv) \ - __asm__ ("divl %4" \ - : "=a" ((USItype) (q)), \ - "=d" ((USItype) (r)) \ - : "0" ((USItype) (n0)), \ - "1" ((USItype) (n1)), \ - "rm" ((USItype) (dv))) -#define count_leading_zeros(count, x) \ - do { \ - USItype __cbtmp; \ - __asm__ ("bsrl %1,%0" \ - : "=r" (__cbtmp) : "rm" ((USItype) (x))); \ - (count) = __cbtmp ^ 31; \ - } while (0) -#else -#error unsupported CPU type -#endif - -/* most of this code is taken from libgcc2.c from gcc */ - -static UDWtype __udivmoddi4 (UDWtype n, UDWtype d, UDWtype *rp) -{ - DWunion ww; - DWunion nn, dd; - DWunion rr; - UWtype d0, d1, n0, n1, n2; - UWtype q0, q1; - UWtype b, bm; - - nn.ll = n; - dd.ll = d; - - d0 = dd.s.low; - d1 = dd.s.high; - n0 = nn.s.low; - n1 = nn.s.high; - -#if !defined(UDIV_NEEDS_NORMALIZATION) - if (d1 == 0) - { - if (d0 > n1) - { - /* 0q = nn / 0D */ - - udiv_qrnnd (q0, n0, n1, n0, d0); - q1 = 0; - - /* Remainder in n0. */ - } - else - { - /* qq = NN / 0d */ - - if (d0 == 0) - d0 = 1 / d0; /* Divide intentionally by zero. */ - - udiv_qrnnd (q1, n1, 0, n1, d0); - udiv_qrnnd (q0, n0, n1, n0, d0); - - /* Remainder in n0. */ - } - - if (rp != 0) - { - rr.s.low = n0; - rr.s.high = 0; - *rp = rr.ll; - } - } - -#else /* UDIV_NEEDS_NORMALIZATION */ - - if (d1 == 0) - { - if (d0 > n1) - { - /* 0q = nn / 0D */ - - count_leading_zeros (bm, d0); - - if (bm != 0) - { - /* Normalize, i.e. make the most significant bit of the - denominator set. */ - - d0 = d0 << bm; - n1 = (n1 << bm) | (n0 >> (W_TYPE_SIZE - bm)); - n0 = n0 << bm; - } - - udiv_qrnnd (q0, n0, n1, n0, d0); - q1 = 0; - - /* Remainder in n0 >> bm. */ - } - else - { - /* qq = NN / 0d */ - - if (d0 == 0) - d0 = 1 / d0; /* Divide intentionally by zero. */ - - count_leading_zeros (bm, d0); - - if (bm == 0) - { - /* From (n1 >= d0) /\ (the most significant bit of d0 is set), - conclude (the most significant bit of n1 is set) /\ (the - leading quotient digit q1 = 1). - - This special case is necessary, not an optimization. - (Shifts counts of W_TYPE_SIZE are undefined.) */ - - n1 -= d0; - q1 = 1; - } - else - { - /* Normalize. */ - - b = W_TYPE_SIZE - bm; - - d0 = d0 << bm; - n2 = n1 >> b; - n1 = (n1 << bm) | (n0 >> b); - n0 = n0 << bm; - - udiv_qrnnd (q1, n1, n2, n1, d0); - } - - /* n1 != d0... */ - - udiv_qrnnd (q0, n0, n1, n0, d0); - - /* Remainder in n0 >> bm. */ - } - - if (rp != 0) - { - rr.s.low = n0 >> bm; - rr.s.high = 0; - *rp = rr.ll; - } - } -#endif /* UDIV_NEEDS_NORMALIZATION */ - - else - { - if (d1 > n1) - { - /* 00 = nn / DD */ - - q0 = 0; - q1 = 0; - - /* Remainder in n1n0. */ - if (rp != 0) - { - rr.s.low = n0; - rr.s.high = n1; - *rp = rr.ll; - } - } - else - { - /* 0q = NN / dd */ - - count_leading_zeros (bm, d1); - if (bm == 0) - { - /* From (n1 >= d1) /\ (the most significant bit of d1 is set), - conclude (the most significant bit of n1 is set) /\ (the - quotient digit q0 = 0 or 1). - - This special case is necessary, not an optimization. */ - - /* The condition on the next line takes advantage of that - n1 >= d1 (true due to program flow). */ - if (n1 > d1 || n0 >= d0) - { - q0 = 1; - sub_ddmmss (n1, n0, n1, n0, d1, d0); - } - else - q0 = 0; - - q1 = 0; - - if (rp != 0) - { - rr.s.low = n0; - rr.s.high = n1; - *rp = rr.ll; - } - } - else - { - UWtype m1, m0; - /* Normalize. */ - - b = W_TYPE_SIZE - bm; - - d1 = (d1 << bm) | (d0 >> b); - d0 = d0 << bm; - n2 = n1 >> b; - n1 = (n1 << bm) | (n0 >> b); - n0 = n0 << bm; - - udiv_qrnnd (q0, n1, n2, n1, d1); - umul_ppmm (m1, m0, q0, d0); - - if (m1 > n1 || (m1 == n1 && m0 > n0)) - { - q0--; - sub_ddmmss (m1, m0, m1, m0, d1, d0); - } - - q1 = 0; - - /* Remainder in (n1n0 - m1m0) >> bm. */ - if (rp != 0) - { - sub_ddmmss (n1, n0, n1, n0, m1, m0); - rr.s.low = (n1 << b) | (n0 >> bm); - rr.s.high = n1 >> bm; - *rp = rr.ll; - } - } - } - } - - ww.s.low = q0; - ww.s.high = q1; - return ww.ll; -} - -#define __negdi2(a) (-(a)) - -long long __divdi3(long long u, long long v) -{ - int c = 0; - DWunion uu, vv; - DWtype w; - - uu.ll = u; - vv.ll = v; - - if (uu.s.high < 0) { - c = ~c; - uu.ll = __negdi2 (uu.ll); - } - if (vv.s.high < 0) { - c = ~c; - vv.ll = __negdi2 (vv.ll); - } - w = __udivmoddi4 (uu.ll, vv.ll, (UDWtype *) 0); - if (c) - w = __negdi2 (w); - return w; -} - -long long __moddi3(long long u, long long v) -{ - int c = 0; - DWunion uu, vv; - DWtype w; - - uu.ll = u; - vv.ll = v; - - if (uu.s.high < 0) { - c = ~c; - uu.ll = __negdi2 (uu.ll); - } - if (vv.s.high < 0) - vv.ll = __negdi2 (vv.ll); - - __udivmoddi4 (uu.ll, vv.ll, (UDWtype *) &w); - if (c) - w = __negdi2 (w); - return w; -} - -unsigned long long __udivdi3(unsigned long long u, unsigned long long v) -{ - return __udivmoddi4 (u, v, (UDWtype *) 0); -} - -unsigned long long __umoddi3(unsigned long long u, unsigned long long v) -{ - UDWtype w; - - __udivmoddi4 (u, v, &w); - return w; -} - -/* XXX: fix tcc's code generator to do this instead */ -long long __ashrdi3(long long a, int b) -{ -#ifdef __TINYC__ - DWunion u; - u.ll = a; - if (b >= 32) { - u.s.low = u.s.high >> (b - 32); - u.s.high = u.s.high >> 31; - } else if (b != 0) { - u.s.low = ((unsigned)u.s.low >> b) | (u.s.high << (32 - b)); - u.s.high = u.s.high >> b; - } - return u.ll; -#else - return a >> b; -#endif -} - -/* XXX: fix tcc's code generator to do this instead */ -unsigned long long __lshrdi3(unsigned long long a, int b) -{ -#ifdef __TINYC__ - DWunion u; - u.ll = a; - if (b >= 32) { - u.s.low = (unsigned)u.s.high >> (b - 32); - u.s.high = 0; - } else if (b != 0) { - u.s.low = ((unsigned)u.s.low >> b) | (u.s.high << (32 - b)); - u.s.high = (unsigned)u.s.high >> b; - } - return u.ll; -#else - return a >> b; -#endif -} - -/* XXX: fix tcc's code generator to do this instead */ -long long __ashldi3(long long a, int b) -{ -#ifdef __TINYC__ - DWunion u; - u.ll = a; - if (b >= 32) { - u.s.high = (unsigned)u.s.low << (b - 32); - u.s.low = 0; - } else if (b != 0) { - u.s.high = ((unsigned)u.s.high << b) | ((unsigned)u.s.low >> (32 - b)); - u.s.low = (unsigned)u.s.low << b; - } - return u.ll; -#else - return a << b; -#endif -} - -#endif /* !__x86_64__ */ - -/* XXX: fix tcc's code generator to do this instead */ -float __floatundisf(unsigned long long a) -{ - DWunion uu; - XFtype r; - - uu.ll = a; - if (uu.s.high >= 0) { - return (float)uu.ll; - } else { - r = (XFtype)uu.ll; - r += 18446744073709551616.0; - return (float)r; - } -} - -double __floatundidf(unsigned long long a) -{ - DWunion uu; - XFtype r; - - uu.ll = a; - if (uu.s.high >= 0) { - return (double)uu.ll; - } else { - r = (XFtype)uu.ll; - r += 18446744073709551616.0; - return (double)r; - } -} - -long double __floatundixf(unsigned long long a) -{ - DWunion uu; - XFtype r; - - uu.ll = a; - if (uu.s.high >= 0) { - return (long double)uu.ll; - } else { - r = (XFtype)uu.ll; - r += 18446744073709551616.0; - return (long double)r; - } -} - -unsigned long long __fixunssfdi (float a1) -{ - register union float_long fl1; - register int exp; - register unsigned long long l; - - fl1.f = a1; - - if (fl1.l == 0) - return (0); - - exp = EXP (fl1.l) - EXCESS - 24; - l = MANT(fl1.l); - - if (exp >= 41) - return 1ULL << 63; - else if (exp >= 0) - l <<= exp; - else if (exp >= -23) - l >>= -exp; - else - return 0; - if (SIGN(fl1.l)) - l = (unsigned long long)-l; - return l; -} - -long long __fixsfdi (float a1) -{ - long long ret; int s; - ret = __fixunssfdi((s = a1 >= 0) ? a1 : -a1); - return s ? ret : -ret; -} - -unsigned long long __fixunsdfdi (double a1) -{ - register union double_long dl1; - register int exp; - register unsigned long long l; - - dl1.d = a1; - - if (dl1.ll == 0) - return (0); - - exp = EXPD (dl1) - EXCESSD - 53; - l = MANTD_LL(dl1); - - if (exp >= 12) - return 1ULL << 63; /* overflow result (like gcc, somewhat) */ - else if (exp >= 0) - l <<= exp; - else if (exp >= -52) - l >>= -exp; - else - return 0; - if (SIGND(dl1)) - l = (unsigned long long)-l; - return l; -} - -long long __fixdfdi (double a1) -{ - long long ret; int s; - ret = __fixunsdfdi((s = a1 >= 0) ? a1 : -a1); - return s ? ret : -ret; -} - -#ifndef __arm__ -unsigned long long __fixunsxfdi (long double a1) -{ - register union ldouble_long dl1; - register int exp; - register unsigned long long l; - - dl1.ld = a1; - - if (dl1.l.lower == 0 && dl1.l.upper == 0) - return (0); - - exp = EXPLD (dl1) - EXCESSLD - 64; - l = dl1.l.lower; - if (exp > 0) - return 1ULL << 63; - if (exp < -63) - return 0; - l >>= -exp; - if (SIGNLD(dl1)) - l = (unsigned long long)-l; - return l; -} - -long long __fixxfdi (long double a1) -{ - long long ret; int s; - ret = __fixunsxfdi((s = a1 >= 0) ? a1 : -a1); - return s ? ret : -ret; -} -#endif /* !ARM */ - -#if defined __x86_64__ -/* float constants used for unary minus operation */ -const float __mzerosf = -0.0; -const double __mzerodf = -0.0; -#endif - -#if defined _WIN64 -/* MSVC x64 intrinsic */ -void __faststorefence(void) -{ - __asm__("lock; orl $0,(%rsp)"); -} -#endif diff --git a/vendor/tcc/libtcc.c b/vendor/tcc/libtcc.c deleted file mode 100644 index 602bb0b2..00000000 --- a/vendor/tcc/libtcc.c +++ /dev/null @@ -1,2253 +0,0 @@ -/* - * TCC - Tiny C Compiler - * - * Copyright (c) 2001-2004 Fabrice Bellard - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 2 of the License, or (at your option) any later version. - * - * This library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public - * License along with this library; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA - */ - -#if !defined ONE_SOURCE || ONE_SOURCE -#include "tccpp.c" -#include "tccgen.c" -#include "tccdbg.c" -#include "tccasm.c" -#include "tccelf.c" -#include "tccrun.c" -#include "x86_64-gen.c" -#include "x86_64-link.c" -#include "i386-asm.c" -#endif /* ONE_SOURCE */ - -#include "tcc.h" - -/********************************************************/ -/* global variables */ - -/* XXX: get rid of this ASAP (or maybe not) */ -ST_DATA struct TCCState *tcc_state; -TCC_SEM(static tcc_compile_sem); -/* an array of pointers to memory to be free'd after errors */ -ST_DATA void** stk_data; -ST_DATA int nb_stk_data; - -/********************************************************/ -#ifdef _WIN32 -ST_FUNC char *normalize_slashes(char *path) -{ - char *p; - for (p = path; *p; ++p) - if (*p == '\\') - *p = '/'; - return path; -} - -#if defined LIBTCC_AS_DLL && !defined CONFIG_TCCDIR -static HMODULE tcc_module; -BOOL WINAPI DllMain (HINSTANCE hDll, DWORD dwReason, LPVOID lpReserved) -{ - if (DLL_PROCESS_ATTACH == dwReason) - tcc_module = hDll; - return TRUE; -} -#else -#define tcc_module NULL /* NULL means executable itself */ -#endif - -#ifndef CONFIG_TCCDIR -/* on win32, we suppose the lib and includes are at the location of 'tcc.exe' */ -static inline char *config_tccdir_w32(char *path) -{ - char *p; - GetModuleFileName(tcc_module, path, MAX_PATH); - p = tcc_basename(normalize_slashes(strlwr(path))); - if (p > path) - --p; - *p = 0; - return path; -} -#define CONFIG_TCCDIR config_tccdir_w32(alloca(MAX_PATH)) -#endif - -#ifdef TCC_TARGET_PE -static void tcc_add_systemdir(TCCState *s) -{ - char buf[1000]; - GetSystemDirectory(buf, sizeof buf); - tcc_add_library_path(s, normalize_slashes(buf)); -} -#endif -#endif - -/********************************************************/ -#if CONFIG_TCC_SEMLOCK -#if defined _WIN32 -ST_FUNC void wait_sem(TCCSem *p) -{ - if (!p->init) - InitializeCriticalSection(&p->cr), p->init = 1; - EnterCriticalSection(&p->cr); -} -ST_FUNC void post_sem(TCCSem *p) -{ - LeaveCriticalSection(&p->cr); -} -#elif defined __APPLE__ -/* Half-compatible MacOS doesn't have non-shared (process local) - semaphores. Use the dispatch framework for lightweight locks. */ -ST_FUNC void wait_sem(TCCSem *p) -{ - if (!p->init) - p->sem = dispatch_semaphore_create(1), p->init = 1; - dispatch_semaphore_wait(p->sem, DISPATCH_TIME_FOREVER); -} -ST_FUNC void post_sem(TCCSem *p) -{ - dispatch_semaphore_signal(p->sem); -} -#else -ST_FUNC void wait_sem(TCCSem *p) -{ - if (!p->init) - sem_init(&p->sem, 0, 1), p->init = 1; - while (sem_wait(&p->sem) < 0 && errno == EINTR); -} -ST_FUNC void post_sem(TCCSem *p) -{ - sem_post(&p->sem); -} -#endif -#endif - -PUB_FUNC void tcc_enter_state(TCCState *s1) -{ - if (s1->error_set_jmp_enabled) - return; - WAIT_SEM(&tcc_compile_sem); - tcc_state = s1; -} - -PUB_FUNC void tcc_exit_state(TCCState *s1) -{ - if (s1->error_set_jmp_enabled) - return; - tcc_state = NULL; - POST_SEM(&tcc_compile_sem); -} - -/********************************************************/ -/* copy a string and truncate it. */ -ST_FUNC char *pstrcpy(char *buf, size_t buf_size, const char *s) -{ - char *q, *q_end; - int c; - - if (buf_size > 0) { - q = buf; - q_end = buf + buf_size - 1; - while (q < q_end) { - c = *s++; - if (c == '\0') - break; - *q++ = c; - } - *q = '\0'; - } - return buf; -} - -/* strcat and truncate. */ -ST_FUNC char *pstrcat(char *buf, size_t buf_size, const char *s) -{ - size_t len; - len = strlen(buf); - if (len < buf_size) - pstrcpy(buf + len, buf_size - len, s); - return buf; -} - -ST_FUNC char *pstrncpy(char *out, const char *in, size_t num) -{ - memcpy(out, in, num); - out[num] = '\0'; - return out; -} - -/* extract the basename of a file */ -PUB_FUNC char *tcc_basename(const char *name) -{ - char *p = strchr(name, 0); - while (p > name && !IS_DIRSEP(p[-1])) - --p; - return p; -} - -/* extract extension part of a file - * - * (if no extension, return pointer to end-of-string) - */ -PUB_FUNC char *tcc_fileextension (const char *name) -{ - char *b = tcc_basename(name); - char *e = strrchr(b, '.'); - return e ? e : strchr(b, 0); -} - -ST_FUNC char *tcc_load_text(int fd) -{ - int len = lseek(fd, 0, SEEK_END); - char *buf = load_data(fd, 0, len + 1); - buf[len] = 0; - return buf; -} - -/********************************************************/ -/* memory management */ - -#undef free -#undef malloc -#undef realloc - -void mem_error(const char *msg) -{ - fprintf(stderr, "%s\n", msg); - exit (1); -} - -#ifndef MEM_DEBUG - -PUB_FUNC void tcc_free(void *ptr) -{ - free(ptr); -} - -PUB_FUNC void *tcc_malloc(unsigned long size) -{ - void *ptr; - ptr = malloc(size ? size : 1); - if (!ptr) - mem_error("memory full (malloc)"); - return ptr; -} - -PUB_FUNC void *tcc_mallocz(unsigned long size) -{ - void *ptr; - ptr = tcc_malloc(size); - if (size) - memset(ptr, 0, size); - return ptr; -} - -PUB_FUNC void *tcc_realloc(void *ptr, unsigned long size) -{ - void *ptr1; - if (size == 0) { - free(ptr); - ptr1 = NULL; - } - else { - ptr1 = realloc(ptr, size); - if (!ptr1) - mem_error("memory full (realloc)"); - } - return ptr1; -} - -PUB_FUNC char *tcc_strdup(const char *str) -{ - char *ptr; - ptr = tcc_malloc(strlen(str) + 1); - strcpy(ptr, str); - return ptr; -} - -#else - -#define MEM_DEBUG_MAGIC1 0xFEEDDEB1 -#define MEM_DEBUG_MAGIC2 0xFEEDDEB2 -#define MEM_DEBUG_MAGIC3 0xFEEDDEB3 -#define MEM_DEBUG_FILE_LEN 40 -#define MEM_DEBUG_CHECK3(header) \ - ((mem_debug_header_t*)((char*)header + header->size))->magic3 -#define MEM_USER_PTR(header) \ - ((char *)header + offsetof(mem_debug_header_t, magic3)) -#define MEM_HEADER_PTR(ptr) \ - (mem_debug_header_t *)((char*)ptr - offsetof(mem_debug_header_t, magic3)) - -struct mem_debug_header { - unsigned magic1; - unsigned size; - struct mem_debug_header *prev; - struct mem_debug_header *next; - int line_num; - char file_name[MEM_DEBUG_FILE_LEN + 1]; - unsigned magic2; - ALIGNED(16) unsigned char magic3[4]; -}; - -typedef struct mem_debug_header mem_debug_header_t; - -TCC_SEM(static mem_sem); -static mem_debug_header_t *mem_debug_chain; -static unsigned mem_cur_size; -static unsigned mem_max_size; -static int nb_states; - -static mem_debug_header_t *malloc_check(void *ptr, const char *msg) -{ - mem_debug_header_t * header = MEM_HEADER_PTR(ptr); - if (header->magic1 != MEM_DEBUG_MAGIC1 || - header->magic2 != MEM_DEBUG_MAGIC2 || - read32le(MEM_DEBUG_CHECK3(header)) != MEM_DEBUG_MAGIC3 || - header->size == (unsigned)-1) { - fprintf(stderr, "%s check failed\n", msg); - if (header->magic1 == MEM_DEBUG_MAGIC1) - fprintf(stderr, "%s:%u: block allocated here.\n", - header->file_name, header->line_num); - exit(1); - } - return header; -} - -PUB_FUNC void *tcc_malloc_debug(unsigned long size, const char *file, int line) -{ - int ofs; - mem_debug_header_t *header; - - header = malloc(sizeof(mem_debug_header_t) + size); - if (!header) - mem_error("memory full (malloc)"); - - header->magic1 = MEM_DEBUG_MAGIC1; - header->magic2 = MEM_DEBUG_MAGIC2; - header->size = size; - write32le(MEM_DEBUG_CHECK3(header), MEM_DEBUG_MAGIC3); - header->line_num = line; - ofs = strlen(file) - MEM_DEBUG_FILE_LEN; - strncpy(header->file_name, file + (ofs > 0 ? ofs : 0), MEM_DEBUG_FILE_LEN); - header->file_name[MEM_DEBUG_FILE_LEN] = 0; - - WAIT_SEM(&mem_sem); - header->next = mem_debug_chain; - header->prev = NULL; - if (header->next) - header->next->prev = header; - mem_debug_chain = header; - mem_cur_size += size; - if (mem_cur_size > mem_max_size) - mem_max_size = mem_cur_size; - POST_SEM(&mem_sem); - - return MEM_USER_PTR(header); -} - -PUB_FUNC void tcc_free_debug(void *ptr) -{ - mem_debug_header_t *header; - if (!ptr) - return; - header = malloc_check(ptr, "tcc_free"); - - WAIT_SEM(&mem_sem); - mem_cur_size -= header->size; - header->size = (unsigned)-1; - if (header->next) - header->next->prev = header->prev; - if (header->prev) - header->prev->next = header->next; - if (header == mem_debug_chain) - mem_debug_chain = header->next; - POST_SEM(&mem_sem); - free(header); -} - -PUB_FUNC void *tcc_mallocz_debug(unsigned long size, const char *file, int line) -{ - void *ptr; - ptr = tcc_malloc_debug(size,file,line); - memset(ptr, 0, size); - return ptr; -} - -PUB_FUNC void *tcc_realloc_debug(void *ptr, unsigned long size, const char *file, int line) -{ - mem_debug_header_t *header; - int mem_debug_chain_update = 0; - if (!ptr) - return tcc_malloc_debug(size, file, line); - header = malloc_check(ptr, "tcc_realloc"); - - WAIT_SEM(&mem_sem); - mem_cur_size -= header->size; - mem_debug_chain_update = (header == mem_debug_chain); - header = realloc(header, sizeof(mem_debug_header_t) + size); - if (!header) - mem_error("memory full (realloc)"); - header->size = size; - write32le(MEM_DEBUG_CHECK3(header), MEM_DEBUG_MAGIC3); - if (header->next) - header->next->prev = header; - if (header->prev) - header->prev->next = header; - if (mem_debug_chain_update) - mem_debug_chain = header; - mem_cur_size += size; - if (mem_cur_size > mem_max_size) - mem_max_size = mem_cur_size; - POST_SEM(&mem_sem); - - return MEM_USER_PTR(header); -} - -PUB_FUNC char *tcc_strdup_debug(const char *str, const char *file, int line) -{ - char *ptr; - ptr = tcc_malloc_debug(strlen(str) + 1, file, line); - strcpy(ptr, str); - return ptr; -} - -PUB_FUNC void tcc_memcheck(int d) -{ - WAIT_SEM(&mem_sem); - nb_states += d; - if (0 == nb_states && mem_cur_size) { - mem_debug_header_t *header = mem_debug_chain; - fflush(stdout); - fprintf(stderr, "MEM_DEBUG: mem_leak= %d bytes, mem_max_size= %d bytes\n", - mem_cur_size, mem_max_size); - while (header) { - fprintf(stderr, "%s:%u: error: %u bytes leaked\n", - header->file_name, header->line_num, header->size); - header = header->next; - } - fflush(stderr); - mem_cur_size = 0; - mem_debug_chain = NULL; -#if MEM_DEBUG-0 == 2 - exit(2); -#endif - } - POST_SEM(&mem_sem); -} - -#endif /* MEM_DEBUG */ - -#ifdef _WIN32 -# define realpath(file, buf) _fullpath(buf, file, 260) -#endif - -/* for #pragma once */ -ST_FUNC int normalized_PATHCMP(const char *f1, const char *f2) -{ - char *p1, *p2; - int ret = 1; - if (!!(p1 = realpath(f1, NULL))) { - if (!!(p2 = realpath(f2, NULL))) { - ret = PATHCMP(p1, p2); - free(p2); /* using original free */ - } - free(p1); - } - return ret; -} - -#define free(p) use_tcc_free(p) -#define malloc(s) use_tcc_malloc(s) -#define realloc(p, s) use_tcc_realloc(p, s) - -/********************************************************/ -/* dynarrays */ - -ST_FUNC void dynarray_add(void *ptab, int *nb_ptr, void *data) -{ - int nb, nb_alloc; - void **pp; - - nb = *nb_ptr; - pp = *(void ***)ptab; - /* every power of two we double array size */ - if ((nb & (nb - 1)) == 0) { - if (!nb) - nb_alloc = 1; - else - nb_alloc = nb * 2; - pp = tcc_realloc(pp, nb_alloc * sizeof(void *)); - *(void***)ptab = pp; - } - pp[nb++] = data; - *nb_ptr = nb; -} - -ST_FUNC void dynarray_reset(void *pp, int *n) -{ - void **p; - for (p = *(void***)pp; *n; ++p, --*n) - if (*p) - tcc_free(*p); - tcc_free(*(void**)pp); - *(void**)pp = NULL; -} - -static void tcc_split_path(TCCState *s, void *p_ary, int *p_nb_ary, const char *in) -{ - const char *p; - do { - int c; - CString str; - - cstr_new(&str); - for (p = in; c = *p, c != '\0' && c != PATHSEP[0]; ++p) { - if (c == '{' && p[1] && p[2] == '}') { - c = p[1], p += 2; - if (c == 'B') - cstr_cat(&str, s->tcc_lib_path, -1); - if (c == 'R') - cstr_cat(&str, CONFIG_SYSROOT, -1); - if (c == 'f' && file) { - /* substitute current file's dir */ - const char *f = file->true_filename; - const char *b = tcc_basename(f); - if (b > f) - cstr_cat(&str, f, b - f - 1); - else - cstr_cat(&str, ".", 1); - } - } else { - cstr_ccat(&str, c); - } - } - if (str.size) { - cstr_ccat(&str, '\0'); - dynarray_add(p_ary, p_nb_ary, tcc_strdup(str.data)); - } - cstr_free(&str); - in = p+1; - } while (*p); -} - -/********************************************************/ -/* warning / error */ - -/* warn_... option bits */ -#define WARN_ON 1 /* warning is on (-Woption) */ -#define WARN_ERR 2 /* warning is an error (-Werror=option) */ -#define WARN_NOE 4 /* warning is not an error (-Wno-error=option) */ - -/* error1() modes */ -enum { ERROR_WARN, ERROR_NOABORT, ERROR_ERROR }; - -static void error1(int mode, const char *fmt, va_list ap) -{ - BufferedFile **pf, *f; - TCCState *s1 = tcc_state; - CString cs; - - tcc_exit_state(s1); - - if (mode == ERROR_WARN) { - if (s1->warn_error) - mode = ERROR_ERROR; - if (s1->warn_num) { - /* handle tcc_warning_c(warn_option)(fmt, ...) */ - int wopt = *(&s1->warn_none + s1->warn_num); - s1->warn_num = 0; - if (0 == (wopt & WARN_ON)) - return; - if (wopt & WARN_ERR) - mode = ERROR_ERROR; - if (wopt & WARN_NOE) - mode = ERROR_WARN; - } - if (s1->warn_none) - return; - } - - cstr_new(&cs); - f = NULL; - if (s1->error_set_jmp_enabled) { /* we're called while parsing a file */ - /* use upper file if inline ":asm:" or token ":paste:" */ - for (f = file; f && f->filename[0] == ':'; f = f->prev) - ; - } - if (f) { - for(pf = s1->include_stack; pf < s1->include_stack_ptr; pf++) - cstr_printf(&cs, "In file included from %s:%d:\n", - (*pf)->filename, (*pf)->line_num - 1); - cstr_printf(&cs, "%s:%d: ", - f->filename, f->line_num - !!(tok_flags & TOK_FLAG_BOL)); - } else if (s1->current_filename) { - cstr_printf(&cs, "%s: ", s1->current_filename); - } - if (0 == cs.size) - cstr_printf(&cs, "tcc: "); - cstr_printf(&cs, mode == ERROR_WARN ? "warning: " : "error: "); - cstr_vprintf(&cs, fmt, ap); - if (!s1 || !s1->error_func) { - /* default case: stderr */ - if (s1 && s1->output_type == TCC_OUTPUT_PREPROCESS && s1->ppfp == stdout) - printf("\n"); /* print a newline during tcc -E */ - fflush(stdout); /* flush -v output */ - fprintf(stderr, "%s\n", (char*)cs.data); - fflush(stderr); /* print error/warning now (win32) */ - } else { - s1->error_func(s1->error_opaque, (char*)cs.data); - } - cstr_free(&cs); - if (mode != ERROR_WARN) - s1->nb_errors++; - if (mode == ERROR_ERROR && s1->error_set_jmp_enabled) { - while (nb_stk_data) - tcc_free(*(void**)stk_data[--nb_stk_data]); - longjmp(s1->error_jmp_buf, 1); - } -} - -LIBTCCAPI void tcc_set_error_func(TCCState *s, void *error_opaque, TCCErrorFunc error_func) -{ - s->error_opaque = error_opaque; - s->error_func = error_func; -} - -LIBTCCAPI TCCErrorFunc tcc_get_error_func(TCCState *s) -{ - return s->error_func; -} - -LIBTCCAPI void *tcc_get_error_opaque(TCCState *s) -{ - return s->error_opaque; -} - -/* error without aborting current compilation */ -PUB_FUNC int _tcc_error_noabort(const char *fmt, ...) -{ - va_list ap; - va_start(ap, fmt); - error1(ERROR_NOABORT, fmt, ap); - va_end(ap); - return -1; -} - -#undef _tcc_error -PUB_FUNC void _tcc_error(const char *fmt, ...) -{ - va_list ap; - va_start(ap, fmt); - error1(ERROR_ERROR, fmt, ap); - exit(1); -} -#define _tcc_error use_tcc_error_noabort - -PUB_FUNC void _tcc_warning(const char *fmt, ...) -{ - va_list ap; - va_start(ap, fmt); - error1(ERROR_WARN, fmt, ap); - va_end(ap); -} - - -/********************************************************/ -/* I/O layer */ - -ST_FUNC void tcc_open_bf(TCCState *s1, const char *filename, int initlen) -{ - BufferedFile *bf; - int buflen = initlen ? initlen : IO_BUF_SIZE; - - bf = tcc_mallocz(sizeof(BufferedFile) + buflen); - bf->buf_ptr = bf->buffer; - bf->buf_end = bf->buffer + initlen; - bf->buf_end[0] = CH_EOB; /* put eob symbol */ - pstrcpy(bf->filename, sizeof(bf->filename), filename); -#ifdef _WIN32 - normalize_slashes(bf->filename); -#endif - bf->true_filename = bf->filename; - bf->line_num = 1; - bf->ifdef_stack_ptr = s1->ifdef_stack_ptr; - bf->fd = -1; - bf->prev = file; - file = bf; - tok_flags = TOK_FLAG_BOL | TOK_FLAG_BOF; -} - -ST_FUNC void tcc_close(void) -{ - TCCState *s1 = tcc_state; - BufferedFile *bf = file; - if (bf->fd > 0) { - close(bf->fd); - total_lines += bf->line_num - 1; - } - if (bf->true_filename != bf->filename) - tcc_free(bf->true_filename); - file = bf->prev; - tcc_free(bf); -} - -static int _tcc_open(TCCState *s1, const char *filename) -{ - int fd; - if (strcmp(filename, "-") == 0) - fd = 0, filename = ""; - else - fd = open(filename, O_RDONLY | O_BINARY); - if ((s1->verbose == 2 && fd >= 0) || s1->verbose == 3) - printf("%s %*s%s\n", fd < 0 ? "nf":"->", - (int)(s1->include_stack_ptr - s1->include_stack), "", filename); - return fd; -} - -ST_FUNC int tcc_open(TCCState *s1, const char *filename) -{ - int fd = _tcc_open(s1, filename); - if (fd < 0) - return -1; - tcc_open_bf(s1, filename, 0); - file->fd = fd; - return 0; -} - -/* compile the file opened in 'file'. Return non zero if errors. */ -static int tcc_compile(TCCState *s1, int filetype, const char *str, int fd) -{ - /* Here we enter the code section where we use the global variables for - parsing and code generation (tccpp.c, tccgen.c, -gen.c). - Other threads need to wait until we're done. - - Alternatively we could use thread local storage for those global - variables, which may or may not have advantages */ - - tcc_enter_state(s1); - s1->error_set_jmp_enabled = 1; - - if (setjmp(s1->error_jmp_buf) == 0) { - s1->nb_errors = 0; - - if (fd == -1) { - int len = strlen(str); - tcc_open_bf(s1, "", len); - memcpy(file->buffer, str, len); - } else { - tcc_open_bf(s1, str, 0); - file->fd = fd; - } - - preprocess_start(s1, filetype); - tccgen_init(s1); - - if (s1->output_type == TCC_OUTPUT_PREPROCESS) { - tcc_preprocess(s1); - } else { - tccelf_begin_file(s1); - if (filetype & (AFF_TYPE_ASM | AFF_TYPE_ASMPP)) { - tcc_assemble(s1, !!(filetype & AFF_TYPE_ASMPP)); - } else { - tccgen_compile(s1); - } - tccelf_end_file(s1); - } - } - tccgen_finish(s1); - preprocess_end(s1); - s1->error_set_jmp_enabled = 0; - tcc_exit_state(s1); - return s1->nb_errors != 0 ? -1 : 0; -} - -LIBTCCAPI int tcc_compile_string(TCCState *s, const char *str) -{ - return tcc_compile(s, s->filetype, str, -1); -} - -/* define a preprocessor symbol. value can be NULL, sym can be "sym=val" */ -LIBTCCAPI void tcc_define_symbol(TCCState *s1, const char *sym, const char *value) -{ - const char *eq; - if (NULL == (eq = strchr(sym, '='))) - eq = strchr(sym, 0); - if (NULL == value) - value = *eq ? eq + 1 : "1"; - cstr_printf(&s1->cmdline_defs, "#define %.*s %s\n", (int)(eq-sym), sym, value); -} - -/* undefine a preprocessor symbol */ -LIBTCCAPI void tcc_undefine_symbol(TCCState *s1, const char *sym) -{ - cstr_printf(&s1->cmdline_defs, "#undef %s\n", sym); -} - - -LIBTCCAPI TCCState *tcc_new(void) -{ - TCCState *s; - - s = tcc_mallocz(sizeof(TCCState)); - if (!s) - return NULL; -#ifdef MEM_DEBUG - tcc_memcheck(1); -#endif - -#undef gnu_ext - - s->gnu_ext = 1; - s->tcc_ext = 1; - s->nocommon = 1; - s->dollars_in_identifiers = 1; /*on by default like in gcc/clang*/ - s->cversion = 199901; /* default unless -std=c11 is supplied */ - s->warn_implicit_function_declaration = 1; - s->warn_discarded_qualifiers = 1; - s->ms_extensions = 1; - -#ifdef CHAR_IS_UNSIGNED - s->char_is_unsigned = 1; -#endif -#ifdef TCC_TARGET_I386 - s->seg_size = 32; -#endif - /* enable this if you want symbols with leading underscore on windows: */ -#if defined TCC_TARGET_MACHO /* || defined TCC_TARGET_PE */ - s->leading_underscore = 1; -#endif -#ifdef TCC_TARGET_ARM - s->float_abi = ARM_FLOAT_ABI; -#endif -#ifdef CONFIG_NEW_DTAGS - s->enable_new_dtags = 1; -#endif - s->ppfp = stdout; - /* might be used in error() before preprocess_start() */ - s->include_stack_ptr = s->include_stack; - - tcc_set_lib_path(s, CONFIG_TCCDIR); - return s; -} - -LIBTCCAPI void tcc_delete(TCCState *s1) -{ - /* free sections */ - tccelf_delete(s1); - - /* free library paths */ - dynarray_reset(&s1->library_paths, &s1->nb_library_paths); - dynarray_reset(&s1->crt_paths, &s1->nb_crt_paths); - - /* free include paths */ - dynarray_reset(&s1->include_paths, &s1->nb_include_paths); - dynarray_reset(&s1->sysinclude_paths, &s1->nb_sysinclude_paths); - - tcc_free(s1->tcc_lib_path); - tcc_free(s1->soname); - tcc_free(s1->rpath); - tcc_free(s1->elf_entryname); - tcc_free(s1->init_symbol); - tcc_free(s1->fini_symbol); - tcc_free(s1->mapfile); - tcc_free(s1->outfile); - tcc_free(s1->deps_outfile); -#if defined TCC_TARGET_MACHO - tcc_free(s1->install_name); -#endif - dynarray_reset(&s1->files, &s1->nb_files); - dynarray_reset(&s1->target_deps, &s1->nb_target_deps); - dynarray_reset(&s1->pragma_libs, &s1->nb_pragma_libs); - dynarray_reset(&s1->argv, &s1->argc); - cstr_free(&s1->cmdline_defs); - cstr_free(&s1->cmdline_incl); - cstr_free(&s1->linker_arg); -#ifdef TCC_IS_NATIVE - /* free runtime memory */ - tcc_run_free(s1); -#endif - tcc_free(s1->dState); - tcc_free(s1); -#ifdef MEM_DEBUG - tcc_memcheck(-1); -#endif -} - -LIBTCCAPI int tcc_set_output_type(TCCState *s, int output_type) -{ -#ifdef CONFIG_TCC_PIE - if (output_type == TCC_OUTPUT_EXE) - output_type |= TCC_OUTPUT_DYN; -#endif - s->output_type = output_type; - - if (!s->nostdinc) { - /* default include paths */ - /* -isystem paths have already been handled */ - tcc_add_sysinclude_path(s, CONFIG_TCC_SYSINCLUDEPATHS); - } - - if (output_type == TCC_OUTPUT_PREPROCESS) { - s->do_debug = 0; - return 0; - } - - tccelf_new(s); - if (s->do_debug) { - /* add debug sections */ - tcc_debug_new(s); - } -#ifdef CONFIG_TCC_BCHECK - if (s->do_bounds_check) { - /* if bound checking, then add corresponding sections */ - tccelf_bounds_new(s); - } -#endif - - if (output_type == TCC_OUTPUT_OBJ) { - /* always elf for objects */ - s->output_format = TCC_OUTPUT_FORMAT_ELF; - return 0; - } - - tcc_add_library_path(s, CONFIG_TCC_LIBPATHS); - -#ifdef TCC_TARGET_PE -# ifdef _WIN32 - /* allow linking with system dll's directly */ - tcc_add_systemdir(s); -# endif - /* target PE has its own startup code in libtcc1.a */ - return 0; - -#elif defined TCC_TARGET_MACHO -# ifdef TCC_IS_NATIVE - tcc_add_macos_sdkpath(s); -# endif - /* Mach-O with LC_MAIN doesn't need any crt startup code. */ - return 0; - -#else - /* paths for crt objects */ - tcc_split_path(s, &s->crt_paths, &s->nb_crt_paths, CONFIG_TCC_CRTPREFIX); - - /* add libc crt1/crti objects */ - if (output_type != TCC_OUTPUT_MEMORY && !s->nostdlib) { -#if TARGETOS_OpenBSD - if (output_type != TCC_OUTPUT_DLL) - tcc_add_crt(s, "crt0.o"); - if (output_type == TCC_OUTPUT_DLL) - tcc_add_crt(s, "crtbeginS.o"); - else - tcc_add_crt(s, "crtbegin.o"); -#elif TARGETOS_FreeBSD - if (output_type != TCC_OUTPUT_DLL) - tcc_add_crt(s, "crt1.o"); - tcc_add_crt(s, "crti.o"); - if (s->static_link) - tcc_add_crt(s, "crtbeginT.o"); - else if (output_type & TCC_OUTPUT_DYN) - tcc_add_crt(s, "crtbeginS.o"); - else - tcc_add_crt(s, "crtbegin.o"); -#elif TARGETOS_NetBSD - if (output_type != TCC_OUTPUT_DLL) - tcc_add_crt(s, "crt0.o"); - tcc_add_crt(s, "crti.o"); - if (s->static_link) - tcc_add_crt(s, "crtbeginT.o"); - else if (output_type & TCC_OUTPUT_DYN) - tcc_add_crt(s, "crtbeginS.o"); - else - tcc_add_crt(s, "crtbegin.o"); -#elif defined TARGETOS_ANDROID - if (output_type != TCC_OUTPUT_DLL) - tcc_add_crt(s, "crtbegin_dynamic.o"); - else - tcc_add_crt(s, "crtbegin_so.o"); -#else - if (output_type != TCC_OUTPUT_DLL) - tcc_add_crt(s, "crt1.o"); - tcc_add_crt(s, "crti.o"); -#endif - } - return 0; -#endif -} - -LIBTCCAPI int tcc_add_include_path(TCCState *s, const char *pathname) -{ - tcc_split_path(s, &s->include_paths, &s->nb_include_paths, pathname); - return 0; -} - -LIBTCCAPI int tcc_add_sysinclude_path(TCCState *s, const char *pathname) -{ - tcc_split_path(s, &s->sysinclude_paths, &s->nb_sysinclude_paths, pathname); - return 0; -} - -/* add/update a 'DLLReference', Just find if level == -1 */ -ST_FUNC DLLReference *tcc_add_dllref(TCCState *s1, const char *dllname, int level) -{ - DLLReference *ref = NULL; - int i; - for (i = 0; i < s1->nb_loaded_dlls; i++) - if (0 == strcmp(s1->loaded_dlls[i]->name, dllname)) { - ref = s1->loaded_dlls[i]; - break; - } - if (level == -1) - return ref; - if (ref) { - if (level < ref->level) - ref->level = level; - ref->found = 1; - return ref; - } - ref = tcc_mallocz(sizeof(DLLReference) + strlen(dllname)); - strcpy(ref->name, dllname); - dynarray_add(&s1->loaded_dlls, &s1->nb_loaded_dlls, ref); - ref->level = level; - ref->index = s1->nb_loaded_dlls; - return ref; -} - -/* OpenBSD: choose latest from libxxx.so.x.y versions */ -#if defined TARGETOS_OpenBSD && !defined _WIN32 -#include -static int tcc_glob_so(TCCState *s1, const char *pattern, char *buf, int size) -{ - const char *star; - glob_t g; - char *p; - int i, v, v1, v2, v3; - - star = strchr(pattern, '*'); - if (!star || glob(pattern, 0, NULL, &g)) - return -1; - for (v = -1, i = 0; i < g.gl_pathc; ++i) { - p = g.gl_pathv[i]; - if (2 != sscanf(p + (star - pattern), "%d.%d.%d", &v1, &v2, &v3)) - continue; - if ((v1 = v1 * 1000 + v2) > v) - v = v1, pstrcpy(buf, size, p); - } - globfree(&g); - return v; -} -#endif - -ST_FUNC int tcc_add_file_internal(TCCState *s1, const char *filename, int flags) -{ - int fd, ret = -1; - -#if defined TARGETOS_OpenBSD && !defined _WIN32 - char buf[1024]; - if (tcc_glob_so(s1, filename, buf, sizeof buf) >= 0) - filename = buf; -#endif - - /* ignore binary files with -E */ - if (s1->output_type == TCC_OUTPUT_PREPROCESS - && (flags & AFF_TYPE_BIN)) - return 0; - - /* open the file */ - fd = _tcc_open(s1, filename); - if (fd < 0) { - if (flags & AFF_PRINT_ERROR) - tcc_error_noabort("file '%s' not found", filename); - return ret; - } - - s1->current_filename = filename; - if (flags & AFF_TYPE_BIN) { - ElfW(Ehdr) ehdr; - int obj_type; - - obj_type = tcc_object_type(fd, &ehdr); - lseek(fd, 0, SEEK_SET); - - switch (obj_type) { - - case AFF_BINTYPE_REL: - ret = tcc_load_object_file(s1, fd, 0); - break; - - case AFF_BINTYPE_AR: - ret = tcc_load_archive(s1, fd, !(flags & AFF_WHOLE_ARCHIVE)); - break; - -#ifdef TCC_TARGET_PE - default: - ret = pe_load_file(s1, fd, filename); - goto check_success; - -#elif defined TCC_TARGET_MACHO - case AFF_BINTYPE_DYN: - case_dyn_or_tbd: - if (s1->output_type == TCC_OUTPUT_MEMORY) { -#ifdef TCC_IS_NATIVE - void* dl; - const char* soname = filename; - if (obj_type != AFF_BINTYPE_DYN) - soname = macho_tbd_soname(filename); - dl = dlopen(soname, RTLD_GLOBAL | RTLD_LAZY); - if (dl) - tcc_add_dllref(s1, soname, 0)->handle = dl, ret = 0; - if (filename != soname) - tcc_free((void *)soname); -#endif - } else if (obj_type == AFF_BINTYPE_DYN) { - ret = macho_load_dll(s1, fd, filename, (flags & AFF_REFERENCED_DLL) != 0); - } else { - ret = macho_load_tbd(s1, fd, filename, (flags & AFF_REFERENCED_DLL) != 0); - } - goto check_success; - default: - { - const char *ext = tcc_fileextension(filename); - if (!strcmp(ext, ".tbd")) - goto case_dyn_or_tbd; - if (!strcmp(ext, ".dylib")) { - obj_type = AFF_BINTYPE_DYN; - goto case_dyn_or_tbd; - } - goto check_success; - } - -#else /* unix */ - case AFF_BINTYPE_DYN: - if (s1->output_type == TCC_OUTPUT_MEMORY) { -#ifdef TCC_IS_NATIVE - void* dl = dlopen(filename, RTLD_GLOBAL | RTLD_LAZY); - if (dl) - tcc_add_dllref(s1, filename, 0)->handle = dl, ret = 0; -#endif - } else - ret = tcc_load_dll(s1, fd, filename, (flags & AFF_REFERENCED_DLL) != 0); - break; - - default: - /* as GNU ld, consider it is an ld script if not recognized */ - ret = tcc_load_ldscript(s1, fd); - goto check_success; - -#endif /* pe / macos / unix */ - -check_success: - if (ret < 0) - tcc_error_noabort("%s: unrecognized file type", filename); - break; - -#ifdef TCC_TARGET_COFF - case AFF_BINTYPE_C67: - ret = tcc_load_coff(s1, fd); - break; -#endif - } - close(fd); - } else { - /* update target deps */ - dynarray_add(&s1->target_deps, &s1->nb_target_deps, tcc_strdup(filename)); - ret = tcc_compile(s1, flags, filename, fd); - } - s1->current_filename = NULL; - return ret; -} - -LIBTCCAPI int tcc_add_file(TCCState *s, const char *filename) -{ - int filetype = s->filetype; - if (0 == (filetype & AFF_TYPE_MASK)) { - /* use a file extension to detect a filetype */ - const char *ext = tcc_fileextension(filename); - if (ext[0]) { - ext++; - if (!strcmp(ext, "S")) - filetype = AFF_TYPE_ASMPP; - else if (!strcmp(ext, "s")) - filetype = AFF_TYPE_ASM; - else if (!PATHCMP(ext, "c") - || !PATHCMP(ext, "h") - || !PATHCMP(ext, "i")) - filetype = AFF_TYPE_C; - else - filetype |= AFF_TYPE_BIN; - } else { - filetype = AFF_TYPE_C; - } - } - return tcc_add_file_internal(s, filename, filetype | AFF_PRINT_ERROR); -} - -LIBTCCAPI int tcc_add_library_path(TCCState *s, const char *pathname) -{ - tcc_split_path(s, &s->library_paths, &s->nb_library_paths, pathname); - return 0; -} - -static int tcc_add_library_internal(TCCState *s, const char *fmt, - const char *filename, int flags, char **paths, int nb_paths) -{ - char buf[1024]; - int i; - - for(i = 0; i < nb_paths; i++) { - snprintf(buf, sizeof(buf), fmt, paths[i], filename); - if (tcc_add_file_internal(s, buf, flags | AFF_TYPE_BIN) == 0) - return 0; - } - return -1; -} - -/* find and load a dll. Return non zero if not found */ -ST_FUNC int tcc_add_dll(TCCState *s, const char *filename, int flags) -{ - return tcc_add_library_internal(s, "%s/%s", filename, flags, - s->library_paths, s->nb_library_paths); -} - -/* find [cross-]libtcc1.a and tcc helper objects in library path */ -ST_FUNC void tcc_add_support(TCCState *s1, const char *filename) -{ - char buf[100]; - if (CONFIG_TCC_CROSSPREFIX[0]) - filename = strcat(strcpy(buf, CONFIG_TCC_CROSSPREFIX), filename); - if (tcc_add_dll(s1, filename, 0) < 0) - tcc_error_noabort("%s not found", filename); -} - -#if !defined TCC_TARGET_PE && !defined TCC_TARGET_MACHO -ST_FUNC int tcc_add_crt(TCCState *s1, const char *filename) -{ - if (-1 == tcc_add_library_internal(s1, "%s/%s", - filename, 0, s1->crt_paths, s1->nb_crt_paths)) - return tcc_error_noabort("file '%s' not found", filename); - return 0; -} -#endif - -/* the library name is the same as the argument of the '-l' option */ -LIBTCCAPI int tcc_add_library(TCCState *s, const char *libraryname) -{ -#if defined TCC_TARGET_PE - static const char * const libs[] = { "%s/%s.def", "%s/lib%s.def", "%s/%s.dll", "%s/lib%s.dll", "%s/lib%s.a", NULL }; - const char * const *pp = s->static_link ? libs + 4 : libs; -#elif defined TCC_TARGET_MACHO - static const char * const libs[] = { "%s/lib%s.dylib", "%s/lib%s.tbd", "%s/lib%s.a", NULL }; - const char * const *pp = s->static_link ? libs + 2 : libs; -#elif defined TARGETOS_OpenBSD - static const char * const libs[] = { "%s/lib%s.so.*", "%s/lib%s.a", NULL }; - const char * const *pp = s->static_link ? libs + 1 : libs; -#else - static const char * const libs[] = { "%s/lib%s.so", "%s/lib%s.a", NULL }; - const char * const *pp = s->static_link ? libs + 1 : libs; -#endif - int flags = s->filetype & AFF_WHOLE_ARCHIVE; - while (*pp) { - if (0 == tcc_add_library_internal(s, *pp, - libraryname, flags, s->library_paths, s->nb_library_paths)) - return 0; - ++pp; - } - return -1; -} - -PUB_FUNC int tcc_add_library_err(TCCState *s1, const char *libname) -{ - int ret = tcc_add_library(s1, libname); - if (ret < 0) - tcc_error_noabort("library '%s' not found", libname); - return ret; -} - -/* handle #pragma comment(lib,) */ -ST_FUNC void tcc_add_pragma_libs(TCCState *s1) -{ - int i; - for (i = 0; i < s1->nb_pragma_libs; i++) - tcc_add_library_err(s1, s1->pragma_libs[i]); -} - -LIBTCCAPI int tcc_add_symbol(TCCState *s1, const char *name, const void *val) -{ -#ifdef TCC_TARGET_PE - /* On x86_64 'val' might not be reachable with a 32bit offset. - So it is handled here as if it were in a DLL. */ - pe_putimport(s1, 0, name, (uintptr_t)val); -#else - char buf[256]; - if (s1->leading_underscore) { - buf[0] = '_'; - pstrcpy(buf + 1, sizeof(buf) - 1, name); - name = buf; - } - set_global_sym(s1, name, NULL, (addr_t)(uintptr_t)val); /* NULL: SHN_ABS */ -#endif - return 0; -} - -LIBTCCAPI void tcc_set_lib_path(TCCState *s, const char *path) -{ - tcc_free(s->tcc_lib_path); - s->tcc_lib_path = tcc_strdup(path); -} - -/********************************************************/ -/* options parser */ - -static int strstart(const char *val, const char **str) -{ - const char *p, *q; - p = *str; - q = val; - while (*q) { - if (*p != *q) - return 0; - p++; - q++; - } - *str = p; - return 1; -} - -/* Like strstart, but automatically takes into account that ld options can - * - * - start with double or single dash (e.g. '--soname' or '-soname') - * - arguments can be given as separate or after '=' (e.g. '-Wl,-soname,x.so' - * or '-Wl,-soname=x.so') - * - * you provide `val` always in 'option[=]' form (no leading -) - */ -static int link_option(const char *str, const char *val, const char **ptr) -{ - const char *p, *q; - int ret; - - /* there should be 1 or 2 dashes */ - if (*str++ != '-') - return 0; - if (*str == '-') - str++; - - /* then str & val should match (potentially up to '=') */ - p = str; - q = val; - - ret = 1; - if (q[0] == '?') { - ++q; - if (strstart("no-", &p)) - ret = -1; - } - - while (*q != '\0' && *q != '=') { - if (*p != *q) - return 0; - p++; - q++; - } - - /* '=' near eos means ',' or '=' is ok */ - if (*q == '=') { - if (*p == 0) - *ptr = p; - if (*p != ',' && *p != '=') - return 0; - p++; - } else if (*p) { - return 0; - } - *ptr = p; - return ret; -} - -static const char *skip_linker_arg(const char **str) -{ - const char *s1 = *str; - const char *s2 = strchr(s1, ','); - *str = s2 ? s2++ : (s2 = s1 + strlen(s1)); - return s2; -} - -static void copy_linker_arg(char **pp, const char *s, int sep) -{ - const char *q = s; - char *p = *pp; - int l = 0; - if (p && sep) - p[l = strlen(p)] = sep, ++l; - skip_linker_arg(&q); - pstrncpy(l + (*pp = tcc_realloc(p, q - s + l + 1)), s, q - s); -} - -static void args_parser_add_file(TCCState *s, const char* filename, int filetype) -{ - struct filespec *f = tcc_malloc(sizeof *f + strlen(filename)); - f->type = filetype; - strcpy(f->name, filename); - dynarray_add(&s->files, &s->nb_files, f); -} - -/* set linker options */ -static int tcc_set_linker(TCCState *s, const char *option) -{ - TCCState *s1 = s; - while (*option) { - - const char *p = NULL; - char *end = NULL; - int ignoring = 0; - int ret; - - if (link_option(option, "Bsymbolic", &p)) { - s->symbolic = 1; - } else if (link_option(option, "nostdlib", &p)) { - s->nostdlib = 1; - } else if (link_option(option, "e=", &p) - || link_option(option, "entry=", &p)) { - copy_linker_arg(&s->elf_entryname, p, 0); - } else if (link_option(option, "fini=", &p)) { - copy_linker_arg(&s->fini_symbol, p, 0); - ignoring = 1; - } else if (link_option(option, "image-base=", &p) - || link_option(option, "Ttext=", &p)) { - s->text_addr = strtoull(p, &end, 16); - s->has_text_addr = 1; - } else if (link_option(option, "init=", &p)) { - copy_linker_arg(&s->init_symbol, p, 0); - ignoring = 1; - } else if (link_option(option, "Map=", &p)) { - copy_linker_arg(&s->mapfile, p, 0); - ignoring = 1; - } else if (link_option(option, "oformat=", &p)) { -#if defined(TCC_TARGET_PE) - if (strstart("pe-", &p)) { -#elif PTR_SIZE == 8 - if (strstart("elf64-", &p)) { -#else - if (strstart("elf32-", &p)) { -#endif - s->output_format = TCC_OUTPUT_FORMAT_ELF; - } else if (!strcmp(p, "binary")) { - s->output_format = TCC_OUTPUT_FORMAT_BINARY; -#ifdef TCC_TARGET_COFF - } else if (!strcmp(p, "coff")) { - s->output_format = TCC_OUTPUT_FORMAT_COFF; -#endif - } else - goto err; - - } else if (link_option(option, "as-needed", &p)) { - ignoring = 1; - } else if (link_option(option, "O", &p)) { - ignoring = 1; - } else if (link_option(option, "export-all-symbols", &p)) { - s->rdynamic = 1; - } else if (link_option(option, "export-dynamic", &p)) { - s->rdynamic = 1; - } else if (link_option(option, "rpath=", &p)) { - copy_linker_arg(&s->rpath, p, ':'); - } else if (link_option(option, "enable-new-dtags", &p)) { - s->enable_new_dtags = 1; - } else if (link_option(option, "section-alignment=", &p)) { - s->section_align = strtoul(p, &end, 16); - } else if (link_option(option, "soname=", &p)) { - copy_linker_arg(&s->soname, p, 0); - } else if (link_option(option, "install_name=", &p)) { - copy_linker_arg(&s->soname, p, 0); -#ifdef TCC_TARGET_PE - } else if (link_option(option, "large-address-aware", &p)) { - s->pe_characteristics |= 0x20; - } else if (link_option(option, "file-alignment=", &p)) { - s->pe_file_align = strtoul(p, &end, 16); - } else if (link_option(option, "stack=", &p)) { - s->pe_stack_size = strtoul(p, &end, 10); - } else if (link_option(option, "subsystem=", &p)) { -#if defined(TCC_TARGET_I386) || defined(TCC_TARGET_X86_64) - if (!strcmp(p, "native")) { - s->pe_subsystem = 1; - } else if (!strcmp(p, "console")) { - s->pe_subsystem = 3; - } else if (!strcmp(p, "gui") || !strcmp(p, "windows")) { - s->pe_subsystem = 2; - } else if (!strcmp(p, "posix")) { - s->pe_subsystem = 7; - } else if (!strcmp(p, "efiapp")) { - s->pe_subsystem = 10; - } else if (!strcmp(p, "efiboot")) { - s->pe_subsystem = 11; - } else if (!strcmp(p, "efiruntime")) { - s->pe_subsystem = 12; - } else if (!strcmp(p, "efirom")) { - s->pe_subsystem = 13; -#elif defined(TCC_TARGET_ARM) - if (!strcmp(p, "wince")) { - s->pe_subsystem = 9; -#endif - } else - goto err; -#endif -#ifdef TCC_TARGET_MACHO - } else if (link_option(option, "all_load", &p)) { - s->filetype |= AFF_WHOLE_ARCHIVE; - } else if (link_option(option, "force_load", &p)) { - s->filetype |= AFF_WHOLE_ARCHIVE; - args_parser_add_file(s, p, AFF_TYPE_LIB | (s->filetype & ~AFF_TYPE_MASK)); - s->nb_libraries++; - } else if (link_option(option, "single_module", &p)) { - ignoring = 1; -#endif - } else if (ret = link_option(option, "?whole-archive", &p), ret) { - if (ret > 0) - s->filetype |= AFF_WHOLE_ARCHIVE; - else - s->filetype &= ~AFF_WHOLE_ARCHIVE; - } else if (link_option(option, "z=", &p)) { - ignoring = 1; - } else if (p) { - return 0; - } else { - err: - return tcc_error_noabort("unsupported linker option '%s'", option); - } - if (ignoring) - tcc_warning_c(warn_unsupported)("unsupported linker option '%s'", option); - option = skip_linker_arg(&p); - } - return 1; -} - -typedef struct TCCOption { - const char *name; - uint16_t index; - uint16_t flags; -} TCCOption; - -enum { - TCC_OPTION_ignored = 0, - TCC_OPTION_HELP, - TCC_OPTION_HELP2, - TCC_OPTION_v, - TCC_OPTION_I, - TCC_OPTION_D, - TCC_OPTION_U, - TCC_OPTION_P, - TCC_OPTION_L, - TCC_OPTION_B, - TCC_OPTION_l, - TCC_OPTION_bench, - TCC_OPTION_bt, - TCC_OPTION_b, - TCC_OPTION_ba, - TCC_OPTION_g, - TCC_OPTION_c, - TCC_OPTION_dumpmachine, - TCC_OPTION_dumpversion, - TCC_OPTION_d, - TCC_OPTION_static, - TCC_OPTION_std, - TCC_OPTION_shared, - TCC_OPTION_soname, - TCC_OPTION_o, - TCC_OPTION_r, - TCC_OPTION_Wl, - TCC_OPTION_Wp, - TCC_OPTION_W, - TCC_OPTION_O, - TCC_OPTION_mfloat_abi, - TCC_OPTION_m, - TCC_OPTION_f, - TCC_OPTION_isystem, - TCC_OPTION_iwithprefix, - TCC_OPTION_include, - TCC_OPTION_nostdinc, - TCC_OPTION_nostdlib, - TCC_OPTION_print_search_dirs, - TCC_OPTION_rdynamic, - TCC_OPTION_pthread, - TCC_OPTION_run, - TCC_OPTION_w, - TCC_OPTION_E, - TCC_OPTION_M, - TCC_OPTION_MD, - TCC_OPTION_MF, - TCC_OPTION_MM, - TCC_OPTION_MMD, - TCC_OPTION_MP, - TCC_OPTION_x, - TCC_OPTION_ar, - TCC_OPTION_impdef, - TCC_OPTION_dynamiclib, - TCC_OPTION_flat_namespace, - TCC_OPTION_two_levelnamespace, - TCC_OPTION_undefined, - TCC_OPTION_install_name, - TCC_OPTION_compatibility_version , - TCC_OPTION_current_version, -}; - -#define TCC_OPTION_HAS_ARG 0x0001 -#define TCC_OPTION_NOSEP 0x0002 /* cannot have space before option and arg */ - -static const TCCOption tcc_options[] = { - { "h", TCC_OPTION_HELP, 0 }, - { "-help", TCC_OPTION_HELP, 0 }, - { "?", TCC_OPTION_HELP, 0 }, - { "hh", TCC_OPTION_HELP2, 0 }, - { "v", TCC_OPTION_v, TCC_OPTION_HAS_ARG | TCC_OPTION_NOSEP }, - { "-version", TCC_OPTION_v, 0 }, /* handle as verbose, also prints version*/ - { "I", TCC_OPTION_I, TCC_OPTION_HAS_ARG }, - { "D", TCC_OPTION_D, TCC_OPTION_HAS_ARG }, - { "U", TCC_OPTION_U, TCC_OPTION_HAS_ARG }, - { "P", TCC_OPTION_P, TCC_OPTION_HAS_ARG | TCC_OPTION_NOSEP }, - { "L", TCC_OPTION_L, TCC_OPTION_HAS_ARG }, - { "B", TCC_OPTION_B, TCC_OPTION_HAS_ARG }, - { "l", TCC_OPTION_l, TCC_OPTION_HAS_ARG }, - { "bench", TCC_OPTION_bench, 0 }, -#ifdef CONFIG_TCC_BACKTRACE - { "bt", TCC_OPTION_bt, TCC_OPTION_HAS_ARG | TCC_OPTION_NOSEP }, -#endif -#ifdef CONFIG_TCC_BCHECK - { "b", TCC_OPTION_b, 0 }, -#endif - { "g", TCC_OPTION_g, TCC_OPTION_HAS_ARG | TCC_OPTION_NOSEP }, -#ifdef TCC_TARGET_MACHO - { "compatibility_version", TCC_OPTION_compatibility_version, TCC_OPTION_HAS_ARG }, - { "current_version", TCC_OPTION_current_version, TCC_OPTION_HAS_ARG }, -#endif - { "c", TCC_OPTION_c, 0 }, -#ifdef TCC_TARGET_MACHO - { "dynamiclib", TCC_OPTION_dynamiclib, 0 }, -#endif - { "dumpmachine", TCC_OPTION_dumpmachine, 0}, - { "dumpversion", TCC_OPTION_dumpversion, 0}, - { "d", TCC_OPTION_d, TCC_OPTION_HAS_ARG | TCC_OPTION_NOSEP }, - { "static", TCC_OPTION_static, 0 }, - { "std", TCC_OPTION_std, TCC_OPTION_HAS_ARG | TCC_OPTION_NOSEP }, - { "shared", TCC_OPTION_shared, 0 }, - { "soname", TCC_OPTION_soname, TCC_OPTION_HAS_ARG }, - { "o", TCC_OPTION_o, TCC_OPTION_HAS_ARG }, - { "pthread", TCC_OPTION_pthread, 0}, - { "run", TCC_OPTION_run, TCC_OPTION_HAS_ARG | TCC_OPTION_NOSEP }, - { "rdynamic", TCC_OPTION_rdynamic, 0 }, - { "r", TCC_OPTION_r, 0 }, - { "Wl,", TCC_OPTION_Wl, TCC_OPTION_HAS_ARG | TCC_OPTION_NOSEP }, - { "Wp,", TCC_OPTION_Wp, TCC_OPTION_HAS_ARG | TCC_OPTION_NOSEP }, - { "W", TCC_OPTION_W, TCC_OPTION_HAS_ARG | TCC_OPTION_NOSEP }, - { "O", TCC_OPTION_O, TCC_OPTION_HAS_ARG | TCC_OPTION_NOSEP }, -#ifdef TCC_TARGET_ARM - { "mfloat-abi", TCC_OPTION_mfloat_abi, TCC_OPTION_HAS_ARG }, -#endif - { "m", TCC_OPTION_m, TCC_OPTION_HAS_ARG | TCC_OPTION_NOSEP }, -#ifdef TCC_TARGET_MACHO - { "flat_namespace", TCC_OPTION_flat_namespace, 0 }, -#endif - { "f", TCC_OPTION_f, TCC_OPTION_HAS_ARG | TCC_OPTION_NOSEP }, - { "isystem", TCC_OPTION_isystem, TCC_OPTION_HAS_ARG }, - { "include", TCC_OPTION_include, TCC_OPTION_HAS_ARG }, - { "nostdinc", TCC_OPTION_nostdinc, 0 }, - { "nostdlib", TCC_OPTION_nostdlib, 0 }, - { "print-search-dirs", TCC_OPTION_print_search_dirs, 0 }, - { "w", TCC_OPTION_w, 0 }, - { "E", TCC_OPTION_E, 0}, - { "M", TCC_OPTION_M, 0}, - { "MD", TCC_OPTION_MD, 0}, - { "MF", TCC_OPTION_MF, TCC_OPTION_HAS_ARG }, - { "MM", TCC_OPTION_MM, 0}, - { "MMD", TCC_OPTION_MMD, 0}, - { "MP", TCC_OPTION_MP, 0}, - { "x", TCC_OPTION_x, TCC_OPTION_HAS_ARG }, - { "ar", TCC_OPTION_ar, 0}, -#ifdef TCC_TARGET_PE - { "impdef", TCC_OPTION_impdef, 0}, -#endif -#ifdef TCC_TARGET_MACHO - { "install_name", TCC_OPTION_install_name, TCC_OPTION_HAS_ARG }, - { "two_levelnamespace", TCC_OPTION_two_levelnamespace, 0 }, - { "undefined", TCC_OPTION_undefined, TCC_OPTION_HAS_ARG }, -#endif - /* ignored (silently, except after -Wunsupported) */ - { "arch", 0, TCC_OPTION_HAS_ARG}, - { "C", 0, 0 }, - { "-param", 0, TCC_OPTION_HAS_ARG }, - { "pedantic", 0, 0 }, - { "pipe", 0, 0 }, - { "s", 0, 0 }, - { "traditional", 0, 0 }, - { NULL, 0, 0 }, -}; - -typedef struct FlagDef { - uint16_t offset; - uint16_t flags; - const char *name; -} FlagDef; - -#define WD_ALL 0x0001 /* warning is activated when using -Wall */ -#define FD_INVERT 0x0002 /* invert value before storing */ - -static const FlagDef options_W[] = { - { offsetof(TCCState, warn_all), WD_ALL, "all" }, - { offsetof(TCCState, warn_error), 0, "error" }, - { offsetof(TCCState, warn_write_strings), 0, "write-strings" }, - { offsetof(TCCState, warn_unsupported), 0, "unsupported" }, - { offsetof(TCCState, warn_implicit_function_declaration), WD_ALL, "implicit-function-declaration" }, - { offsetof(TCCState, warn_discarded_qualifiers), WD_ALL, "discarded-qualifiers" }, - { 0, 0, NULL } -}; - -static const FlagDef options_f[] = { - { offsetof(TCCState, char_is_unsigned), 0, "unsigned-char" }, - { offsetof(TCCState, char_is_unsigned), FD_INVERT, "signed-char" }, - { offsetof(TCCState, nocommon), FD_INVERT, "common" }, - { offsetof(TCCState, leading_underscore), 0, "leading-underscore" }, - { offsetof(TCCState, ms_extensions), 0, "ms-extensions" }, - { offsetof(TCCState, dollars_in_identifiers), 0, "dollars-in-identifiers" }, - { offsetof(TCCState, test_coverage), 0, "test-coverage" }, - { 0, 0, NULL } -}; - -static const FlagDef options_m[] = { - { offsetof(TCCState, ms_bitfields), 0, "ms-bitfields" }, -#ifdef TCC_TARGET_X86_64 - { offsetof(TCCState, nosse), FD_INVERT, "sse" }, -#endif - { 0, 0, NULL } -}; - -static int set_flag(TCCState *s, const FlagDef *flags, const char *name) -{ - int value, mask, ret; - const FlagDef *p; - const char *r; - unsigned char *f; - - r = name, value = !strstart("no-", &r), mask = 0; - - /* when called with options_W, look for -W[no-]error=